From e540d56d372934524862961360106cfda5694e17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Thu, 8 Nov 2018 00:02:11 +0100 Subject: [PATCH 001/135] Simple docker-compose containing activemq docker from webcenter. #7 --- docker-compose.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..32fa58f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,14 @@ +activemq: + image: webcenter/activemq:latest + ports: + - 8161:8161 + - 61616:61616 + - 61613:61613 + environment: + ACTIVEMQ_NAME: amq + ACTIVEMQ_REMOVE_DEFAULT_ACCOUNT: 'True' + ACTIVEMQ_ADMIN_LOGIN: admin + ACTIVEMQ_ADMIN_PASSWORD: admin + volumes: + - /container_data/activemq/data:/data/activemq + - /container_data/activemq/log:/var/log/activemq \ No newline at end of file From d0e328255ab885c687d40f6a8a42c462e7c1dced Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Fri, 9 Nov 2018 11:50:39 +0100 Subject: [PATCH 002/135] Added scheduler to run application continuously. #18 --- .../AnnouncementProcessorParserApplication.java | 2 ++ .../src/main/java/parser/Comp.java | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 announcement-processor-parser/src/main/java/parser/Comp.java diff --git a/announcement-processor-parser/src/main/java/parser/AnnouncementProcessorParserApplication.java b/announcement-processor-parser/src/main/java/parser/AnnouncementProcessorParserApplication.java index 3a48b93..2218fbd 100644 --- a/announcement-processor-parser/src/main/java/parser/AnnouncementProcessorParserApplication.java +++ b/announcement-processor-parser/src/main/java/parser/AnnouncementProcessorParserApplication.java @@ -4,9 +4,11 @@ import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.scheduling.annotation.EnableScheduling; @Log @SpringBootApplication +@EnableScheduling public class AnnouncementProcessorParserApplication implements CommandLineRunner { public static void main(String[] args) { SpringApplication app = new SpringApplication(AnnouncementProcessorParserApplication.class); diff --git a/announcement-processor-parser/src/main/java/parser/Comp.java b/announcement-processor-parser/src/main/java/parser/Comp.java new file mode 100644 index 0000000..e5f3db9 --- /dev/null +++ b/announcement-processor-parser/src/main/java/parser/Comp.java @@ -0,0 +1,16 @@ +package parser; + +import lombok.extern.java.Log; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Controller; + +import java.time.LocalDateTime; + +@Controller +@Log +public class Comp { + @Scheduled(fixedRate = 5000) + public void reportCurrentTime() { + log.info("The time is now: " + LocalDateTime.now().toString()); + } +} From 344cf770c50f4fb932f9881c372a53e99c2206a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Fri, 9 Nov 2018 11:51:07 +0100 Subject: [PATCH 003/135] Changed server port to 8081. #18 --- .../src/main/resources/application.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/announcement-processor-parser/src/main/resources/application.properties b/announcement-processor-parser/src/main/resources/application.properties index e69de29..b988063 100644 --- a/announcement-processor-parser/src/main/resources/application.properties +++ b/announcement-processor-parser/src/main/resources/application.properties @@ -0,0 +1 @@ +server.port = 8081 \ No newline at end of file From 7e7cb3434b81f534ca4348f32d44d3bf48f749d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Fri, 9 Nov 2018 11:51:36 +0100 Subject: [PATCH 004/135] Changed logging format type. #18 --- announcement-processor-parser/src/main/resources/logback.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/announcement-processor-parser/src/main/resources/logback.xml b/announcement-processor-parser/src/main/resources/logback.xml index 5cf7b69..5097d20 100644 --- a/announcement-processor-parser/src/main/resources/logback.xml +++ b/announcement-processor-parser/src/main/resources/logback.xml @@ -4,11 +4,11 @@ - PARSER - %d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n + %d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n - + From b038a28d88ca112bd5ce74d42286619d83751696 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Fri, 9 Nov 2018 11:52:47 +0100 Subject: [PATCH 005/135] Added Dockerfile and applied it to docker-compose. #18 --- announcement-processor-parser/Dockerfile | 14 ++++++++ docker-compose.yml | 42 ++++++++++++++++-------- 2 files changed, 42 insertions(+), 14 deletions(-) create mode 100644 announcement-processor-parser/Dockerfile diff --git a/announcement-processor-parser/Dockerfile b/announcement-processor-parser/Dockerfile new file mode 100644 index 0000000..084b583 --- /dev/null +++ b/announcement-processor-parser/Dockerfile @@ -0,0 +1,14 @@ +FROM openjdk:8-jdk-alpine + +RUN apk update +RUN apk add maven + +RUN mkdir -p /srv/announcement-processor/parser +WORKDIR /srv/announcement-processor/parser + +ADD pom.xml /srv/announcement-processor/parser +COPY src /srv/announcement-processor/parser/src + +RUN ["mvn", "clean", "install", "-DskipTests"] + +CMD java -jar /srv/announcement-processor/parser/target/parser-service.jar \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 32fa58f..f508933 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,14 +1,28 @@ -activemq: - image: webcenter/activemq:latest - ports: - - 8161:8161 - - 61616:61616 - - 61613:61613 - environment: - ACTIVEMQ_NAME: amq - ACTIVEMQ_REMOVE_DEFAULT_ACCOUNT: 'True' - ACTIVEMQ_ADMIN_LOGIN: admin - ACTIVEMQ_ADMIN_PASSWORD: admin - volumes: - - /container_data/activemq/data:/data/activemq - - /container_data/activemq/log:/var/log/activemq \ No newline at end of file +version: '3' + +services: + + activemq: + image: webcenter/activemq:latest + container_name: activemq-service + ports: + - 8161:8161 + - 61616:61616 + - 61613:61613 + environment: + ACTIVEMQ_NAME: amq + ACTIVEMQ_REMOVE_DEFAULT_ACCOUNT: 'True' + ACTIVEMQ_ADMIN_LOGIN: admin + ACTIVEMQ_ADMIN_PASSWORD: admin + volumes: + - /container_data/activemq/data:/data/activemq + - /container_data/activemq/log:/var/log/activemq + + parser: + build: ./announcement-processor-parser + container_name: parser-service + ports: + - 8081:8081 + volumes: + - ./announcement-processor-parser/src:/srv/announcement-processor/parser/src + command: java -jar ./target/parser-service.jar From e19da9dd25132b5dc169c5233becbf070cdceb47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Fri, 9 Nov 2018 23:12:01 +0100 Subject: [PATCH 006/135] Added configuration for CircleCI --- .circleci/config.yml | 32 +++++++++++++++++++++++++++ announcement-processor-parser/pom.xml | 7 ++++++ 2 files changed, 39 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..31d2cf2 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,32 @@ +version: 2 +jobs: + parser-test: + docker: + - image: circleci/openjdk:8-jdk-browsers + + steps: + - checkout + + - restore_cache: # restore the saved cache after the first run or if `pom.xml` has changed + key: circleci-announcement-processor-parser-{{ checksum "announcement-processor-parser/pom.xml" }} + + - run: cd announcement-processor-parser && mvn dependency:go-offline # gets the project dependencies + + - save_cache: # saves the project dependencies + paths: + - ~/.m2 + key: circleci-announcement-processor-parser-{{ checksum "announcement-processor-parser/pom.xml" }} + + - run: cd announcement-processor-parser && mvn package # run the actual tests + + - store_test_results: # uploads the test metadata from the `target/surefire-reports` directory so that it can show up in the CircleCI dashboard. + path: announcement-processor-parser/target/surefire-reports + + - store_artifacts: + path: announcement-processor-parser/target/parser-service.jar + +workflows: + version: 2 + announcement_processor: + jobs: + - parser-test \ No newline at end of file diff --git a/announcement-processor-parser/pom.xml b/announcement-processor-parser/pom.xml index 6d5c26f..27ecc38 100644 --- a/announcement-processor-parser/pom.xml +++ b/announcement-processor-parser/pom.xml @@ -70,6 +70,13 @@ org.springframework.boot spring-boot-maven-plugin + + org.apache.maven.plugins + maven-surefire-plugin + + false + + org.jacoco jacoco-maven-plugin From cfeac32a775db2c260d76af62103f58d87969070 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Sroka?= Date: Sat, 10 Nov 2018 00:13:44 +0100 Subject: [PATCH 007/135] Add extractor sproject structure --- announcement-processor-extractor/.gitignore | 27 +++++ announcement-processor-extractor/pom.xml | 104 ++++++++++++++++++ ...uncementProcessorExtractorApplication.java | 23 ++++ .../src/main/java/extractor/Comp.java | 16 +++ .../src/main/resources/application.properties | 0 .../src/main/resources/logback.xml | 0 6 files changed, 170 insertions(+) create mode 100644 announcement-processor-extractor/.gitignore create mode 100644 announcement-processor-extractor/pom.xml create mode 100644 announcement-processor-extractor/src/main/java/extractor/AnnouncementProcessorExtractorApplication.java create mode 100644 announcement-processor-extractor/src/main/java/extractor/Comp.java create mode 100644 announcement-processor-extractor/src/main/resources/application.properties create mode 100644 announcement-processor-extractor/src/main/resources/logback.xml diff --git a/announcement-processor-extractor/.gitignore b/announcement-processor-extractor/.gitignore new file mode 100644 index 0000000..c4702dd --- /dev/null +++ b/announcement-processor-extractor/.gitignore @@ -0,0 +1,27 @@ +/target/ +.mvn +mvnw +mvnw.cmd + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/build/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ \ No newline at end of file diff --git a/announcement-processor-extractor/pom.xml b/announcement-processor-extractor/pom.xml new file mode 100644 index 0000000..4e6d47b --- /dev/null +++ b/announcement-processor-extractor/pom.xml @@ -0,0 +1,104 @@ + + + 4.0.0 + + announcement-processor + extractor + 0.0.1-SNAPSHOT + jar + Announcement processor extractor + + + org.springframework.boot + spring-boot-starter-parent + 2.1.0.RELEASE + + + + + UTF-8 + UTF-8 + 1.8 + + + + + org.springframework.boot + spring-boot-starter + + + org.springframework.boot + spring-boot-starter-test + test + + + + + org.apache.activemq + activemq-client + 5.15.7 + + + + + org.projectlombok + lombok + true + + + + + org.mockito + mockito-all + 2.0.2-beta + test + + + org.jacoco + jacoco-maven-plugin + 0.8.2 + + + + + + parser-service + + + + org.springframework.boot + spring-boot-maven-plugin + + + org.jacoco + jacoco-maven-plugin + 0.8.2 + + + + prepare-agent + + + + report + test + + report + + + + + extractor/AnnouncementProcessorExtractorApplication.java + + + + + + + + + + + \ No newline at end of file diff --git a/announcement-processor-extractor/src/main/java/extractor/AnnouncementProcessorExtractorApplication.java b/announcement-processor-extractor/src/main/java/extractor/AnnouncementProcessorExtractorApplication.java new file mode 100644 index 0000000..ed56a2b --- /dev/null +++ b/announcement-processor-extractor/src/main/java/extractor/AnnouncementProcessorExtractorApplication.java @@ -0,0 +1,23 @@ +package extractor; + + +import lombok.extern.java.Log; +import org.springframework.boot.CommandLineRunner; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.scheduling.annotation.EnableScheduling; + +@Log +@SpringBootApplication +@EnableScheduling +public class AnnouncementProcessorExtractorApplication implements CommandLineRunner { + + public static void main(String[] args) { + SpringApplication app = new SpringApplication(AnnouncementProcessorExtractorApplication.class); + } + + @Override + public void run(String... args) throws Exception { + + } +} diff --git a/announcement-processor-extractor/src/main/java/extractor/Comp.java b/announcement-processor-extractor/src/main/java/extractor/Comp.java new file mode 100644 index 0000000..2e5ac5c --- /dev/null +++ b/announcement-processor-extractor/src/main/java/extractor/Comp.java @@ -0,0 +1,16 @@ +package extractor; + +import lombok.extern.java.Log; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Controller; + +import java.time.LocalDateTime; + +@Controller +@Log +public class Comp { + @Scheduled(fixedRate = 5000) + public void reportCurrentTime() { + log.info("The time is now: " + LocalDateTime.now().toString()); + } +} \ No newline at end of file diff --git a/announcement-processor-extractor/src/main/resources/application.properties b/announcement-processor-extractor/src/main/resources/application.properties new file mode 100644 index 0000000..e69de29 diff --git a/announcement-processor-extractor/src/main/resources/logback.xml b/announcement-processor-extractor/src/main/resources/logback.xml new file mode 100644 index 0000000..e69de29 From 83babbf86c8ee8d4de7d418bac8a373ac0fafe28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Sroka?= Date: Sat, 10 Nov 2018 00:17:57 +0100 Subject: [PATCH 008/135] Add logback config --- .../src/main/resources/logback.xml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/announcement-processor-extractor/src/main/resources/logback.xml b/announcement-processor-extractor/src/main/resources/logback.xml index e69de29..5097d20 100644 --- a/announcement-processor-extractor/src/main/resources/logback.xml +++ b/announcement-processor-extractor/src/main/resources/logback.xml @@ -0,0 +1,18 @@ + + + + + + + %d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + \ No newline at end of file From dd93ff25b3b3d67bb4659b39ccbee9c7cd75e47d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Sroka?= Date: Sat, 10 Nov 2018 00:35:54 +0100 Subject: [PATCH 009/135] Add docker file --- announcement-processor-extractor/Dockerfile | 14 ++++++++++++++ docker-compose.yml | 15 +++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 announcement-processor-extractor/Dockerfile diff --git a/announcement-processor-extractor/Dockerfile b/announcement-processor-extractor/Dockerfile new file mode 100644 index 0000000..3fc7ed5 --- /dev/null +++ b/announcement-processor-extractor/Dockerfile @@ -0,0 +1,14 @@ +FROM openjdk:8-jdk-alpine + +RUN apk update +RUN apk add maven + +RUN mkdir -p /srv/announcement-processor/extractor +WORKDIR /srv/announcement-processor/extractor + +ADD pom.xml /srv/announcement-processor/extractor +COPY src /srv/announcement-processor/extractor/src + +RUN ["mvn", "clean", "install", "-DskipTests"] + +CMD java -jar /srv/announcement-processor/extractor/target/extractor-service.jar diff --git a/docker-compose.yml b/docker-compose.yml index f508933..238c0d9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,5 +24,16 @@ services: ports: - 8081:8081 volumes: - - ./announcement-processor-parser/src:/srv/announcement-processor/parser/src - command: java -jar ./target/parser-service.jar + - ./announcement-processor-parser/src:/srv/announcement-processor/parser/src + command: java -jar ./target/parser-service.jar + + extractor: + build: ./announcement-processor-extractor + container_name: extractor-service + ports: + - 8082:8082 + volumes: + - ./announcement-processor-extractor/src:/srv/announcement-extractor/extractor/src + + command: java -jar ./target/extractor-service.jar + From a26c8a40b073abe0896a220dc50b4311f7a0c3a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sat, 10 Nov 2018 21:18:03 +0100 Subject: [PATCH 010/135] Created simple project structure. #6 --- announcement-processor-api/.gitignore | 27 +++++ announcement-processor-api/pom.xml | 108 ++++++++++++++++++ .../AnnouncementProcessorApiApplication.java | 12 ++ .../src/main/resources/application.properties | 0 .../src/main/resources/logback.xml | 18 +++ ...ouncementProcessorApiApplicationTests.java | 16 +++ 6 files changed, 181 insertions(+) create mode 100644 announcement-processor-api/.gitignore create mode 100644 announcement-processor-api/pom.xml create mode 100644 announcement-processor-api/src/main/java/api/AnnouncementProcessorApiApplication.java create mode 100644 announcement-processor-api/src/main/resources/application.properties create mode 100644 announcement-processor-api/src/main/resources/logback.xml create mode 100644 announcement-processor-api/src/test/java/api/AnnouncementProcessorApiApplicationTests.java diff --git a/announcement-processor-api/.gitignore b/announcement-processor-api/.gitignore new file mode 100644 index 0000000..c4702dd --- /dev/null +++ b/announcement-processor-api/.gitignore @@ -0,0 +1,27 @@ +/target/ +.mvn +mvnw +mvnw.cmd + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/build/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ \ No newline at end of file diff --git a/announcement-processor-api/pom.xml b/announcement-processor-api/pom.xml new file mode 100644 index 0000000..9dd0bdf --- /dev/null +++ b/announcement-processor-api/pom.xml @@ -0,0 +1,108 @@ + + + 4.0.0 + + announcement-processor + api + 0.0.1-SNAPSHOT + jar + Announcement processor api + + + org.springframework.boot + spring-boot-starter-parent + 2.1.0.RELEASE + + + + + UTF-8 + UTF-8 + 1.8 + + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-devtools + runtime + + + org.springframework.boot + spring-boot-starter-test + test + + + + + org.projectlombok + lombok + true + + + + + org.mockito + mockito-all + 2.0.2-beta + test + + + org.jacoco + jacoco-maven-plugin + 0.8.2 + + + + + + api-service + + + + org.springframework.boot + spring-boot-maven-plugin + + + org.apache.maven.plugins + maven-surefire-plugin + + false + + + + org.jacoco + jacoco-maven-plugin + 0.8.2 + + + + prepare-agent + + + + report + test + + report + + + + + api/AnnouncementProcessorApiApplication.class + + + + + + + + + + + diff --git a/announcement-processor-api/src/main/java/api/AnnouncementProcessorApiApplication.java b/announcement-processor-api/src/main/java/api/AnnouncementProcessorApiApplication.java new file mode 100644 index 0000000..2b09fea --- /dev/null +++ b/announcement-processor-api/src/main/java/api/AnnouncementProcessorApiApplication.java @@ -0,0 +1,12 @@ +package api; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class AnnouncementProcessorApiApplication { + + public static void main(String[] args) { + SpringApplication.run(AnnouncementProcessorApiApplication.class, args); + } +} diff --git a/announcement-processor-api/src/main/resources/application.properties b/announcement-processor-api/src/main/resources/application.properties new file mode 100644 index 0000000..e69de29 diff --git a/announcement-processor-api/src/main/resources/logback.xml b/announcement-processor-api/src/main/resources/logback.xml new file mode 100644 index 0000000..5097d20 --- /dev/null +++ b/announcement-processor-api/src/main/resources/logback.xml @@ -0,0 +1,18 @@ + + + + + + + %d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + \ No newline at end of file diff --git a/announcement-processor-api/src/test/java/api/AnnouncementProcessorApiApplicationTests.java b/announcement-processor-api/src/test/java/api/AnnouncementProcessorApiApplicationTests.java new file mode 100644 index 0000000..2b5751e --- /dev/null +++ b/announcement-processor-api/src/test/java/api/AnnouncementProcessorApiApplicationTests.java @@ -0,0 +1,16 @@ +package api; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +@SpringBootTest +public class AnnouncementProcessorApiApplicationTests { + + @Test + public void contextLoads() { + } + +} From 95f2f4684acc12712f108a614c4db0c7d5263651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sat, 10 Nov 2018 21:53:02 +0100 Subject: [PATCH 011/135] Added H2 database for dev usage and mysql for prod usage. #6 --- announcement-processor-api/.gitignore | 2 ++ announcement-processor-api/pom.xml | 20 +++++++++++++++++++ .../main/resources/application-dev.properties | 11 ++++++++++ .../src/main/resources/application.properties | 1 + 4 files changed, 34 insertions(+) create mode 100644 announcement-processor-api/src/main/resources/application-dev.properties diff --git a/announcement-processor-api/.gitignore b/announcement-processor-api/.gitignore index c4702dd..20e9cad 100644 --- a/announcement-processor-api/.gitignore +++ b/announcement-processor-api/.gitignore @@ -3,6 +3,8 @@ mvnw mvnw.cmd +src/main/resources/application-prod.properties + ### STS ### .apt_generated .classpath diff --git a/announcement-processor-api/pom.xml b/announcement-processor-api/pom.xml index 9dd0bdf..bff3df2 100644 --- a/announcement-processor-api/pom.xml +++ b/announcement-processor-api/pom.xml @@ -57,6 +57,26 @@ jacoco-maven-plugin 0.8.2 + + + + org.springframework.data + spring-data-jpa + 2.1.2.RELEASE + + + mysql + mysql-connector-java + + + com.h2database + h2 + 1.4.197 + + + org.springframework.boot + spring-boot-starter-data-jpa + diff --git a/announcement-processor-api/src/main/resources/application-dev.properties b/announcement-processor-api/src/main/resources/application-dev.properties new file mode 100644 index 0000000..4204610 --- /dev/null +++ b/announcement-processor-api/src/main/resources/application-dev.properties @@ -0,0 +1,11 @@ +# H2 +spring.h2.console.enabled=true +spring.h2.console.path=/h2 + +# Datasource +spring.datasource.url=jdbc:h2:file:~/dev +spring.datasource.username=sa +spring.datasource.password= +spring.datasource.driver-class-name=org.h2.Driver + +spring.jpa.hibernate.ddl-auto=create \ No newline at end of file diff --git a/announcement-processor-api/src/main/resources/application.properties b/announcement-processor-api/src/main/resources/application.properties index e69de29..426b904 100644 --- a/announcement-processor-api/src/main/resources/application.properties +++ b/announcement-processor-api/src/main/resources/application.properties @@ -0,0 +1 @@ +spring.profiles.default=dev \ No newline at end of file From c7d26db74fad3ccbba753639d70f8e7e949e9757 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sat, 10 Nov 2018 21:57:50 +0100 Subject: [PATCH 012/135] Added api-service to CircleCI pipe. #6 --- .circleci/config.yml | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 31d2cf2..7abf206 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,8 +25,34 @@ jobs: - store_artifacts: path: announcement-processor-parser/target/parser-service.jar + api-test: + docker: + - image: circleci/openjdk:8-jdk-browsers + + steps: + - checkout + + - restore_cache: # restore the saved cache after the first run or if `pom.xml` has changed + key: circleci-announcement-processor-api-{{ checksum "announcement-processor-api/pom.xml" }} + + - run: cd announcement-processor-api && mvn dependency:go-offline # gets the project dependencies + + - save_cache: # saves the project dependencies + paths: + - ~/.m2 + key: circleci-announcement-processor-api-{{ checksum "announcement-processor-api/pom.xml" }} + + - run: cd announcement-processor-api && mvn package # run the actual tests + + - store_test_results: # uploads the test metadata from the `target/surefire-reports` directory so that it can show up in the CircleCI dashboard. + path: announcement-processor-api/target/surefire-reports + + - store_artifacts: + path: announcement-processor-api/target/api-service.jar + workflows: version: 2 announcement_processor: jobs: - - parser-test \ No newline at end of file + - parser-test + - api-test \ No newline at end of file From af61c8addcc830fbbda90c8b386be219d58e3f66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Fri, 9 Nov 2018 23:12:01 +0100 Subject: [PATCH 013/135] Added configuration for CircleCI --- .circleci/config.yml | 32 +++++++++++++++++++++++++++ announcement-processor-parser/pom.xml | 7 ++++++ 2 files changed, 39 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..31d2cf2 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,32 @@ +version: 2 +jobs: + parser-test: + docker: + - image: circleci/openjdk:8-jdk-browsers + + steps: + - checkout + + - restore_cache: # restore the saved cache after the first run or if `pom.xml` has changed + key: circleci-announcement-processor-parser-{{ checksum "announcement-processor-parser/pom.xml" }} + + - run: cd announcement-processor-parser && mvn dependency:go-offline # gets the project dependencies + + - save_cache: # saves the project dependencies + paths: + - ~/.m2 + key: circleci-announcement-processor-parser-{{ checksum "announcement-processor-parser/pom.xml" }} + + - run: cd announcement-processor-parser && mvn package # run the actual tests + + - store_test_results: # uploads the test metadata from the `target/surefire-reports` directory so that it can show up in the CircleCI dashboard. + path: announcement-processor-parser/target/surefire-reports + + - store_artifacts: + path: announcement-processor-parser/target/parser-service.jar + +workflows: + version: 2 + announcement_processor: + jobs: + - parser-test \ No newline at end of file diff --git a/announcement-processor-parser/pom.xml b/announcement-processor-parser/pom.xml index 6d5c26f..27ecc38 100644 --- a/announcement-processor-parser/pom.xml +++ b/announcement-processor-parser/pom.xml @@ -70,6 +70,13 @@ org.springframework.boot spring-boot-maven-plugin + + org.apache.maven.plugins + maven-surefire-plugin + + false + + org.jacoco jacoco-maven-plugin From cdca4b3670fc789af0fb69607ad5413c6ef0ca8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Sroka?= Date: Sat, 10 Nov 2018 00:13:44 +0100 Subject: [PATCH 014/135] Add extractor sproject structure --- announcement-processor-extractor/.gitignore | 27 +++++ announcement-processor-extractor/pom.xml | 104 ++++++++++++++++++ ...uncementProcessorExtractorApplication.java | 23 ++++ .../src/main/java/extractor/Comp.java | 16 +++ .../src/main/resources/application.properties | 0 .../src/main/resources/logback.xml | 0 6 files changed, 170 insertions(+) create mode 100644 announcement-processor-extractor/.gitignore create mode 100644 announcement-processor-extractor/pom.xml create mode 100644 announcement-processor-extractor/src/main/java/extractor/AnnouncementProcessorExtractorApplication.java create mode 100644 announcement-processor-extractor/src/main/java/extractor/Comp.java create mode 100644 announcement-processor-extractor/src/main/resources/application.properties create mode 100644 announcement-processor-extractor/src/main/resources/logback.xml diff --git a/announcement-processor-extractor/.gitignore b/announcement-processor-extractor/.gitignore new file mode 100644 index 0000000..c4702dd --- /dev/null +++ b/announcement-processor-extractor/.gitignore @@ -0,0 +1,27 @@ +/target/ +.mvn +mvnw +mvnw.cmd + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/build/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ \ No newline at end of file diff --git a/announcement-processor-extractor/pom.xml b/announcement-processor-extractor/pom.xml new file mode 100644 index 0000000..4e6d47b --- /dev/null +++ b/announcement-processor-extractor/pom.xml @@ -0,0 +1,104 @@ + + + 4.0.0 + + announcement-processor + extractor + 0.0.1-SNAPSHOT + jar + Announcement processor extractor + + + org.springframework.boot + spring-boot-starter-parent + 2.1.0.RELEASE + + + + + UTF-8 + UTF-8 + 1.8 + + + + + org.springframework.boot + spring-boot-starter + + + org.springframework.boot + spring-boot-starter-test + test + + + + + org.apache.activemq + activemq-client + 5.15.7 + + + + + org.projectlombok + lombok + true + + + + + org.mockito + mockito-all + 2.0.2-beta + test + + + org.jacoco + jacoco-maven-plugin + 0.8.2 + + + + + + parser-service + + + + org.springframework.boot + spring-boot-maven-plugin + + + org.jacoco + jacoco-maven-plugin + 0.8.2 + + + + prepare-agent + + + + report + test + + report + + + + + extractor/AnnouncementProcessorExtractorApplication.java + + + + + + + + + + + \ No newline at end of file diff --git a/announcement-processor-extractor/src/main/java/extractor/AnnouncementProcessorExtractorApplication.java b/announcement-processor-extractor/src/main/java/extractor/AnnouncementProcessorExtractorApplication.java new file mode 100644 index 0000000..ed56a2b --- /dev/null +++ b/announcement-processor-extractor/src/main/java/extractor/AnnouncementProcessorExtractorApplication.java @@ -0,0 +1,23 @@ +package extractor; + + +import lombok.extern.java.Log; +import org.springframework.boot.CommandLineRunner; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.scheduling.annotation.EnableScheduling; + +@Log +@SpringBootApplication +@EnableScheduling +public class AnnouncementProcessorExtractorApplication implements CommandLineRunner { + + public static void main(String[] args) { + SpringApplication app = new SpringApplication(AnnouncementProcessorExtractorApplication.class); + } + + @Override + public void run(String... args) throws Exception { + + } +} diff --git a/announcement-processor-extractor/src/main/java/extractor/Comp.java b/announcement-processor-extractor/src/main/java/extractor/Comp.java new file mode 100644 index 0000000..2e5ac5c --- /dev/null +++ b/announcement-processor-extractor/src/main/java/extractor/Comp.java @@ -0,0 +1,16 @@ +package extractor; + +import lombok.extern.java.Log; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Controller; + +import java.time.LocalDateTime; + +@Controller +@Log +public class Comp { + @Scheduled(fixedRate = 5000) + public void reportCurrentTime() { + log.info("The time is now: " + LocalDateTime.now().toString()); + } +} \ No newline at end of file diff --git a/announcement-processor-extractor/src/main/resources/application.properties b/announcement-processor-extractor/src/main/resources/application.properties new file mode 100644 index 0000000..e69de29 diff --git a/announcement-processor-extractor/src/main/resources/logback.xml b/announcement-processor-extractor/src/main/resources/logback.xml new file mode 100644 index 0000000..e69de29 From 5f463b539535221091430cad029202cd0674bcf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Sroka?= Date: Sat, 10 Nov 2018 00:17:57 +0100 Subject: [PATCH 015/135] Add logback config --- .../src/main/resources/logback.xml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/announcement-processor-extractor/src/main/resources/logback.xml b/announcement-processor-extractor/src/main/resources/logback.xml index e69de29..5097d20 100644 --- a/announcement-processor-extractor/src/main/resources/logback.xml +++ b/announcement-processor-extractor/src/main/resources/logback.xml @@ -0,0 +1,18 @@ + + + + + + + %d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + \ No newline at end of file From 3592fc040d245bf628232c2e886d9752dc9b59f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Sroka?= Date: Sat, 10 Nov 2018 00:35:54 +0100 Subject: [PATCH 016/135] Add docker file --- announcement-processor-extractor/Dockerfile | 14 ++++++++++++++ docker-compose.yml | 15 +++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 announcement-processor-extractor/Dockerfile diff --git a/announcement-processor-extractor/Dockerfile b/announcement-processor-extractor/Dockerfile new file mode 100644 index 0000000..3fc7ed5 --- /dev/null +++ b/announcement-processor-extractor/Dockerfile @@ -0,0 +1,14 @@ +FROM openjdk:8-jdk-alpine + +RUN apk update +RUN apk add maven + +RUN mkdir -p /srv/announcement-processor/extractor +WORKDIR /srv/announcement-processor/extractor + +ADD pom.xml /srv/announcement-processor/extractor +COPY src /srv/announcement-processor/extractor/src + +RUN ["mvn", "clean", "install", "-DskipTests"] + +CMD java -jar /srv/announcement-processor/extractor/target/extractor-service.jar diff --git a/docker-compose.yml b/docker-compose.yml index f508933..238c0d9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,5 +24,16 @@ services: ports: - 8081:8081 volumes: - - ./announcement-processor-parser/src:/srv/announcement-processor/parser/src - command: java -jar ./target/parser-service.jar + - ./announcement-processor-parser/src:/srv/announcement-processor/parser/src + command: java -jar ./target/parser-service.jar + + extractor: + build: ./announcement-processor-extractor + container_name: extractor-service + ports: + - 8082:8082 + volumes: + - ./announcement-processor-extractor/src:/srv/announcement-extractor/extractor/src + + command: java -jar ./target/extractor-service.jar + From 71c18b6d2c06c6fa1c3ada0752db4a9e3e6528c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 11 Nov 2018 17:51:06 +0100 Subject: [PATCH 017/135] Fixed spring context was not starting. --- .../extractor/AnnouncementProcessorExtractorApplication.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/announcement-processor-extractor/src/main/java/extractor/AnnouncementProcessorExtractorApplication.java b/announcement-processor-extractor/src/main/java/extractor/AnnouncementProcessorExtractorApplication.java index ed56a2b..e4337fe 100644 --- a/announcement-processor-extractor/src/main/java/extractor/AnnouncementProcessorExtractorApplication.java +++ b/announcement-processor-extractor/src/main/java/extractor/AnnouncementProcessorExtractorApplication.java @@ -14,10 +14,11 @@ public class AnnouncementProcessorExtractorApplication implements CommandLineRun public static void main(String[] args) { SpringApplication app = new SpringApplication(AnnouncementProcessorExtractorApplication.class); + app.run(args); } @Override - public void run(String... args) throws Exception { + public void run(String... args) { } } From 91fd4084867812a6309b44f846b7aaa89f5a293f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 11 Nov 2018 17:51:42 +0100 Subject: [PATCH 018/135] Added example test to verify if jacoco works. --- .../src/test/java/parser/ExampleTest.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 announcement-processor-extractor/src/test/java/parser/ExampleTest.java diff --git a/announcement-processor-extractor/src/test/java/parser/ExampleTest.java b/announcement-processor-extractor/src/test/java/parser/ExampleTest.java new file mode 100644 index 0000000..09de7c0 --- /dev/null +++ b/announcement-processor-extractor/src/test/java/parser/ExampleTest.java @@ -0,0 +1,14 @@ +package parser; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class ExampleTest { + @Test + public void test(){ + Assert.assertTrue(true); + } +} From 1af637558db928924eee328d76a4186666014a99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 11 Nov 2018 18:38:15 +0100 Subject: [PATCH 019/135] Added .gitignore to the main directory --- .gitignore | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..20e9cad --- /dev/null +++ b/.gitignore @@ -0,0 +1,29 @@ +/target/ +.mvn +mvnw +mvnw.cmd + +src/main/resources/application-prod.properties + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/build/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ \ No newline at end of file From 31581a8d0f97fdecabdd20f0ad3f25ac5bf2df7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 11 Nov 2018 19:20:04 +0100 Subject: [PATCH 020/135] Added Dockerfile for api service. #20 --- announcement-processor-api/Dockerfile | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 announcement-processor-api/Dockerfile diff --git a/announcement-processor-api/Dockerfile b/announcement-processor-api/Dockerfile new file mode 100644 index 0000000..7cfa381 --- /dev/null +++ b/announcement-processor-api/Dockerfile @@ -0,0 +1,16 @@ +FROM openjdk:8-jdk-alpine + +RUN apk update +RUN apk add maven + +RUN mkdir -p /srv/announcement-processor/api +WORKDIR /srv/announcement-processor/api + +ADD pom.xml /srv/announcement-processor/api +COPY src /srv/announcement-processor/api/src + +RUN ["mvn", "clean", "install", "-DskipTests"] + +EXPOSE 8080 + +CMD java -jar -Dspring.profiles.active=dev /srv/announcement-processor/api/target/api-service.jar \ No newline at end of file From f6a93d65a849b37737bb26625970ae507564357c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 11 Nov 2018 19:20:49 +0100 Subject: [PATCH 021/135] Allow to view h2 console on docker_id:8080/h2. #20 --- .../src/main/resources/application-dev.properties | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/announcement-processor-api/src/main/resources/application-dev.properties b/announcement-processor-api/src/main/resources/application-dev.properties index 4204610..87c4b9f 100644 --- a/announcement-processor-api/src/main/resources/application-dev.properties +++ b/announcement-processor-api/src/main/resources/application-dev.properties @@ -8,4 +8,6 @@ spring.datasource.username=sa spring.datasource.password= spring.datasource.driver-class-name=org.h2.Driver -spring.jpa.hibernate.ddl-auto=create \ No newline at end of file +spring.jpa.hibernate.ddl-auto=create + +spring.h2.console.settings.web-allow-others=true \ No newline at end of file From 1ca69e1aaac9bdb9e856c477702fe8c59f33ffae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 11 Nov 2018 19:35:29 +0100 Subject: [PATCH 022/135] Updated docker-compose.yml according to new Dockerfile for api service. #20 --- docker-compose.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index f508933..b1cafd7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,3 +26,12 @@ services: volumes: - ./announcement-processor-parser/src:/srv/announcement-processor/parser/src command: java -jar ./target/parser-service.jar + + api: + build: ./announcement-processor-api + container_name: api-service + ports: + - 8080:8080 + volumes: + - ./announcement-processor-api/src:/srv/announcement-processor/api/src + command: java -jar -Dspring.profiles.active=dev ./target/api-service.jar From 4a9389bc80c33ed86064bcf480530da35c7d2c01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Sroka?= Date: Wed, 14 Nov 2018 20:52:06 +0100 Subject: [PATCH 023/135] Docker fixes --- announcement-processor-extractor/Dockerfile | 4 +++- announcement-processor-extractor/pom.xml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/announcement-processor-extractor/Dockerfile b/announcement-processor-extractor/Dockerfile index 3fc7ed5..5913a80 100644 --- a/announcement-processor-extractor/Dockerfile +++ b/announcement-processor-extractor/Dockerfile @@ -11,4 +11,6 @@ COPY src /srv/announcement-processor/extractor/src RUN ["mvn", "clean", "install", "-DskipTests"] -CMD java -jar /srv/announcement-processor/extractor/target/extractor-service.jar +RUN cd target && ls + +CMD java -jar /srv/announcement-processor/extractor/target/extractor-service.jar \ No newline at end of file diff --git a/announcement-processor-extractor/pom.xml b/announcement-processor-extractor/pom.xml index 4e6d47b..d791994 100644 --- a/announcement-processor-extractor/pom.xml +++ b/announcement-processor-extractor/pom.xml @@ -64,7 +64,7 @@ - parser-service + extractor-service From a7be85dbea68155860691a4ce979a67189a9f324 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Sroka?= Date: Wed, 14 Nov 2018 21:02:30 +0100 Subject: [PATCH 024/135] docker-compose fix --- .circleci/config.yml | 28 +++- announcement-processor-api/.gitignore | 29 ++++ announcement-processor-api/pom.xml | 128 ++++++++++++++++++ .../AnnouncementProcessorApiApplication.java | 12 ++ .../main/resources/application-dev.properties | 11 ++ .../src/main/resources/application.properties | 1 + .../src/main/resources/logback.xml | 18 +++ ...ouncementProcessorApiApplicationTests.java | 16 +++ announcement-processor-extractor/Dockerfile | 2 - ...uncementProcessorExtractorApplication.java | 3 +- .../src/test/java/parser/ExampleTest.java | 14 ++ 11 files changed, 258 insertions(+), 4 deletions(-) create mode 100644 announcement-processor-api/.gitignore create mode 100644 announcement-processor-api/pom.xml create mode 100644 announcement-processor-api/src/main/java/api/AnnouncementProcessorApiApplication.java create mode 100644 announcement-processor-api/src/main/resources/application-dev.properties create mode 100644 announcement-processor-api/src/main/resources/application.properties create mode 100644 announcement-processor-api/src/main/resources/logback.xml create mode 100644 announcement-processor-api/src/test/java/api/AnnouncementProcessorApiApplicationTests.java create mode 100644 announcement-processor-extractor/src/test/java/parser/ExampleTest.java diff --git a/.circleci/config.yml b/.circleci/config.yml index 31d2cf2..4913611 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,8 +25,34 @@ jobs: - store_artifacts: path: announcement-processor-parser/target/parser-service.jar + api-test: + docker: + - image: circleci/openjdk:8-jdk-browsers + + steps: + - checkout + + - restore_cache: # restore the saved cache after the first run or if `pom.xml` has changed + key: circleci-announcement-processor-api-{{ checksum "announcement-processor-api/pom.xml" }} + + - run: cd announcement-processor-api && mvn dependency:go-offline # gets the project dependencies + + - save_cache: # saves the project dependencies + paths: + - ~/.m2 + key: circleci-announcement-processor-api-{{ checksum "announcement-processor-api/pom.xml" }} + + - run: cd announcement-processor-api && mvn package # run the actual tests + + - store_test_results: # uploads the test metadata from the `target/surefire-reports` directory so that it can show up in the CircleCI dashboard. + path: announcement-processor-api/target/surefire-reports + + - store_artifacts: + path: announcement-processor-api/target/api-service.jar + workflows: version: 2 announcement_processor: jobs: - - parser-test \ No newline at end of file + - parser-test + - api-test \ No newline at end of file diff --git a/announcement-processor-api/.gitignore b/announcement-processor-api/.gitignore new file mode 100644 index 0000000..20e9cad --- /dev/null +++ b/announcement-processor-api/.gitignore @@ -0,0 +1,29 @@ +/target/ +.mvn +mvnw +mvnw.cmd + +src/main/resources/application-prod.properties + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/build/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ \ No newline at end of file diff --git a/announcement-processor-api/pom.xml b/announcement-processor-api/pom.xml new file mode 100644 index 0000000..bff3df2 --- /dev/null +++ b/announcement-processor-api/pom.xml @@ -0,0 +1,128 @@ + + + 4.0.0 + + announcement-processor + api + 0.0.1-SNAPSHOT + jar + Announcement processor api + + + org.springframework.boot + spring-boot-starter-parent + 2.1.0.RELEASE + + + + + UTF-8 + UTF-8 + 1.8 + + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-devtools + runtime + + + org.springframework.boot + spring-boot-starter-test + test + + + + + org.projectlombok + lombok + true + + + + + org.mockito + mockito-all + 2.0.2-beta + test + + + org.jacoco + jacoco-maven-plugin + 0.8.2 + + + + + org.springframework.data + spring-data-jpa + 2.1.2.RELEASE + + + mysql + mysql-connector-java + + + com.h2database + h2 + 1.4.197 + + + org.springframework.boot + spring-boot-starter-data-jpa + + + + + + api-service + + + + org.springframework.boot + spring-boot-maven-plugin + + + org.apache.maven.plugins + maven-surefire-plugin + + false + + + + org.jacoco + jacoco-maven-plugin + 0.8.2 + + + + prepare-agent + + + + report + test + + report + + + + + api/AnnouncementProcessorApiApplication.class + + + + + + + + + + + diff --git a/announcement-processor-api/src/main/java/api/AnnouncementProcessorApiApplication.java b/announcement-processor-api/src/main/java/api/AnnouncementProcessorApiApplication.java new file mode 100644 index 0000000..2b09fea --- /dev/null +++ b/announcement-processor-api/src/main/java/api/AnnouncementProcessorApiApplication.java @@ -0,0 +1,12 @@ +package api; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class AnnouncementProcessorApiApplication { + + public static void main(String[] args) { + SpringApplication.run(AnnouncementProcessorApiApplication.class, args); + } +} diff --git a/announcement-processor-api/src/main/resources/application-dev.properties b/announcement-processor-api/src/main/resources/application-dev.properties new file mode 100644 index 0000000..4204610 --- /dev/null +++ b/announcement-processor-api/src/main/resources/application-dev.properties @@ -0,0 +1,11 @@ +# H2 +spring.h2.console.enabled=true +spring.h2.console.path=/h2 + +# Datasource +spring.datasource.url=jdbc:h2:file:~/dev +spring.datasource.username=sa +spring.datasource.password= +spring.datasource.driver-class-name=org.h2.Driver + +spring.jpa.hibernate.ddl-auto=create \ No newline at end of file diff --git a/announcement-processor-api/src/main/resources/application.properties b/announcement-processor-api/src/main/resources/application.properties new file mode 100644 index 0000000..426b904 --- /dev/null +++ b/announcement-processor-api/src/main/resources/application.properties @@ -0,0 +1 @@ +spring.profiles.default=dev \ No newline at end of file diff --git a/announcement-processor-api/src/main/resources/logback.xml b/announcement-processor-api/src/main/resources/logback.xml new file mode 100644 index 0000000..5097d20 --- /dev/null +++ b/announcement-processor-api/src/main/resources/logback.xml @@ -0,0 +1,18 @@ + + + + + + + %d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + \ No newline at end of file diff --git a/announcement-processor-api/src/test/java/api/AnnouncementProcessorApiApplicationTests.java b/announcement-processor-api/src/test/java/api/AnnouncementProcessorApiApplicationTests.java new file mode 100644 index 0000000..2b5751e --- /dev/null +++ b/announcement-processor-api/src/test/java/api/AnnouncementProcessorApiApplicationTests.java @@ -0,0 +1,16 @@ +package api; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +@SpringBootTest +public class AnnouncementProcessorApiApplicationTests { + + @Test + public void contextLoads() { + } + +} diff --git a/announcement-processor-extractor/Dockerfile b/announcement-processor-extractor/Dockerfile index 5913a80..a8a909c 100644 --- a/announcement-processor-extractor/Dockerfile +++ b/announcement-processor-extractor/Dockerfile @@ -11,6 +11,4 @@ COPY src /srv/announcement-processor/extractor/src RUN ["mvn", "clean", "install", "-DskipTests"] -RUN cd target && ls - CMD java -jar /srv/announcement-processor/extractor/target/extractor-service.jar \ No newline at end of file diff --git a/announcement-processor-extractor/src/main/java/extractor/AnnouncementProcessorExtractorApplication.java b/announcement-processor-extractor/src/main/java/extractor/AnnouncementProcessorExtractorApplication.java index ed56a2b..e4337fe 100644 --- a/announcement-processor-extractor/src/main/java/extractor/AnnouncementProcessorExtractorApplication.java +++ b/announcement-processor-extractor/src/main/java/extractor/AnnouncementProcessorExtractorApplication.java @@ -14,10 +14,11 @@ public class AnnouncementProcessorExtractorApplication implements CommandLineRun public static void main(String[] args) { SpringApplication app = new SpringApplication(AnnouncementProcessorExtractorApplication.class); + app.run(args); } @Override - public void run(String... args) throws Exception { + public void run(String... args) { } } diff --git a/announcement-processor-extractor/src/test/java/parser/ExampleTest.java b/announcement-processor-extractor/src/test/java/parser/ExampleTest.java new file mode 100644 index 0000000..09de7c0 --- /dev/null +++ b/announcement-processor-extractor/src/test/java/parser/ExampleTest.java @@ -0,0 +1,14 @@ +package parser; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class ExampleTest { + @Test + public void test(){ + Assert.assertTrue(true); + } +} From e62351c0c15a16978823425cf5cd50c63c0e993a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Sroka?= Date: Wed, 14 Nov 2018 21:10:43 +0100 Subject: [PATCH 025/135] docker-compose fix --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 238c0d9..855d3c2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,7 +25,7 @@ services: - 8081:8081 volumes: - ./announcement-processor-parser/src:/srv/announcement-processor/parser/src - command: java -jar ./target/parser-service.jar + command: java -jar ./target/parser-service.jar extractor: build: ./announcement-processor-extractor From 6e9f4728a418484dd66a7997b9ab510747875fb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Sroka?= Date: Wed, 14 Nov 2018 21:40:48 +0100 Subject: [PATCH 026/135] Add circleci confif for extractor --- .circleci/config.yml | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 08de715..f93df82 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -50,9 +50,35 @@ jobs: - store_artifacts: path: announcement-processor-api/target/api-service.jar + extractor-test: + docker: + - image: circleci/openjdk:8-jdk-browsers + + steps: + - checkout + + - restore_cache: # restore the saved cache after the first run or if `pom.xml` has changed + key: circleci-announcement-processor-extractor-{{ checksum "announcement-processor-extractor/pom.xml" }} + + - run: cd announcement-processor-extractor && mvn dependency:go-offline # gets the project dependencies + + - save_cache: # saves the project dependencies + paths: + - ~/.m2 + key: circleci-announcement-processor-extractor-{{ checksum "announcement-processor-extractor/pom.xml" }} + + - run: cd announcement-processor-extractor && mvn package # run the actual tests + + - store_test_results: # uploads the test metadata from the `target/surefire-reports` directory so that it can show up in the CircleCI dashboard. + path: announcement-processor-extractor/target/surefire-reports + + - store_artifacts: + path: announcement-processor-extractor/target/extractor-service.jar + workflows: version: 2 announcement_processor: jobs: - parser-test - - api-test \ No newline at end of file + - api-test + - extractor-test \ No newline at end of file From 6004c4d027b7a1a016f28df2ad6e1fcf251173d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Sroka?= Date: Wed, 14 Nov 2018 21:45:55 +0100 Subject: [PATCH 027/135] Fix extractor's pom file --- announcement-processor-extractor/pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/announcement-processor-extractor/pom.xml b/announcement-processor-extractor/pom.xml index d791994..f54eb5b 100644 --- a/announcement-processor-extractor/pom.xml +++ b/announcement-processor-extractor/pom.xml @@ -71,6 +71,13 @@ org.springframework.boot spring-boot-maven-plugin + + org.apache.maven.plugins + maven-surefire-plugin + + false + + org.jacoco jacoco-maven-plugin From faa34e0a399c6453634379548a3c161fa50c9306 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Fri, 16 Nov 2018 22:52:39 +0100 Subject: [PATCH 028/135] Interface to implement by concrete parsers. #31 --- .../main/java/parser/SinglePageParser.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 announcement-processor-parser/src/main/java/parser/SinglePageParser.java diff --git a/announcement-processor-parser/src/main/java/parser/SinglePageParser.java b/announcement-processor-parser/src/main/java/parser/SinglePageParser.java new file mode 100644 index 0000000..439a17f --- /dev/null +++ b/announcement-processor-parser/src/main/java/parser/SinglePageParser.java @@ -0,0 +1,28 @@ +package parser; + +import java.math.BigDecimal; +import java.time.LocalDateTime; + +public interface SinglePageParser { + String parseTitle(); + BigDecimal parsePrice(); + LocalDateTime parseCreationDate(); + String parseDescription(); + String parseProvider(); + String parseUrl(); + + String parseLessor(); + String parserLessorName(); + String parsePhoneNumber(); + + String parsePropertyType(); + Double parseFlatArea(); + Integer parseRoomAmount(); + Integer parseBathAmount(); + Boolean parseParking(); + Boolean parserSmokers(); + Boolean parserPetFriendly(); + BigDecimal parseAdditionalRentCost(); + String parseLevel(); + String parseFurnishings(); +} From 075801f99a1c0b76fa52d56fdbb7c0f11e57a9e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Fri, 16 Nov 2018 22:54:56 +0100 Subject: [PATCH 029/135] Added class containing announcement properties. #31 --- .../src/main/java/parser/Announcement.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 announcement-processor-parser/src/main/java/parser/Announcement.java diff --git a/announcement-processor-parser/src/main/java/parser/Announcement.java b/announcement-processor-parser/src/main/java/parser/Announcement.java new file mode 100644 index 0000000..f4679b7 --- /dev/null +++ b/announcement-processor-parser/src/main/java/parser/Announcement.java @@ -0,0 +1,26 @@ +package parser; + +import java.math.BigDecimal; +import java.time.LocalDateTime; + +public class Announcement { + private String title; + private BigDecimal price; + private LocalDateTime creationDate; + private String lessor; + private String propertyType; + private Double flatArea; + private Integer roomAmount; + private Integer bathAmount; + private Boolean isParkingAvailable; + private Boolean isSmokingAllowed; + private Boolean isPetFriendly; + private String lessorName; + private BigDecimal additionalRentCost; + private String level; + private String isFurnished; + private String description; + private String provider; + private String url; + private String phoneNumber; +} From 4a7382092acd8bcebce604a42b07e66d2a41ff4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Fri, 16 Nov 2018 23:21:14 +0100 Subject: [PATCH 030/135] Added builder pattern to Announcement class. #31 --- .../src/main/java/parser/Announcement.java | 157 +++++++++++++++++- 1 file changed, 156 insertions(+), 1 deletion(-) diff --git a/announcement-processor-parser/src/main/java/parser/Announcement.java b/announcement-processor-parser/src/main/java/parser/Announcement.java index f4679b7..f2185f4 100644 --- a/announcement-processor-parser/src/main/java/parser/Announcement.java +++ b/announcement-processor-parser/src/main/java/parser/Announcement.java @@ -1,8 +1,13 @@ package parser; +import lombok.Getter; +import lombok.Setter; + import java.math.BigDecimal; import java.time.LocalDateTime; +@Getter +@Setter public class Announcement { private String title; private BigDecimal price; @@ -18,9 +23,159 @@ public class Announcement { private String lessorName; private BigDecimal additionalRentCost; private String level; - private String isFurnished; + private String furnishing; private String description; private String provider; private String url; private String phoneNumber; + + private Announcement() { + } + + private Announcement(AnnouncementBuilder builder) { + this.title = builder.title; + this.price = builder.price; + this.creationDate = builder.creationDate; + this.lessor = builder.lessor; + this.propertyType = builder.propertyType; + this.flatArea = builder.flatArea; + this.roomAmount = builder.roomAmount; + this.bathAmount = builder.bathAmount; + this.isParkingAvailable = builder.isParkingAvailable; + this.isSmokingAllowed = builder.isSmokingAllowed; + this.isPetFriendly = builder.isPetFriendly; + this.lessorName = builder.lessorName; + this.additionalRentCost = builder.additionalRentCost; + this.level = builder.level; + this.furnishing = builder.furnishing; + this.description = builder.description; + this.provider = builder.provider; + this.url = builder.url; + this.phoneNumber = builder.phoneNumber; + } + + static AnnouncementBuilder builder() { + return new AnnouncementBuilder(); + } + + public static final class AnnouncementBuilder { + private String title; + private BigDecimal price; + private LocalDateTime creationDate; + private String lessor; + private String propertyType; + private Double flatArea; + private Integer roomAmount; + private Integer bathAmount; + private Boolean isParkingAvailable; + private Boolean isSmokingAllowed; + private Boolean isPetFriendly; + private String lessorName; + private BigDecimal additionalRentCost; + private String level; + private String furnishing; + private String description; + private String provider; + private String url; + private String phoneNumber; + + AnnouncementBuilder title(String title) { + this.title = title; + return this; + } + + AnnouncementBuilder price(BigDecimal price) { + this.price = price; + return this; + } + + AnnouncementBuilder creationDate(LocalDateTime creationDate) { + this.creationDate = creationDate; + return this; + } + + AnnouncementBuilder lessor(String lessor) { + this.lessor = lessor; + return this; + } + + AnnouncementBuilder propertyType(String propertyType) { + this.propertyType = propertyType; + return this; + } + + AnnouncementBuilder flatArea(Double flatArea) { + this.flatArea = flatArea; + return this; + } + + AnnouncementBuilder roomAmount(Integer roomAmount) { + this.roomAmount = roomAmount; + return this; + } + + AnnouncementBuilder bathAmount(Integer bathAmount) { + this.bathAmount = bathAmount; + return this; + } + + AnnouncementBuilder isParkingAvailable(Boolean isParkingAvailable) { + this.isParkingAvailable = isParkingAvailable; + return this; + } + + AnnouncementBuilder isSmokingAllowed(Boolean isSmokingAllowed) { + this.isSmokingAllowed = isSmokingAllowed; + return this; + } + + AnnouncementBuilder isPetFriendly(Boolean isPetFriendly) { + this.isPetFriendly = isPetFriendly; + return this; + } + + AnnouncementBuilder lessorName(String lessorName) { + this.lessor = lessorName; + return this; + } + + AnnouncementBuilder additionalRentCost(BigDecimal additionalRentCost) { + this.additionalRentCost = additionalRentCost; + return this; + } + + AnnouncementBuilder level(String level) { + this.level = level; + return this; + } + + AnnouncementBuilder furnishing(String furnishing) { + this.furnishing = furnishing; + return this; + } + + AnnouncementBuilder description(String description) { + this.description = description; + return this; + } + + AnnouncementBuilder provider(String provider) { + this.provider = provider; + return this; + } + + AnnouncementBuilder url(String url) { + this.url = url; + return this; + } + + AnnouncementBuilder phoneNumber(String phoneNumber) { + this.phoneNumber = phoneNumber; + return this; + } + + Announcement build() { + return new Announcement(this); + } + } } From 042ca094797369d6bdeb77f597682fe89e383120 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Fri, 16 Nov 2018 23:30:36 +0100 Subject: [PATCH 031/135] Disable code coverage check for lombok annotations (getters & setters). #31 --- announcement-processor-parser/lombok.config | 1 + 1 file changed, 1 insertion(+) create mode 100644 announcement-processor-parser/lombok.config diff --git a/announcement-processor-parser/lombok.config b/announcement-processor-parser/lombok.config new file mode 100644 index 0000000..8f7e8aa --- /dev/null +++ b/announcement-processor-parser/lombok.config @@ -0,0 +1 @@ +lombok.addLombokGeneratedAnnotation = true \ No newline at end of file From bb3ed097036c11830f2f36430716e01b2229d9e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Fri, 16 Nov 2018 23:34:15 +0100 Subject: [PATCH 032/135] Added abstract class AnnouncementParser as a base class for concrete parsers. #31 --- .../src/main/java/parser/AnnouncementParser.java | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 announcement-processor-parser/src/main/java/parser/AnnouncementParser.java diff --git a/announcement-processor-parser/src/main/java/parser/AnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/AnnouncementParser.java new file mode 100644 index 0000000..a3363e2 --- /dev/null +++ b/announcement-processor-parser/src/main/java/parser/AnnouncementParser.java @@ -0,0 +1,5 @@ +package parser; + +public abstract class AnnouncementParser implements SinglePageParser { + public abstract Announcement parsePage(String url); +} From 73915c2214f10604eae6ff315a8e4a8537a6239b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sat, 17 Nov 2018 00:38:08 +0100 Subject: [PATCH 033/135] Moved files to another package. #31 --- .../main/java/parser/{ => scrapper}/AnnouncementParser.java | 5 ++++- .../main/java/parser/{ => scrapper}/SinglePageParser.java | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) rename announcement-processor-parser/src/main/java/parser/{ => scrapper}/AnnouncementParser.java (56%) rename announcement-processor-parser/src/main/java/parser/{ => scrapper}/SinglePageParser.java (92%) diff --git a/announcement-processor-parser/src/main/java/parser/AnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/AnnouncementParser.java similarity index 56% rename from announcement-processor-parser/src/main/java/parser/AnnouncementParser.java rename to announcement-processor-parser/src/main/java/parser/scrapper/AnnouncementParser.java index a3363e2..db45864 100644 --- a/announcement-processor-parser/src/main/java/parser/AnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/AnnouncementParser.java @@ -1,5 +1,8 @@ -package parser; +package parser.scrapper; + +import parser.Announcement; public abstract class AnnouncementParser implements SinglePageParser { public abstract Announcement parsePage(String url); + abstract String getPageContent(String url); } diff --git a/announcement-processor-parser/src/main/java/parser/SinglePageParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/SinglePageParser.java similarity index 92% rename from announcement-processor-parser/src/main/java/parser/SinglePageParser.java rename to announcement-processor-parser/src/main/java/parser/scrapper/SinglePageParser.java index 439a17f..c6fed2a 100644 --- a/announcement-processor-parser/src/main/java/parser/SinglePageParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/SinglePageParser.java @@ -1,9 +1,9 @@ -package parser; +package parser.scrapper; import java.math.BigDecimal; import java.time.LocalDateTime; -public interface SinglePageParser { +interface SinglePageParser { String parseTitle(); BigDecimal parsePrice(); LocalDateTime parseCreationDate(); From d7d419ff005fcf86a359b6a1379c47c9c087e0d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sat, 17 Nov 2018 00:43:14 +0100 Subject: [PATCH 034/135] Created default methods returning null for properties which are not in every provider. #31 --- .../parser/scrapper/SinglePageParser.java | 65 +++++++++++++++---- 1 file changed, 51 insertions(+), 14 deletions(-) diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/SinglePageParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/SinglePageParser.java index c6fed2a..2cd203d 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/SinglePageParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/SinglePageParser.java @@ -11,18 +11,55 @@ interface SinglePageParser { String parseProvider(); String parseUrl(); - String parseLessor(); - String parserLessorName(); - String parsePhoneNumber(); - - String parsePropertyType(); - Double parseFlatArea(); - Integer parseRoomAmount(); - Integer parseBathAmount(); - Boolean parseParking(); - Boolean parserSmokers(); - Boolean parserPetFriendly(); - BigDecimal parseAdditionalRentCost(); - String parseLevel(); - String parseFurnishings(); + default String parseLessor() { + return null; + } + + default String parserLessorName() { + return null; + } + + default String parsePhoneNumber() { + return null; + } + + default String parsePropertyType() { + return null; + } + + default Double parseFlatArea(){ + return null; + } + + default Integer parseRoomAmount(){ + return null; + } + + default Integer parseBathAmount(){ + return null; + } + + default Boolean parseParking(){ + return null; + } + + default Boolean parserSmokers(){ + return null; + } + + default Boolean parserPetFriendly(){ + return null; + } + + default BigDecimal parseAdditionalRentCost(){ + return null; + } + + default String parseLevel(){ + return null; + } + + default String parseFurnishings(){ + return null; + } } From 15806c34ff823ecf6da2a1342f92077095ab01a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sat, 17 Nov 2018 01:15:15 +0100 Subject: [PATCH 035/135] Added JSoup library and implemented getPageContent. #31 --- announcement-processor-parser/pom.xml | 7 +++++++ .../main/java/parser/scrapper/AnnouncementParser.java | 9 ++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/announcement-processor-parser/pom.xml b/announcement-processor-parser/pom.xml index 27ecc38..e603e00 100644 --- a/announcement-processor-parser/pom.xml +++ b/announcement-processor-parser/pom.xml @@ -59,6 +59,13 @@ jacoco-maven-plugin 0.8.2 + + + + org.jsoup + jsoup + 1.11.3 + diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/AnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/AnnouncementParser.java index db45864..32f588d 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/AnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/AnnouncementParser.java @@ -1,8 +1,15 @@ package parser.scrapper; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; import parser.Announcement; +import java.io.IOException; + public abstract class AnnouncementParser implements SinglePageParser { public abstract Announcement parsePage(String url); - abstract String getPageContent(String url); + + protected Document getPageContent(String url) throws IOException { + return Jsoup.connect(url).get(); + } } From f2d99a02db6d4c2dd887a2b4f34abd4271f856c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sat, 17 Nov 2018 15:20:31 +0100 Subject: [PATCH 036/135] Added gumtree page sample as txt for testing purpose. #3 --- .../resources/sample-gumtree-announcement.txt | 1232 +++++++++++++++++ 1 file changed, 1232 insertions(+) create mode 100644 announcement-processor-parser/src/test/resources/sample-gumtree-announcement.txt diff --git a/announcement-processor-parser/src/test/resources/sample-gumtree-announcement.txt b/announcement-processor-parser/src/test/resources/sample-gumtree-announcement.txt new file mode 100644 index 0000000..2ab3719 --- /dev/null +++ b/announcement-processor-parser/src/test/resources/sample-gumtree-announcement.txt @@ -0,0 +1,1232 @@ + + + + + + + + + + Kawalerka 40m Głowackiego 4 Kraków PRZESTRONNE niskie koszty WIDOK na PANORAMĘ Krakowa POŁUDNIE – Kraków – 355501407 | Gumtree + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ Aby zapewnić najwyższą jakość usług i wygodne korzystanie z serwisu, używamy informacji zapisanych w przeglądarce za pomocą plików cookies (pol.: ciasteczek). Korzystając z serwisu wyrażasz zgodę na stosowanie plików + cookies. W każdej chwili możesz je zablokować korzystając z ustawień swojej przeglądarki internetowej. + +
+
+ + +
+
+ +
+
+
+
+ +
+
+
+ Dodaj ogłoszenie + +
+
+ +
+ +
+
+
+
+
+
+
+
+
+ Dodaj ogłoszenie +
+
+
+
+
+
+
+
+
+
+
+ +
+
+ + + + + +
+
+ + + + + +
+
+ +
+
+
+
+
+
+
+
+
+ Małopolskie + + + + Nieruchomości + + + + mieszkania i domy do wynajęcia | Kraków + + + + ogłoszenie 355501407 +
+
+
+
+
+ + +
+
+
+
+
+
+
+
+
+

Kawalerka 40m Głowackiego 4 Kraków PRZESTRONNE niskie koszty WIDOK na PANORAMĘ Krakowa POŁUDNIE

+
+ 1 300 zł +
+
+
+
+
+
+ + + Kawalerka 40m Głowackiego 4 Kraków PRZESTRONNE niskie koszty WIDOK na PANORAMĘ Krakowa POŁUDNIE + + +
+
+
+
+ Kawalerka 40m Głowackiego 4 Kraków PRZESTRONNE niskie koszty WIDOK na PANORAMĘ Krakowa POŁUDNIE z Krakow, miniatura: 1 + Kawalerka 40m Głowackiego 4 Kraków PRZESTRONNE niskie koszty WIDOK na PANORAMĘ Krakowa POŁUDNIE z Krakow, miniatura: 2 + Kawalerka 40m Głowackiego 4 Kraków PRZESTRONNE niskie koszty WIDOK na PANORAMĘ Krakowa POŁUDNIE z Krakow, miniatura: 3 + Kawalerka 40m Głowackiego 4 Kraków PRZESTRONNE niskie koszty WIDOK na PANORAMĘ Krakowa POŁUDNIE z Krakow, miniatura: 4 + Kawalerka 40m Głowackiego 4 Kraków PRZESTRONNE niskie koszty WIDOK na PANORAMĘ Krakowa POŁUDNIE z Krakow, miniatura: 5 + Kawalerka 40m Głowackiego 4 Kraków PRZESTRONNE niskie koszty WIDOK na PANORAMĘ Krakowa POŁUDNIE z Krakow, miniatura: 6 + Kawalerka 40m Głowackiego 4 Kraków PRZESTRONNE niskie koszty WIDOK na PANORAMĘ Krakowa POŁUDNIE z Krakow, miniatura: 7 +
+
+ +
+
+
+ Dodaj takie ogłoszenie! +
+
+
+ +
    +
  • +
    + Data dodania + 17/11/2018 +
    • +
  • +
    + Lokalizacja + +
    + Kraków, + Małopolskie +
    +
    • + + + +
  • +
    + Do wynajęcia przez + Właściciel +
    • +
  • +
    + Dostępny + 17/11/2018 +
    • +
  • +
    + Rodzaj nieruchomości + Mieszkanie +
    • +
  • +
    + Liczba pokoi + Kawalerka lub garsoniera +
    • +
  • +
    + Liczba łazienek + 1 łazienka +
    • +
  • +
    + Wielkość (m2) + 40 +
    • +
  • +
    + Parking + Ulica +
    • +
  • +
    + Palący + Tak +
    • +
  • +
    + Przyjazne zwierzakom + Nie +
    • + + + + + + + + + + + + + + + + +
  • +
    • +
+
+ +
+
+ Proponuję mieszkanie na wynajem (kawalerka) o powierzchni 40m2 zlokalizowane przy ulicy Głowackiego 4 w Krakowie. Dzielnica Krowodrza. Rejon Bronowice. +
+
+
+
+
+ Kawalerka mieście się na 5 piętrze od strony południowej z widokiem na panoramę Krakowa. +
+
+ 3 minuty pieszo do przystanków tramwajowych. Bliskie sąsiedztwo Uniwersytetu Pedagogicznego, AGH, Wydziału Fizyki i Architektury PK. +
+
+
+
+
+ Budynek oddany do użytkowania w 2008r. Podłoga z litego drewna, w aneksie kuchennym i w łazience płytki ceramiczne. Mieszkanie wyposażone tak jak na zdjęciach: (lodówka, płyta grzewcza, piekarnik, pralka, stół, krzesła, łóżko 2os., łóżko 1os.) +
+
+ Możliwość usunięcia mebli wg upodobań wynajmującego jak i ich dokupienia. +
+
+
+
+
+
+ Nieruchomość posiada przestronny hol i windę. Na parterze zlokalizowane są sklepy m. in. Lewiatan - można w pantoflach robić zakupy ;) +
+
+
+
+
+ Koszty najmu (miesięczne): wynajem + 1500zł, + czynsz administracyjny + 250zł, + media wg zużycia (woda (~60zł/1os), prąd (~60zł/1os) , ogrzewanie, internet (60zł)). +
+
+
+
+
+
+ SZACOWANY CAŁKOWITY KOSZT wynajmu i eksploatacji miesięcznej: 1900-2000zł (2 osoby) +
+
+
+
+
+ 2K PLN per month ;) +
+
+
+
Zalety to niskie koszty ogrzewania, w zasadzie pomijalne. +
+
+
+
+
+
+ INTERNET od UPC. Mieszkanie wyposażone w gniazdo RTV i SAT. +
+
+
+
+
+ Umowa min. na 9 miesięcy. W razie pytań proszę się nie krępować :) +
+
+
+
+
+
+
+ Kontak mailowy lub telefoniczny (w rozsądnych godzinach): +
+
+
+
+
+ 6 0 2 3 2 7 7 0 6 +
+
+
+
+
+
+ nioobi (malpa) + op (kropka) + pl +
+


+


+

Możliwość oglądnięcia mieszkania

+


+

Palić papierosy można na dużym balkonie ;)

+


+
+
+
+
+ Pośrednikom z góry dziękuję za zainteresowanie! +
+


+


+
+
+
+
+
+
+
+
    +
    • +
    • +
    • +
    • +
    • +
+
+
+
+
+
+
+
+
+
+ + Bartek (Zobacz więcej ogłoszeń) + Użytkownik od 09-2015 +
+
+
+
+ +
+ + + +
+ +
+ +
+
    +
    • +
    • +
+
+ + + + + + + +
+ Klikając "Wyślij", wyrażasz zgodę na nasze + Zasady korzystania i + Politykę prywatności oraz zgadzasz się na otrzymywanie naszych newsletterów i ofert promocyjnych. +
+
+
+ +
+
+
+
+
+
+ + Dodaj do Zachowanych +
+
+
+
+ +
+ Porady Bezpieczeństwa - Twoje bezpieczeństwo jest dla nas ważne, zachęcamy do zachowania czujności. Dowiedz się więcej + Zgłoś ogłoszenie +
+ +
+ + +
    +
    • + +
    • + +
    • + +
    • + +
    • + +
+
+ + + + +
+ + +
+
+
+
+
+
+ +
Bartosza Głowackiego 4, 33-332 Kraków, Poland
+
+
+
+
+
+
+
+
+
+ Popularne +
+ +
+
+
+
+ Proponowane +
+ +
+
+
+
+
+
+ Podobne ogłoszenia, które mogą Cię zainteresować. +
+
+
+
+
+
+ Nowa, piękna, duża kawalerka, Bonarka, Wadowicka z Krakow, zobacz zdjęcie + +
+ Nowa, piękna, duża kawalerka, Bonarka, Wadowicka + +
+ 1 600 zł +
+
+ Posiadamy zarówno kawalerki jak i dwupokojowe mieszkania. Zapraszamy do kontaktu: właściciel - Ania tel. (pokaż numer) Komfort i design Oferujemy Państwu komfortowe mieszkanie w kameralnej kamienicy, która łączy w sobie historię oraz nowoczesność. Kamienica została oddana we wrześniu 2018 roku. Ideą designu było połączenie elementów vintage, takich jak piękna cegła, z nowoczesnymi i elegancki ... +
+
+
+
+
+
+ Flat for rent! Wynajmę mieszkanie w centrum Krakowa! Dietla – Kazimierz z Krakow, zobacz zdjęcie + +
+ Flat for rent! Wynajmę mieszkanie w centrum Krakowa! Dietla – Kazimierz + +
+ 1 590 zł +
+
+ You don't speak polish? - don't worry! - I do speak english. Just call me +48 600 416 390 Opis mieszkania: • powierzchnia: 32m2 + antresola • mieszkanie składa się z pokoju, kuchni, łazienki, przedpokoju • lokalizacja: Kazimierz Kraków, ul. Dietla 55 (3min piechotą od placu Nowego, 10min od Rynku Głównego) • mieszkanie bardzo zadbane - usytuowane na parterze w kamienicy • ogrz ... +
+
+
+
+
+
+ 35m2 15 min pieszo od Galerii Krakowskiej z Krakow, zobacz zdjęcie + +
+ 35m2 15 min pieszo od Galerii Krakowskiej + +
+ 1 399 zł +
+
+ 35m2 w świetnej lokalizacji, - 10-15 min pieszo od Galerii Krakowskiej, Politechniki lub Akademii Rolniczej, - ul.Rogatka. Jest to fajne 1-pokojowe mieszkanie na 1 piętrze w wyremontowanej i ocieplonej wełną kamienicy.1 duży pokój 20m2 (4X5). Duża wyposażona kuchnia 9m2 (3X3), - bardzo widokowa. W sezonie dookoła dużo zieleni. Przystanek autobusowy – 2 min (dużo połączeń we wszystkich kierunk ... +
+
+
+
+
+
+ Mieszkanie dwupokojowe – Kraków – Śródmieście – Olsza II – Celarowska 20 z Krakow, zobacz zdjęcie + +
+ Mieszkanie dwupokojowe – Kraków – Śródmieście – Olsza II – Celarowska 20 + +
+ 1 400 zł +
+
+ Oferuję do wynajęcia mieszkanie 36m, usytuowane na 4 piętrze.Składa się z 2 pokoi, oddzielnej kuchni, przedpokoju oraz łazienki.Jest w pełni umeblowane i wyposażone w niezbędne sprzęty i meble. W jednym z pokoi duża, bardzo pojemna szafa.Miesięczny czynsz wynajmu wynosi: 1400 PLN + dodatkowo opłaty do spółdzielni 200PLN (1os) / 250PLN (2os),media prąd gaz wg. rachunków, telewizja i internet U ... +
+
+
+
+
+
+ 3pok przy Błoniach bez czynszu! 70m2, ul. Emaus z Krakow, zobacz zdjęcie + +
+ 3pok przy Błoniach bez czynszu! 70m2, ul. Emaus + +
+ 2 600 zł +
+
+ Nowodworski Estates prezentuje 3 pokojowe mieszkanie tuż przy Krakowskich Błoniach do wynajęcia!LOKALIZACJA:Mieszkanie znajduje się przy cichej i spokojnej ulicy Emaus, w bezpośrednim sąsiedztwie ulic takich jak: Piastowska, Królowej Jadwigi, Focha, al. 3 maja. Spacer na Krakowskie Błonia wzdłuż rzeki Rudawa zajmuje zaledwie kilka minut, a dotarcie do centrum samochodem lub komunikacją miejską ok. ... +
+
+
+
+
+
+ Mieszkanie Kraków Grzegórzki Park z Krakow, zobacz zdjęcie + +
+ Mieszkanie Kraków Grzegórzki Park + +
+ 2 400 zł +
+
+ Dodatkowe zaletyinternet, TV kablowa, winda, ogrzewanie miejskie, recepcja, domofon, wideofon, ochrona, podjazd dla niepełnosprawnychOpis dodatkowyDo wynajęcia dwupokojowy apartament w nowym budynku Grzegórzki Park przy ul. Francesco Nullo w Krakowie.LOKALIZACJA: Grzegórzki Park to nowo wybudowany kompleks usługowo- mieszkalny przy Francesco Nullo na krakowskich Grzegórzkach. Bliskość zarówno Star ... +
+
+
+
+
+
+ Kawalerka na POLONIJNEJ stan IDEALNY. z Krakow, zobacz zdjęcie + +
+ Kawalerka na POLONIJNEJ stan IDEALNY. + +
+ 1 500 zł +
+
+ Witam, oferuję do wynajęcia od zaraz kawalerkę na ul. Polonijnej, nowy blok, teren ogrodzony, wjazd na pilota, monitoring, pod blokiem sklep spożywczy Biedronka, mieszkanie znajduje się na XI piętrze (widokowe), posiada ekspozycję południowo-wschodnią (bardzo jasne). Mieszkanie jest praktycznie nowe, zamieszkane od kilku miesięcy, w pełni wykończone, wyposażone i umeblowane. Składa się z przedpoko ... +
+
+
+
+
+ Więcej ogłoszeń +
+
+
+
+
+
+
+ +
+ + + + + + + +
+
+
+
+
+
+
+ +
+
+
+
+ + + + + + + + + From ac33051081ec21212ecf53449ec1c831fa947263 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sat, 17 Nov 2018 15:23:11 +0100 Subject: [PATCH 037/135] More utilities for tests. #3 --- .../src/test/java/parser/ReaderUtil.java | 20 +++++++++++++++++++ .../src/test/resources/logback-test.xml | 1 + 2 files changed, 21 insertions(+) create mode 100644 announcement-processor-parser/src/test/java/parser/ReaderUtil.java create mode 100644 announcement-processor-parser/src/test/resources/logback-test.xml diff --git a/announcement-processor-parser/src/test/java/parser/ReaderUtil.java b/announcement-processor-parser/src/test/java/parser/ReaderUtil.java new file mode 100644 index 0000000..fbff4c7 --- /dev/null +++ b/announcement-processor-parser/src/test/java/parser/ReaderUtil.java @@ -0,0 +1,20 @@ +package parser; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.stream.Stream; + +public class ReaderUtil { + + public static String readAllLinesFromFile(String filePath) { + StringBuilder contentBuilder = new StringBuilder(); + try (Stream stream = Files.lines(Paths.get(filePath), StandardCharsets.UTF_8)) { + stream.forEach(s -> contentBuilder.append(s).append("\n")); + } catch (IOException e) { + e.printStackTrace(); + } + return contentBuilder.toString(); + } +} diff --git a/announcement-processor-parser/src/test/resources/logback-test.xml b/announcement-processor-parser/src/test/resources/logback-test.xml new file mode 100644 index 0000000..adfa02c --- /dev/null +++ b/announcement-processor-parser/src/test/resources/logback-test.xml @@ -0,0 +1 @@ + \ No newline at end of file From 7ba980d2d83315282a371f7c7d9842e5ef697bab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sat, 17 Nov 2018 15:24:56 +0100 Subject: [PATCH 038/135] Added Gumtree provider, SinglePageParser interface is implemented by concrete parsers. #3 --- .../parser/scrapper/AnnouncementParser.java | 4 +- .../scrapper/GumtreeAnnouncementParser.java | 95 +++++++++++++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/AnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/AnnouncementParser.java index 32f588d..f0ab1e4 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/AnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/AnnouncementParser.java @@ -6,10 +6,10 @@ import java.io.IOException; -public abstract class AnnouncementParser implements SinglePageParser { +public abstract class AnnouncementParser { public abstract Announcement parsePage(String url); - protected Document getPageContent(String url) throws IOException { + Document getPageContent(String url) throws IOException { return Jsoup.connect(url).get(); } } diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java new file mode 100644 index 0000000..75a24f8 --- /dev/null +++ b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java @@ -0,0 +1,95 @@ +package parser.scrapper; + +import lombok.extern.slf4j.Slf4j; +import parser.Announcement; + +import java.math.BigDecimal; +import java.time.LocalDateTime; + +@Slf4j +public class GumtreeAnnouncementParser extends AnnouncementParser implements SinglePageParser { + @Override + public Announcement parsePage(String url) { + return null; + } + + @Override + public String parseTitle() { + return null; + } + + @Override + public BigDecimal parsePrice() { + return null; + } + + @Override + public LocalDateTime parseCreationDate() { + return null; + } + + @Override + public String parseDescription() { + return null; + } + + @Override + public String parseProvider() { + return null; + } + + @Override + public String parseUrl() { + return null; + } + + @Override + public String parseLessor() { + return null; + } + + @Override + public String parserLessorName() { + return null; + } + + @Override + public String parsePhoneNumber() { + return null; + } + + @Override + public String parsePropertyType() { + return null; + } + + @Override + public Double parseFlatArea() { + return null; + } + + @Override + public Integer parseRoomAmount() { + return null; + } + + @Override + public Integer parseBathAmount() { + return null; + } + + @Override + public Boolean parseParking() { + return null; + } + + @Override + public Boolean parserSmokers() { + return null; + } + + @Override + public Boolean parserPetFriendly() { + return null; + } +} From 57808603b3d767d4d323fb03e934f8e4a751014c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sat, 17 Nov 2018 16:18:07 +0100 Subject: [PATCH 039/135] Added Element as an argument for parse functions. #3 --- .../parser/scrapper/SinglePageParser.java | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/SinglePageParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/SinglePageParser.java index 2cd203d..133e5ad 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/SinglePageParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/SinglePageParser.java @@ -1,65 +1,65 @@ package parser.scrapper; +import org.jsoup.nodes.Element; + import java.math.BigDecimal; import java.time.LocalDateTime; interface SinglePageParser { - String parseTitle(); - BigDecimal parsePrice(); - LocalDateTime parseCreationDate(); - String parseDescription(); - String parseProvider(); - String parseUrl(); - - default String parseLessor() { + String parseTitle(Element titleElement); + BigDecimal parsePrice(Element priceElement); + LocalDateTime parseCreationDate(Element creationDateElement); + String parseDescription(Element descriptionElement); + + default String parseLessor(Element lessorElement) { return null; } - default String parserLessorName() { + default String parserLessorName(Element lessorNameElement) { return null; } - default String parsePhoneNumber() { + default String parsePhoneNumber(Element phoneNumberElement) { return null; } - default String parsePropertyType() { + default String parsePropertyType(Element propertyTypeElement) { return null; } - default Double parseFlatArea(){ + default Double parseFlatArea(Element flatAreaElement){ return null; } - default Integer parseRoomAmount(){ + default Integer parseRoomAmount(Element roomAmountElement){ return null; } - default Integer parseBathAmount(){ + default Integer parseBathAmount(Element bathAmountElement){ return null; } - default Boolean parseParking(){ + default Boolean parseParking(Element parkingElement){ return null; } - default Boolean parserSmokers(){ + default Boolean parserSmokers(Element smokersElement){ return null; } - default Boolean parserPetFriendly(){ + default Boolean parserPetFriendly(Element petFriendlyElement){ return null; } - default BigDecimal parseAdditionalRentCost(){ + default BigDecimal parseAdditionalRentCost(Element additionalRentCostElement){ return null; } - default String parseLevel(){ + default String parseLevel(Element levelElement){ return null; } - default String parseFurnishings(){ + default String parseFurnishings(Element furnishingsElement){ return null; } } From 29daf613f89bc3eb8fd7a52dab8407271c5fe9bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sat, 17 Nov 2018 16:18:33 +0100 Subject: [PATCH 040/135] Throw GumtreePageParseException when page cannot be parsed. #3 --- .../exceptions/GumtreePageParseException.java | 7 +++ .../scrapper/GumtreeAnnouncementParser.java | 48 ++++++++++++------- 2 files changed, 39 insertions(+), 16 deletions(-) create mode 100644 announcement-processor-parser/src/main/java/parser/exceptions/GumtreePageParseException.java diff --git a/announcement-processor-parser/src/main/java/parser/exceptions/GumtreePageParseException.java b/announcement-processor-parser/src/main/java/parser/exceptions/GumtreePageParseException.java new file mode 100644 index 0000000..7c1ffc4 --- /dev/null +++ b/announcement-processor-parser/src/main/java/parser/exceptions/GumtreePageParseException.java @@ -0,0 +1,7 @@ +package parser.exceptions; + +public class GumtreePageParseException extends RuntimeException { + public GumtreePageParseException(String message) { + super(message); + } +} diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java index 75a24f8..9ad6ce0 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java @@ -1,8 +1,12 @@ package parser.scrapper; import lombok.extern.slf4j.Slf4j; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; import parser.Announcement; +import parser.exceptions.GumtreePageParseException; +import java.io.IOException; import java.math.BigDecimal; import java.time.LocalDateTime; @@ -10,86 +14,98 @@ public class GumtreeAnnouncementParser extends AnnouncementParser implements SinglePageParser { @Override public Announcement parsePage(String url) { + try { + Document pageContent = getPageContent(url); + log.info("Received content from url: " + url); + + return null; + } catch (IOException e) { + throw new GumtreePageParseException("Nie można przetworzyć ogłoszenia o podanym url: " + url); + } + } + + @Override + public String parseTitle(Element titleElement) { return null; } @Override - public String parseTitle() { + public BigDecimal parsePrice(Element priceElement) { return null; } @Override - public BigDecimal parsePrice() { + public LocalDateTime parseCreationDate(Element creationDateElement) { return null; } @Override - public LocalDateTime parseCreationDate() { + public String parseDescription(Element element) { return null; } @Override - public String parseDescription() { + public String parseLessor(Element lessorElement) { return null; } @Override - public String parseProvider() { + public String parserLessorName(Element lessorNameElement) { return null; } @Override - public String parseUrl() { + public String parsePhoneNumber(Element phoneNumberElement) { return null; } @Override - public String parseLessor() { + public String parsePropertyType(Element propertyTypeElement) { return null; } @Override - public String parserLessorName() { + public Double parseFlatArea(Element flatAreaElement) { return null; } @Override - public String parsePhoneNumber() { + public Integer parseRoomAmount(Element roomAmountElement) { return null; } @Override - public String parsePropertyType() { + public Integer parseBathAmount(Element bathAmountElement) { return null; } @Override - public Double parseFlatArea() { + public Boolean parseParking(Element parkingElement) { return null; } @Override - public Integer parseRoomAmount() { + public Boolean parserSmokers(Element smokersElement) { return null; } @Override - public Integer parseBathAmount() { + public Boolean parserPetFriendly(Element petFriendlyElement) { return null; } @Override - public Boolean parseParking() { + public BigDecimal parseAdditionalRentCost(Element additionalRentCostElement) { return null; } @Override - public Boolean parserSmokers() { + public String parseLevel(Element levelElement) { return null; } @Override - public Boolean parserPetFriendly() { + public String parseFurnishings(Element furnishingsElement) { return null; } } From 68a2c6672f8550d999108e8850c575b07306bf35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sat, 17 Nov 2018 16:32:49 +0100 Subject: [PATCH 041/135] Changed parking property to String. #3 --- .../src/main/java/parser/Announcement.java | 12 ++++++------ .../parser/scrapper/GumtreeAnnouncementParser.java | 2 +- .../main/java/parser/scrapper/SinglePageParser.java | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/announcement-processor-parser/src/main/java/parser/Announcement.java b/announcement-processor-parser/src/main/java/parser/Announcement.java index f2185f4..8f2af6b 100644 --- a/announcement-processor-parser/src/main/java/parser/Announcement.java +++ b/announcement-processor-parser/src/main/java/parser/Announcement.java @@ -17,7 +17,7 @@ public class Announcement { private Double flatArea; private Integer roomAmount; private Integer bathAmount; - private Boolean isParkingAvailable; + private String parkingAvailability; private Boolean isSmokingAllowed; private Boolean isPetFriendly; private String lessorName; @@ -41,7 +41,7 @@ private Announcement(AnnouncementBuilder builder) { this.flatArea = builder.flatArea; this.roomAmount = builder.roomAmount; this.bathAmount = builder.bathAmount; - this.isParkingAvailable = builder.isParkingAvailable; + this.parkingAvailability = builder.parkingAvailability; this.isSmokingAllowed = builder.isSmokingAllowed; this.isPetFriendly = builder.isPetFriendly; this.lessorName = builder.lessorName; @@ -67,7 +67,7 @@ public static final class AnnouncementBuilder { private Double flatArea; private Integer roomAmount; private Integer bathAmount; - private Boolean isParkingAvailable; + private String parkingAvailability; private Boolean isSmokingAllowed; private Boolean isPetFriendly; private String lessorName; @@ -119,8 +119,8 @@ AnnouncementBuilder bathAmount(Integer bathAmount) { return this; } - AnnouncementBuilder isParkingAvailable(Boolean isParkingAvailable) { - this.isParkingAvailable = isParkingAvailable; + AnnouncementBuilder parkingAvailability(String parkingAvailability) { + this.parkingAvailability = parkingAvailability; return this; } @@ -135,7 +135,7 @@ AnnouncementBuilder isPetFriendly(Boolean isPetFriendly) { } AnnouncementBuilder lessorName(String lessorName) { - this.lessor = lessorName; + this.lessorName = lessorName; return this; } diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java index 9ad6ce0..b905425 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java @@ -80,7 +80,7 @@ public Integer parseBathAmount(Element bathAmountElement) { } @Override - public Boolean parseParking(Element parkingElement) { + public String parseParking(Element parkingElement) { return null; } diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/SinglePageParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/SinglePageParser.java index 133e5ad..67eb7ac 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/SinglePageParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/SinglePageParser.java @@ -39,7 +39,7 @@ default Integer parseBathAmount(Element bathAmountElement){ return null; } - default Boolean parseParking(Element parkingElement){ + default String parseParking(Element parkingElement){ return null; } From 3ed210c657bad456f46402bdd92d743490329829 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sat, 17 Nov 2018 16:36:19 +0100 Subject: [PATCH 042/135] Test util function to get file as JSoup Document type. #3 --- .../src/test/java/parser/ReaderUtil.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/announcement-processor-parser/src/test/java/parser/ReaderUtil.java b/announcement-processor-parser/src/test/java/parser/ReaderUtil.java index fbff4c7..55c4530 100644 --- a/announcement-processor-parser/src/test/java/parser/ReaderUtil.java +++ b/announcement-processor-parser/src/test/java/parser/ReaderUtil.java @@ -1,6 +1,10 @@ package parser; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; + import java.io.IOException; +import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; @@ -8,7 +12,19 @@ public class ReaderUtil { - public static String readAllLinesFromFile(String filePath) { + public static Document getDocumentToTest(String resourceName) { + Document spyDoc = null; + try { + String path = Paths.get(ReaderUtil.class.getResource(resourceName).toURI()).toString(); + String fileContent = ReaderUtil.readAllLinesFromFile(path); + spyDoc = Jsoup.parse(fileContent); + } catch (URISyntaxException e) { + e.printStackTrace(); + } + return spyDoc; + } + + private static String readAllLinesFromFile(String filePath) { StringBuilder contentBuilder = new StringBuilder(); try (Stream stream = Files.lines(Paths.get(filePath), StandardCharsets.UTF_8)) { stream.forEach(s -> contentBuilder.append(s).append("\n")); From 2a1ddedeb21638e37e25880b14ff2b011c0deec2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 18 Nov 2018 18:21:16 +0100 Subject: [PATCH 043/135] Added basic tests which checks if field exists / not exists. #3 --- .../scrapper/GumtreeAnnouncementParser.java | 35 +- .../src/test/java/parser/ExampleTest.java | 14 - .../scrapper/GumtreeExpectedProperties.java | 51 + .../GumtreeParserNonRequiredFieldsTest.java | 253 ++++ .../GumtreeParserRequiredFieldsTest.java | 105 ++ .../sample-announcement-with-phone.html | 1102 +++++++++++++++++ .../sample-announcement-without-phone.html} | 0 .../sample-details-box-with-all-elements.html | 192 +++ ...mple-details-box-without-any-elements.html | 118 ++ 9 files changed, 1839 insertions(+), 31 deletions(-) delete mode 100644 announcement-processor-parser/src/test/java/parser/ExampleTest.java create mode 100644 announcement-processor-parser/src/test/java/parser/scrapper/GumtreeExpectedProperties.java create mode 100644 announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserNonRequiredFieldsTest.java create mode 100644 announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserRequiredFieldsTest.java create mode 100644 announcement-processor-parser/src/test/resources/gumtree/sample-announcement-with-phone.html rename announcement-processor-parser/src/test/resources/{sample-gumtree-announcement.txt => gumtree/sample-announcement-without-phone.html} (100%) create mode 100644 announcement-processor-parser/src/test/resources/gumtree/sample-details-box-with-all-elements.html create mode 100644 announcement-processor-parser/src/test/resources/gumtree/sample-details-box-without-any-elements.html diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java index b905425..30f7c44 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java @@ -3,6 +3,7 @@ import lombok.extern.slf4j.Slf4j; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; +import org.springframework.cglib.core.Local; import parser.Announcement; import parser.exceptions.GumtreePageParseException; @@ -26,86 +27,86 @@ public Announcement parsePage(String url) { @Override public String parseTitle(Element titleElement) { - return null; + return ""; } @Override public BigDecimal parsePrice(Element priceElement) { - return null; + return BigDecimal.ZERO; } @Override public LocalDateTime parseCreationDate(Element creationDateElement) { - return null; + return LocalDateTime.MIN; } @Override public String parseDescription(Element element) { - return null; + return ""; } @Override public String parseLessor(Element lessorElement) { - return null; + return ""; } @Override public String parserLessorName(Element lessorNameElement) { - return null; + return ""; } @Override public String parsePhoneNumber(Element phoneNumberElement) { - return null; + return ""; } @Override public String parsePropertyType(Element propertyTypeElement) { - return null; + return ""; } @Override public Double parseFlatArea(Element flatAreaElement) { - return null; + return 0d; } @Override public Integer parseRoomAmount(Element roomAmountElement) { - return null; + return -1; } @Override public Integer parseBathAmount(Element bathAmountElement) { - return null; + return -1; } @Override public String parseParking(Element parkingElement) { - return null; + return ""; } @Override public Boolean parserSmokers(Element smokersElement) { - return null; + return false; } @Override public Boolean parserPetFriendly(Element petFriendlyElement) { - return null; + return true; } @Override public BigDecimal parseAdditionalRentCost(Element additionalRentCostElement) { - return null; + return BigDecimal.ZERO; } @Override public String parseLevel(Element levelElement) { - return null; + return ""; } @Override public String parseFurnishings(Element furnishingsElement) { - return null; + return ""; } } diff --git a/announcement-processor-parser/src/test/java/parser/ExampleTest.java b/announcement-processor-parser/src/test/java/parser/ExampleTest.java deleted file mode 100644 index 09de7c0..0000000 --- a/announcement-processor-parser/src/test/java/parser/ExampleTest.java +++ /dev/null @@ -1,14 +0,0 @@ -package parser; - -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -@RunWith(JUnit4.class) -public class ExampleTest { - @Test - public void test(){ - Assert.assertTrue(true); - } -} diff --git a/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeExpectedProperties.java b/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeExpectedProperties.java new file mode 100644 index 0000000..3df5cea --- /dev/null +++ b/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeExpectedProperties.java @@ -0,0 +1,51 @@ +package parser.scrapper; + +import java.math.BigDecimal; +import java.time.LocalDateTime; + +class GumtreeExpectedProperties { + static String expectedTitle = "Kawalerka 40m Głowackiego 4 Kraków PRZESTRONNE niskie koszty WIDOK na PANORAMĘ Krakowa POŁUDNIE"; + static BigDecimal expectedPrice = BigDecimal.valueOf(1300); + static LocalDateTime expectedCreationDateTime = LocalDateTime.of(2018, 11, 17, 0, 0); + + static String expectedLessor = "Właściciel"; + static String expectedLessorName = "Bartek"; + static String expectedPhoneNumberNotExists = null; + static String expectedPhoneNumberExists = "795002620"; + static String expectedPropertyType = "Mieszkanie"; + static Double expectedFlatArea = 40d; + static Integer expectedRoomAmount = 1; + static Integer expectedBathAmount = 1; + static String expectedParking = "Ulica"; + static Boolean expectedSmokers = true; + static Boolean expectedPetFriendly = false; + + static String expectedDescription = "Proponuję mieszkanie na wynajem (kawalerka) o powierzchni 40m2 zlokalizowane przy ulicy Głowackiego 4 w Krakowie. Dzielnica Krowodrza. Rejon Bronowice.\n" + + "\n" + + "Kawalerka mieście się na 5 piętrze od strony południowej z widokiem na panoramę Krakowa.\n" + + "3 minuty pieszo do przystanków tramwajowych. Bliskie sąsiedztwo Uniwersytetu Pedagogicznego, AGH, Wydziału Fizyki i Architektury PK.\n" + + "\n" + + "Budynek oddany do użytkowania w 2008r. Podłoga z litego drewna, w aneksie kuchennym i w łazience płytki ceramiczne. Mieszkanie wyposażone tak jak na zdjęciach: (lodówka, płyta grzewcza, piekarnik, pralka, stół, krzesła, łóżko 2os., łóżko 1os.)\n" + + "Możliwość usunięcia mebli wg upodobań wynajmującego jak i ich dokupienia.\n" + + "\n" + + "Nieruchomość posiada przestronny hol i windę. Na parterze zlokalizowane są sklepy m. in. Lewiatan - można w pantoflach robić zakupy ;)\n" + + "\n" + + "Koszty najmu (miesięczne): wynajem 1500zł, + czynsz administracyjny 250zł, + media wg zużycia (woda (~60zł/1os), prąd (~60zł/1os) , ogrzewanie, internet (60zł)).\n" + + "\n" + + "SZACOWANY CAŁKOWITY KOSZT wynajmu i eksploatacji miesięcznej: 1900-2000zł (2 osoby)\n" + + "\n" + + "2K PLN per month ;)\n" + + "Zalety to niskie koszty ogrzewania, w zasadzie pomijalne.\n" + + "\n" + + "INTERNET od UPC. Mieszkanie wyposażone w gniazdo RTV i SAT.\n" + + "\n" + + "Umowa min. na 9 miesięcy. W razie pytań proszę się nie krępować :)\n" + + "\n" + + "Kontak mailowy lub telefoniczny (w rozsądnych godzinach):\n" + + "\n" + + "6 0 2 3 2 7 7 0 6\n" + + "\n" + + "nioobi (malpa) op (kropka) pl\n" + + "\n" + + "Pośrednikom z góry dziękuję za zainteresowanie!\n"; +} diff --git a/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserNonRequiredFieldsTest.java b/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserNonRequiredFieldsTest.java new file mode 100644 index 0000000..a22f080 --- /dev/null +++ b/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserNonRequiredFieldsTest.java @@ -0,0 +1,253 @@ +package parser.scrapper; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import parser.ReaderUtil; + + +public class GumtreeParserNonRequiredFieldsTest { + + private GumtreeAnnouncementParser gumtreeParser; + + @Before + public void setUp() { + gumtreeParser = new GumtreeAnnouncementParser(); + } + + @Test + public void testPhoneNumberFieldNotExists() { + // given + Document spyDocWithoutPhone = ReaderUtil.getDocumentToTest("/gumtree/sample-announcement-without-phone.html"); + Element contactElement = spyDocWithoutPhone.selectFirst(".vip.vip-contact"); + + // when + String phoneNumber = gumtreeParser.parsePhoneNumber(contactElement); + + // then + Assert.assertEquals(GumtreeExpectedProperties.expectedPhoneNumberNotExists, phoneNumber); + } + + @Test + public void testPhoneNumberFieldExists() { + // given + Document spyDocWithoutPhone = ReaderUtil.getDocumentToTest("/gumtree/sample-announcement-with-phone.html"); + Element contactElement = spyDocWithoutPhone.selectFirst(".vip.vip-contact"); + + // when + String phoneNumber = gumtreeParser.parsePhoneNumber(contactElement); + + // then + Assert.assertEquals(GumtreeExpectedProperties.expectedPhoneNumberExists, phoneNumber); + } + + @Test + public void testLessorFieldExists() { + // given + Document spyDocWithLessor = ReaderUtil.getDocumentToTest("/gumtree/sample-details-box-with-all-elements.html"); + Element detailsElement = spyDocWithLessor.selectFirst(".vip-details"); + + // when + String lessor = gumtreeParser.parseLessor(detailsElement); + + // then + Assert.assertEquals(GumtreeExpectedProperties.expectedLessor, lessor); + } + + @Test + public void testLessorFieldNotExists() { + // given + Document spyDocWithoutLessor = ReaderUtil.getDocumentToTest("/gumtree/sample-details-box-without-any-elements.html"); + Element detailsElement = spyDocWithoutLessor.selectFirst(".vip-details"); + + // when + String lessor = gumtreeParser.parseLessor(detailsElement); + + // then + Assert.assertNull(lessor); + } + + @Test + public void testPropertyTypeFieldExists() { + // given + Document spyDocWithoutLessor = ReaderUtil.getDocumentToTest("/gumtree/sample-details-box-with-all-elements.html"); + Element detailsElement = spyDocWithoutLessor.selectFirst(".vip-details"); + + // when + String propertyType = gumtreeParser.parsePropertyType(detailsElement); + + // then + Assert.assertEquals(GumtreeExpectedProperties.expectedPropertyType, propertyType); + } + + @Test + public void testPropertyTypeFieldNotExists() { + // given + Document spyDocWithoutLessor = ReaderUtil.getDocumentToTest("/gumtree/sample-details-box-without-any-elements.html"); + Element detailsElement = spyDocWithoutLessor.selectFirst(".vip-details"); + + // when + String propertyType = gumtreeParser.parsePropertyType(detailsElement); + + // then + Assert.assertNull(propertyType); + } + + @Test + public void testFlatAreaFieldExists() { + // given + Document spyDocWithoutLessor = ReaderUtil.getDocumentToTest("/gumtree/sample-details-box-with-all-elements.html"); + Element detailsElement = spyDocWithoutLessor.selectFirst(".vip-details"); + + // when + Double flatArea = gumtreeParser.parseFlatArea(detailsElement); + + // then + Assert.assertEquals(GumtreeExpectedProperties.expectedFlatArea, flatArea); + } + + @Test + public void testFlatAreaFieldNotExists() { + // given + Document spyDocWithoutLessor = ReaderUtil.getDocumentToTest("/gumtree/sample-details-box-without-any-elements.html"); + Element detailsElement = spyDocWithoutLessor.selectFirst(".vip-details"); + + // when + Double flatArea = gumtreeParser.parseFlatArea(detailsElement); + + // then + Assert.assertNull(flatArea); + } + + @Test + public void testRoomAmountFieldExists() { + // given + Document spyDocWithoutLessor = ReaderUtil.getDocumentToTest("/gumtree/sample-details-box-with-all-elements.html"); + Element detailsElement = spyDocWithoutLessor.selectFirst(".vip-details"); + + // when + Integer roomAmount = gumtreeParser.parseRoomAmount(detailsElement); + + // then + Assert.assertEquals(GumtreeExpectedProperties.expectedRoomAmount, roomAmount); + } + + @Test + public void testRoomAmountFieldNotExists() { + // given + Document spyDocWithoutLessor = ReaderUtil.getDocumentToTest("/gumtree/sample-details-box-without-any-elements.html"); + Element detailsElement = spyDocWithoutLessor.selectFirst(".vip-details"); + + // when + Integer roomAmount = gumtreeParser.parseRoomAmount(detailsElement); + + // then + Assert.assertNull(roomAmount); + } + + @Test + public void testBathAmountFieldExists() { + // given + Document spyDocWithoutLessor = ReaderUtil.getDocumentToTest("/gumtree/sample-details-box-with-all-elements.html"); + Element detailsElement = spyDocWithoutLessor.selectFirst(".vip-details"); + + // when + Integer bathAmount = gumtreeParser.parseBathAmount(detailsElement); + + // then + Assert.assertEquals(GumtreeExpectedProperties.expectedBathAmount, bathAmount); + } + + @Test + public void testBathAmountFieldNotExists() { + // given + Document spyDocWithoutLessor = ReaderUtil.getDocumentToTest("/gumtree/sample-details-box-without-any-elements.html"); + Element detailsElement = spyDocWithoutLessor.selectFirst(".vip-details"); + + // when + Integer bathAmount = gumtreeParser.parseBathAmount(detailsElement); + + // then + Assert.assertNull(bathAmount); + } + + @Test + public void testParkingFieldExists() { + // given + Document spyDocWithoutLessor = ReaderUtil.getDocumentToTest("/gumtree/sample-details-box-with-all-elements.html"); + Element detailsElement = spyDocWithoutLessor.selectFirst(".vip-details"); + + // when + String parking = gumtreeParser.parseParking(detailsElement); + + // then + Assert.assertEquals(GumtreeExpectedProperties.expectedParking, parking); + } + + @Test + public void testParkingFieldNotExists() { + // given + Document spyDocWithoutLessor = ReaderUtil.getDocumentToTest("/gumtree/sample-details-box-without-any-elements.html"); + Element detailsElement = spyDocWithoutLessor.selectFirst(".vip-details"); + + // when + String parking = gumtreeParser.parseParking(detailsElement); + + // then + Assert.assertNull(parking); + } + + @Test + public void testSmokersFieldExists() { + // given + Document spyDocWithoutLessor = ReaderUtil.getDocumentToTest("/gumtree/sample-details-box-with-all-elements.html"); + Element detailsElement = spyDocWithoutLessor.selectFirst(".vip-details"); + + // when + Boolean smokers = gumtreeParser.parserSmokers(detailsElement); + + // then + Assert.assertEquals(GumtreeExpectedProperties.expectedSmokers, smokers); + } + + @Test + public void testSmokersFieldNotExists() { + // given + Document spyDocWithoutLessor = ReaderUtil.getDocumentToTest("/gumtree/sample-details-box-without-any-elements.html"); + Element detailsElement = spyDocWithoutLessor.selectFirst(".vip-details"); + + // when + Boolean smokers = gumtreeParser.parserSmokers(detailsElement); + + // then + Assert.assertNull(smokers); + } + + @Test + public void testPetFriendlyFieldExists() { + // given + Document spyDocWithoutLessor = ReaderUtil.getDocumentToTest("/gumtree/sample-details-box-with-all-elements.html"); + Element detailsElement = spyDocWithoutLessor.selectFirst(".vip-details"); + + // when + Boolean petFriendly = gumtreeParser.parserPetFriendly(detailsElement); + + // then + Assert.assertEquals(GumtreeExpectedProperties.expectedPetFriendly, petFriendly); + } + + @Test + public void testPetFriendlyFieldNotExists() { + // given + Document spyDocWithoutLessor = ReaderUtil.getDocumentToTest("/gumtree/sample-details-box-without-any-elements.html"); + Element detailsElement = spyDocWithoutLessor.selectFirst(".vip-details"); + + // when + Boolean petFriendly = gumtreeParser.parserPetFriendly(detailsElement); + + // then + Assert.assertNull(petFriendly); + } +} \ No newline at end of file diff --git a/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserRequiredFieldsTest.java b/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserRequiredFieldsTest.java new file mode 100644 index 0000000..863f1ff --- /dev/null +++ b/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserRequiredFieldsTest.java @@ -0,0 +1,105 @@ +package parser.scrapper; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import parser.ReaderUtil; +import parser.exceptions.GumtreePageParseException; + +import java.io.IOException; +import java.math.BigDecimal; +import java.time.LocalDateTime; + +public class GumtreeParserRequiredFieldsTest { + + private GumtreeAnnouncementParser gumtreeParser; + private Document spyDoc; + + @Rule + public MockitoRule mockitoRule = MockitoJUnit.rule(); + + @Before + public void setUp() { + gumtreeParser = new GumtreeAnnouncementParser(); + spyDoc = ReaderUtil.getDocumentToTest("/gumtree/sample-announcement-without-phone.html"); + } + + @Test(expected = GumtreePageParseException.class) + public void throwGumtreePageParseExceptionInParsePage() throws IOException { + // given + AnnouncementParser spy = Mockito.spy(new GumtreeAnnouncementParser()); + + // when + Mockito.doThrow(GumtreePageParseException.class).when(spy).getPageContent(Mockito.anyString()); + + // then + spy.parsePage("test"); + } + + @Test + public void testParseTitle() { + // given + Element titleElement = spyDoc.selectFirst(".item-title"); + + // when + String title = gumtreeParser.parseTitle(titleElement); + + // then + Assert.assertEquals(GumtreeExpectedProperties.expectedTitle, title); + } + + @Test + public void testParsePrice() { + // given + Element priceElement = spyDoc.selectFirst(".price"); + + // when + BigDecimal price = gumtreeParser.parsePrice(priceElement); + + // then + Assert.assertEquals(GumtreeExpectedProperties.expectedPrice, price); + } + + @Test + public void testLessorName() { + // given + Element lessorNameElement = spyDoc.selectFirst(".username"); + + // when + String lessorName = gumtreeParser.parserLessorName(lessorNameElement); + + // then + Assert.assertEquals(GumtreeExpectedProperties.expectedLessorName, lessorName); + } + + + @Test + public void testCreationDate() { + // given + Element detailsElement = spyDoc.selectFirst(".vip.vip-contact"); + + // when + LocalDateTime dateTime = gumtreeParser.parseCreationDate(detailsElement); + + // then + Assert.assertEquals(GumtreeExpectedProperties.expectedCreationDateTime, dateTime); + } + + @Test + public void testDescription() { + // given + Element descriptionElement = spyDoc.selectFirst(".description"); + + // when + String description = gumtreeParser.parseDescription(descriptionElement); + + // then + Assert.assertEquals(GumtreeExpectedProperties.expectedDescription, description); + } +} diff --git a/announcement-processor-parser/src/test/resources/gumtree/sample-announcement-with-phone.html b/announcement-processor-parser/src/test/resources/gumtree/sample-announcement-with-phone.html new file mode 100644 index 0000000..b10284a --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/sample-announcement-with-phone.html @@ -0,0 +1,1102 @@ + + + + + + + + + + ŚCISŁE CENTRUM, 3 POKOJE, UEK, ENG – Kraków – 356135438 | Gumtree + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ Aby zapewnić najwyższą jakość usług i wygodne korzystanie z serwisu, używamy informacji zapisanych w przeglądarce za pomocą plików cookies (pol.: ciasteczek). Korzystając z serwisu wyrażasz zgodę na stosowanie plików + cookies. W każdej chwili możesz je zablokować korzystając z ustawień swojej przeglądarki internetowej. + +
+
+ + +
+
+ +
+
+
+
+ +
+
+
+ Dodaj ogłoszenie + +
+
+ +
+ +
+
+
+
+
+
+
+
+
+ Dodaj ogłoszenie +
+
+
+
+
+
+
+
+
+
+
+ +
+
+ + + + + +
+
+ + + + + +
+
+ +
+
+
+
+
+
+
+
+
+ Małopolskie + + + + Nieruchomości + + + + mieszkania i domy do wynajęcia | Kraków + + + + ogłoszenie 356135438 +
+
+
+
+
+ + +
+
+
+
+
+
+
+
+
+

ŚCISŁE CENTRUM, 3 POKOJE, UEK, ENG

+
+ 2 400 zł +
+
+
+
+
+
+ + + ŚCISŁE CENTRUM, 3 POKOJE, UEK, ENG + + +
+
+
+
+ ŚCISŁE CENTRUM, 3 POKOJE, UEK, ENG z Krakow, miniatura: 1 + ŚCISŁE CENTRUM, 3 POKOJE, UEK, ENG z Krakow, miniatura: 2 + ŚCISŁE CENTRUM, 3 POKOJE, UEK, ENG z Krakow, miniatura: 3 + ŚCISŁE CENTRUM, 3 POKOJE, UEK, ENG z Krakow, miniatura: 4 + ŚCISŁE CENTRUM, 3 POKOJE, UEK, ENG z Krakow, miniatura: 5 + ŚCISŁE CENTRUM, 3 POKOJE, UEK, ENG z Krakow, miniatura: 6 + ŚCISŁE CENTRUM, 3 POKOJE, UEK, ENG z Krakow, miniatura: 7 +
+
+ +
+
+
+ Dodaj takie ogłoszenie! +
+
+
+ +
    +
  • +
    + Data dodania + 18/11/2018 +
    • +
  • +
    + Lokalizacja + +
    + Kraków, + Małopolskie +
    +
    • + + + +
  • +
    + Do wynajęcia przez + Agencja +
    • +
  • +
    + Rodzaj nieruchomości + Mieszkanie +
    • +
  • +
    + Liczba pokoi + 3 pokoje +
    • +
  • +
    + Wielkość (m2) + 44 +
    • + + + + + + + + + + + + + + + + +
  • +
    • +
+
+ Zapraszamy do wynajęcia 3- pokojowego apartamentu niedaleko Galerii Krakowskiej

If You want to contact us in English, no problem!
Call: 795 002 620

Lokalizacja :

Mieszkanie znajduje się na 3 piętrze w 5 - piętrowym apartamentowcu przy ul. Rakowickiej na osiedlu Podkowa. Bardzo blisko do Uniwersytetu Ekonomicznego, Galerii Krakowskiej oraz Starego Miasta. Świetne połączenia tramwajowe i autobusowe ( Rondo Mogilskie w promieniu 500 m

Opis mieszkania:

Lokal nadaje się do zamieszkania przez 3- 4 osoby. Składa się z 3 pokojów, z których jeden (salon) połączony jest a aneksem kuchennym wyposażonym w lodówkę z zamrażarką, zmywarkę, płytę indukcyjną, mikrofalówkę. Salon zostanie umeblowany przez właściciela, pozostałe pokoje do zagospodarowania przez najemcę. Mieszkanie zaopatrzone w klimatyzację - możliwość dokładnej regulacji temperatury 

Atutem jest przestronny balkon.

Koszty:

Odstępne: 2400 zł
Czynsz administracyjny: ok. 400 zł
Kaucja zwrotna + prowizja dla biura

Możliwość wynajęcia garażu z 2 stanowiskami za 300 zł ( do negocjacji)

Zapraszam do kontaktu i oglądania mieszkania.

PAWEŁ CZECH
Tel.: 795 002 620
Email: pawel.incentro@gmail.com

www.incentro.pl



Treść niniejszego ogłoszenia nie stanowi oferty handlowej w rozumieniu Kodeksu Cywilnego.

Oferta wysłana z programu IMO dla biur nieruchomości +
+
+
+
+
+
+
+
    +
    • +
    • +
    • +
    • +
    • +
+
+
+
+
+
+
+
+
+
+ + Paweł Czech (Zobacz więcej ogłoszeń) + Użytkownik od 09-2017 +
+
+
+
+
+ 795002620 + E-mail +
+
+ + + +
+ +
+ +
+
    +
    • +
    • +
+
+ + + + + + + +
+ Klikając "Wyślij", wyrażasz zgodę na nasze + Zasady korzystania i + Politykę prywatności oraz zgadzasz się na otrzymywanie naszych newsletterów i ofert promocyjnych. +
+
+
+ +
+
+
+
+
+
+ + Dodaj do Zachowanych +
+
+
+
+ +
+ Porady Bezpieczeństwa - Twoje bezpieczeństwo jest dla nas ważne, zachęcamy do zachowania czujności. Dowiedz się więcej + Zgłoś ogłoszenie +
+ +
+ + +
    +
    • + +
    • + +
    • + +
    • + +
    • + +
+
+ + + + +
+ + +
+
+
+
+
+
+ +
Grzegórzki, Kraków
+
+
+
+
+
+
+
+
+
+ Popularne +
+ +
+
+
+
+ Proponowane +
+ +
+
+
+
+
+
+ Podobne ogłoszenia, które mogą Cię zainteresować. +
+
+
+
+
+
+ Przestrzenne Trzypokojowe mieszkanie w Centrum ! z Krakow, zobacz zdjęcie + +
+ Przestrzenne Trzypokojowe mieszkanie w Centrum ! + +
+ 2 600 zł +
+
+ English version below. OKAZJA! Trzypokojowe mieszkanie w Centrum !! LOKALIZACJA: Oferowane na wynajem mieszkanie znajduje się w bezpośrednim sąsiedztwie Starego miasta. Rynek Główny, Planty, Galeria Krakowska czy Dworzec PKP znajdują się w odległości 10min spacerem. Tuż obok Nowy kleparz z bogatą ofertą regionalnych wyrobów oraz świeżych warzyw i owoców. W pobliżu liczne przystanki komunikac ... +
+
+
+
+
+
+ ślicze 2 pokoje w Angel City z Krakow, zobacz zdjęcie + +
+ ślicze 2 pokoje w Angel City + +
+ 2 500 zł +
+
+ We speak english 601 612 620, 603 520 804, Мы говорим по-русски 601 250 251Hablamos Español 601 250 251www.maxnieruchomosci.comDo wynajęcia mieszkanie w centrum Krakowa. Apartament zlokalizowany jest przy ul. Pawiej , 5 minut od Rynku Głównego, 20 minut od Wawelu, w pobliżu Dworca Kolejowego i Autobusowego oraz największego Centrum Handlowego w Krakowie - Galerii Krakowskiej. Budynek wyposażony je ... +
+
+
+
+
+
+ Mieszkanie Kraków Śródmieście 54m2 (nr: BS1-MW-224850-14) z Krakow, zobacz zdjęcie + +
+ Mieszkanie Kraków Śródmieście 54m2 (nr: BS1-MW-224850-14) + +
+ 1 750 zł +
+
+ Do wynajęcia oferuję 2 pokojowe mieszkanie zlokalizowane w samym centrum Krakowa, na drugim piętrze w kamienicy przy ulicy Topolowej.Mieszkanie o powierzchni 54 m2 składa się z dwóch pokojów (jeden z nich jest przechodni), jasnej kuchni, łazienki i przedpokoju. Kuchnia z zabudowanymi szafkami stojącymi i wiszącymi wyposażona w niezbędny sprzęt AGD (lodówka, kuchenka gazowa, piekarnik, pralka), ora ... +
+
+
+
+
+
+ Nowoczesny Apartament - 110 m2 - centrum miasta z Krakow, zobacz zdjęcie + +
+ Nowoczesny Apartament - 110 m2 - centrum miasta + +
+ 7 500 zł +
+
+ English version below. Do zaoferowania piękny, przestronny i nowoczesny apartament bardzo blisko centrum miasta ! LOKALIZACJA: Apartament znajduję się w doskonałej lokalizacji w samym sercu Krakowa. Miejsce jest doskonale skomunikowane, zaledwie kilka minut od przystanku tramwajowego oraz hali targowej. Bliskość bulwarów wiślanych sprzyja spacerom a także porannemu joggingowi. W pobliżu znaj ... +
+
+
+
+
+
+ Dwupokojowe atrakcyjne mieszkanie - Stare miasto! z Krakow, zobacz zdjęcie + +
+ Dwupokojowe atrakcyjne mieszkanie - Stare miasto! + +
+ 2 500 zł +
+
+ English version below. LOKALIZACJA: Przedstawiane mieszkanie znajduje się w całkowicie odremontowanej kamienicy przy ulicy Rakowickiej. Jest to ulica znajdująca się bardzo blisko Dworca Głównego w Krakowie oraz Galerii Krakowskiej. Położenie ulicy Rakowickiej to bezpośrednie sąsiedztwo centrum miasta co zapewnia świetny punkt zarówno dla samego mieszkania jak i wynajmu. W bezpośrednim sąsiedzt ... +
+
+
+
+
+
+ Piękny 2 pokojowy apartament w Centrum z Krakow, zobacz zdjęcie + +
+ Piękny 2 pokojowy apartament w Centrum + +
+ 2 800 zł +
+
+ English version below. Do wynajęcia od zaraz piękny apartament w ścisłym Centrum Krakowa LOKALIZACJA: Położenie mieszkania jest jego niezaprzeczalnym atutem. Zlokalizowane na Starym Mieście w odległości zaledwie 10 minut spacerem od Rynku Głównego, 5 minut spacerem od Dworca Głównego i Galerii Krakowskiej. W najbliższej okolicy liczne sklepy (m.in. delikatesy spożywcze Piotr i Paweł), loka ... +
+
+
+
+
+
+ Nowa inwestycja - Kazimierz - 2 pokoje - 43 m2 z Krakow, zobacz zdjęcie + +
+ Nowa inwestycja - Kazimierz - 2 pokoje - 43 m2 + +
+ 2 400 zł +
+
+ English version below. Stylowy apartament położony przy ulicy Skawińskiej w dzielnicy Kazimierz. LOKALIZACJA: Mieszkanie znajduje się przy ul. Skawińskiej - Kazimierz - w odległości 2 min od Wisły. W pobliżu liczne punkty usługowo - handlowe: siłownia, fitness, kosmetyczka, kwiaciarnia, fryzjer, bank, sklepy spożywcze: Carrefour, Żabka. Jest to jedna z najbardziej prestiżowych, zabytkowych ... +
+
+
+
+
+ Więcej ogłoszeń +
+
+
+
+
+
+
+ +
+ + + + + + + +
+
+
+
+
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/announcement-processor-parser/src/test/resources/sample-gumtree-announcement.txt b/announcement-processor-parser/src/test/resources/gumtree/sample-announcement-without-phone.html similarity index 100% rename from announcement-processor-parser/src/test/resources/sample-gumtree-announcement.txt rename to announcement-processor-parser/src/test/resources/gumtree/sample-announcement-without-phone.html diff --git a/announcement-processor-parser/src/test/resources/gumtree/sample-details-box-with-all-elements.html b/announcement-processor-parser/src/test/resources/gumtree/sample-details-box-with-all-elements.html new file mode 100644 index 0000000..2a0126a --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/sample-details-box-with-all-elements.html @@ -0,0 +1,192 @@ +
+ +
    +
  • +
    + Data dodania + 17/11/2018 +
    • +
  • +
    + Lokalizacja + +
    + Kraków, + Małopolskie +
    +
    • + + + +
  • +
    + Do wynajęcia przez + Właściciel +
    • +
  • +
    + Dostępny + 17/11/2018 +
    • +
  • +
    + Rodzaj nieruchomości + Mieszkanie +
    • +
  • +
    + Liczba pokoi + Kawalerka lub garsoniera +
    • +
  • +
    + Liczba łazienek + 1 łazienka +
    • +
  • +
    + Wielkość (m2) + 40 +
    • +
  • +
    + Parking + Ulica +
    • +
  • +
    + Palący + Tak +
    • +
  • +
    + Przyjazne zwierzakom + Nie +
    • + + + + + + + + + + + + + + + + +
  • +
    • +
+
+ +
+
+ Proponuję mieszkanie na wynajem (kawalerka) o powierzchni 40m2 zlokalizowane przy ulicy Głowackiego 4 w Krakowie. Dzielnica Krowodrza. Rejon Bronowice. +
+
+
+
+
+ Kawalerka mieście się na 5 piętrze od strony południowej z widokiem na panoramę Krakowa. +
+
+ 3 minuty pieszo do przystanków tramwajowych. Bliskie sąsiedztwo Uniwersytetu Pedagogicznego, AGH, Wydziału Fizyki i Architektury PK. +
+
+
+
+
+ Budynek oddany do użytkowania w 2008r. Podłoga z litego drewna, w aneksie kuchennym i w łazience płytki ceramiczne. Mieszkanie wyposażone tak jak na zdjęciach: (lodówka, płyta grzewcza, piekarnik, pralka, stół, krzesła, łóżko 2os., łóżko 1os.) +
+
+ Możliwość usunięcia mebli wg upodobań wynajmującego jak i ich dokupienia. +
+
+
+
+
+
+ Nieruchomość posiada przestronny hol i windę. Na parterze zlokalizowane są sklepy m. in. Lewiatan - można w pantoflach robić zakupy ;) +
+
+
+
+
+ Koszty najmu (miesięczne): wynajem + 1500zł, + czynsz administracyjny + 250zł, + media wg zużycia (woda (~60zł/1os), prąd (~60zł/1os) , ogrzewanie, internet (60zł)). +
+
+
+
+
+
+ SZACOWANY CAŁKOWITY KOSZT wynajmu i eksploatacji miesięcznej: 1900-2000zł (2 osoby) +
+
+
+
+
+ 2K PLN per month ;) +
+
+
+
Zalety to niskie koszty ogrzewania, w zasadzie pomijalne. +
+
+
+
+
+
+ INTERNET od UPC. Mieszkanie wyposażone w gniazdo RTV i SAT. +
+
+
+
+
+ Umowa min. na 9 miesięcy. W razie pytań proszę się nie krępować :) +
+
+
+
+
+
+
+ Kontak mailowy lub telefoniczny (w rozsądnych godzinach): +
+
+
+
+
+ 6 0 2 3 2 7 7 0 6 +
+
+
+
+
+
+ nioobi (malpa) + op (kropka) + pl +
+


+


+

Możliwość oglądnięcia mieszkania

+


+

Palić papierosy można na dużym balkonie ;)

+


+
+
+
+
+ Pośrednikom z góry dziękuję za zainteresowanie! +
+


+


+
+
\ No newline at end of file diff --git a/announcement-processor-parser/src/test/resources/gumtree/sample-details-box-without-any-elements.html b/announcement-processor-parser/src/test/resources/gumtree/sample-details-box-without-any-elements.html new file mode 100644 index 0000000..02aa07a --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/sample-details-box-without-any-elements.html @@ -0,0 +1,118 @@ +
+ +
    +
  • +
    + Data dodania + 17/11/2018 +
    +
    • +
+
+ +
+
+ Proponuję mieszkanie na wynajem (kawalerka) o powierzchni 40m2 zlokalizowane przy ulicy Głowackiego 4 w Krakowie. Dzielnica Krowodrza. Rejon Bronowice. +
+
+
+
+
+ Kawalerka mieście się na 5 piętrze od strony południowej z widokiem na panoramę Krakowa. +
+
+ 3 minuty pieszo do przystanków tramwajowych. Bliskie sąsiedztwo Uniwersytetu Pedagogicznego, AGH, Wydziału Fizyki i Architektury PK. +
+
+
+
+
+ Budynek oddany do użytkowania w 2008r. Podłoga z litego drewna, w aneksie kuchennym i w łazience płytki ceramiczne. Mieszkanie wyposażone tak jak na zdjęciach: (lodówka, płyta grzewcza, piekarnik, pralka, stół, krzesła, łóżko 2os., łóżko 1os.) +
+
+ Możliwość usunięcia mebli wg upodobań wynajmującego jak i ich dokupienia. +
+
+
+
+
+
+ Nieruchomość posiada przestronny hol i windę. Na parterze zlokalizowane są sklepy m. in. Lewiatan - można w pantoflach robić zakupy ;) +
+
+
+
+
+ Koszty najmu (miesięczne): wynajem + 1500zł, + czynsz administracyjny + 250zł, + media wg zużycia (woda (~60zł/1os), prąd (~60zł/1os) , ogrzewanie, internet (60zł)). +
+
+
+
+
+
+ SZACOWANY CAŁKOWITY KOSZT wynajmu i eksploatacji miesięcznej: 1900-2000zł (2 osoby) +
+
+
+
+
+ 2K PLN per month ;) +
+
+
+
Zalety to niskie koszty ogrzewania, w zasadzie pomijalne. +
+
+
+
+
+
+ INTERNET od UPC. Mieszkanie wyposażone w gniazdo RTV i SAT. +
+
+
+
+
+ Umowa min. na 9 miesięcy. W razie pytań proszę się nie krępować :) +
+
+
+
+
+
+
+ Kontak mailowy lub telefoniczny (w rozsądnych godzinach): +
+
+
+
+
+ 6 0 2 3 2 7 7 0 6 +
+
+
+
+
+
+ nioobi (malpa) + op (kropka) + pl +
+


+


+

Możliwość oglądnięcia mieszkania

+


+

Palić papierosy można na dużym balkonie ;)

+


+
+
+
+
+ Pośrednikom z góry dziękuję za zainteresowanie! +
+


+


+
+
\ No newline at end of file From 9f5185e57f1384aace5e4fc2b1b0f78f5902d74c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 18 Nov 2018 19:57:50 +0100 Subject: [PATCH 044/135] Additional tests for properties options. #3 --- .../GumtreeBathAmountOptionsTests.java | 71 ++++++++++++++ .../options/GumtreeParkingOptionsTests.java | 71 ++++++++++++++ .../GumtreePetFriendlyOptionsTests.java | 45 +++++++++ .../GumtreePropertyTypeOptionsTests.java | 58 +++++++++++ .../options/GumtreeRoomAmountOptionsTest.java | 97 +++++++++++++++++++ .../options/GumtreeSmokerOptionsTests.java | 45 +++++++++ .../resources/gumtree/options/baths/1.html | 10 ++ .../resources/gumtree/options/baths/2.html | 10 ++ .../resources/gumtree/options/baths/3.html | 10 ++ .../gumtree/options/baths/4-or-more.html | 10 ++ .../gumtree/options/parking/absence.html | 10 ++ .../gumtree/options/parking/covered.html | 10 ++ .../gumtree/options/parking/garage.html | 10 ++ .../gumtree/options/parking/street.html | 10 ++ .../gumtree/options/pet-friendly/no.html | 10 ++ .../gumtree/options/pet-friendly/yes.html | 10 ++ .../gumtree/options/property-type/flat.html | 10 ++ .../gumtree/options/property-type/house.html | 10 ++ .../gumtree/options/property-type/other.html | 10 ++ .../resources/gumtree/options/rooms/1.html | 10 ++ .../resources/gumtree/options/rooms/2.html | 10 ++ .../resources/gumtree/options/rooms/3.html | 10 ++ .../resources/gumtree/options/rooms/4.html | 10 ++ .../resources/gumtree/options/rooms/5.html | 10 ++ .../gumtree/options/rooms/6-or-more.html | 10 ++ .../resources/gumtree/options/smoker/no.html | 10 ++ .../resources/gumtree/options/smoker/yes.html | 10 ++ 27 files changed, 597 insertions(+) create mode 100644 announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreeBathAmountOptionsTests.java create mode 100644 announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreeParkingOptionsTests.java create mode 100644 announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreePetFriendlyOptionsTests.java create mode 100644 announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreePropertyTypeOptionsTests.java create mode 100644 announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreeRoomAmountOptionsTest.java create mode 100644 announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreeSmokerOptionsTests.java create mode 100644 announcement-processor-parser/src/test/resources/gumtree/options/baths/1.html create mode 100644 announcement-processor-parser/src/test/resources/gumtree/options/baths/2.html create mode 100644 announcement-processor-parser/src/test/resources/gumtree/options/baths/3.html create mode 100644 announcement-processor-parser/src/test/resources/gumtree/options/baths/4-or-more.html create mode 100644 announcement-processor-parser/src/test/resources/gumtree/options/parking/absence.html create mode 100644 announcement-processor-parser/src/test/resources/gumtree/options/parking/covered.html create mode 100644 announcement-processor-parser/src/test/resources/gumtree/options/parking/garage.html create mode 100644 announcement-processor-parser/src/test/resources/gumtree/options/parking/street.html create mode 100644 announcement-processor-parser/src/test/resources/gumtree/options/pet-friendly/no.html create mode 100644 announcement-processor-parser/src/test/resources/gumtree/options/pet-friendly/yes.html create mode 100644 announcement-processor-parser/src/test/resources/gumtree/options/property-type/flat.html create mode 100644 announcement-processor-parser/src/test/resources/gumtree/options/property-type/house.html create mode 100644 announcement-processor-parser/src/test/resources/gumtree/options/property-type/other.html create mode 100644 announcement-processor-parser/src/test/resources/gumtree/options/rooms/1.html create mode 100644 announcement-processor-parser/src/test/resources/gumtree/options/rooms/2.html create mode 100644 announcement-processor-parser/src/test/resources/gumtree/options/rooms/3.html create mode 100644 announcement-processor-parser/src/test/resources/gumtree/options/rooms/4.html create mode 100644 announcement-processor-parser/src/test/resources/gumtree/options/rooms/5.html create mode 100644 announcement-processor-parser/src/test/resources/gumtree/options/rooms/6-or-more.html create mode 100644 announcement-processor-parser/src/test/resources/gumtree/options/smoker/no.html create mode 100644 announcement-processor-parser/src/test/resources/gumtree/options/smoker/yes.html diff --git a/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreeBathAmountOptionsTests.java b/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreeBathAmountOptionsTests.java new file mode 100644 index 0000000..b629c07 --- /dev/null +++ b/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreeBathAmountOptionsTests.java @@ -0,0 +1,71 @@ +package parser.scrapper.options; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import parser.ReaderUtil; +import parser.scrapper.GumtreeAnnouncementParser; + +public class GumtreeBathAmountOptionsTests { + + private GumtreeAnnouncementParser gumtreeParser; + + @Before + public void setUp() { + gumtreeParser = new GumtreeAnnouncementParser(); + } + + @Test + public void testBathsOne() { + // given + Document spyDoc = ReaderUtil.getDocumentToTest("/gumtree/options/baths/1.html"); + Element detailsElement = spyDoc.selectFirst(".vip-details"); + + // when + Integer bathsAmount = gumtreeParser.parseBathAmount(detailsElement); + + // then + Assert.assertEquals(Integer.valueOf(1), bathsAmount); + } + + @Test + public void testBathsTwo(){ + // given + Document spyDoc = ReaderUtil.getDocumentToTest("/gumtree/options/baths/2.html"); + Element detailsElement = spyDoc.selectFirst(".vip-details"); + + // when + Integer bathsAmount = gumtreeParser.parseBathAmount(detailsElement); + + // then + Assert.assertEquals(Integer.valueOf(2), bathsAmount); + } + + @Test + public void testBathsThree(){ + // given + Document spyDoc = ReaderUtil.getDocumentToTest("/gumtree/options/baths/3.html"); + Element detailsElement = spyDoc.selectFirst(".vip-details"); + + // when + Integer bathsAmount = gumtreeParser.parseBathAmount(detailsElement); + + // then + Assert.assertEquals(Integer.valueOf(3), bathsAmount); + } + + @Test + public void testBathsFourOrMore(){ + // given + Document spyDoc = ReaderUtil.getDocumentToTest("/gumtree/options/baths/4-or-more.html"); + Element detailsElement = spyDoc.selectFirst(".vip-details"); + + // when + Integer bathsAmount = gumtreeParser.parseBathAmount(detailsElement); + + // then + Assert.assertEquals(Integer.valueOf(4), bathsAmount); + } +} diff --git a/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreeParkingOptionsTests.java b/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreeParkingOptionsTests.java new file mode 100644 index 0000000..8aa228e --- /dev/null +++ b/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreeParkingOptionsTests.java @@ -0,0 +1,71 @@ +package parser.scrapper.options; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import parser.ReaderUtil; +import parser.scrapper.GumtreeAnnouncementParser; + +public class GumtreeParkingOptionsTests { + + private GumtreeAnnouncementParser gumtreeParser; + + @Before + public void setUp() { + gumtreeParser = new GumtreeAnnouncementParser(); + } + + @Test + public void testParkingAbsence() { + // given + Document spyDoc = ReaderUtil.getDocumentToTest("/gumtree/options/parking/absence.html"); + Element detailsElement = spyDoc.selectFirst(".vip-details"); + + // when + String parking = gumtreeParser.parseParking(detailsElement); + + // then + Assert.assertEquals("Brak", parking); + } + + @Test + public void testParkingCovered() { + // given + Document spyDoc = ReaderUtil.getDocumentToTest("/gumtree/options/parking/covered.html"); + Element detailsElement = spyDoc.selectFirst(".vip-details"); + + // when + String parking = gumtreeParser.parseParking(detailsElement); + + // then + Assert.assertEquals("Kryty", parking); + } + + @Test + public void testParkingGarage() { + // given + Document spyDoc = ReaderUtil.getDocumentToTest("/gumtree/options/parking/garage.html"); + Element detailsElement = spyDoc.selectFirst(".vip-details"); + + // when + String parking = gumtreeParser.parseParking(detailsElement); + + // then + Assert.assertEquals("Garaż", parking); + } + + @Test + public void testParkingStreet() { + // given + Document spyDoc = ReaderUtil.getDocumentToTest("/gumtree/options/parking/street.html"); + Element detailsElement = spyDoc.selectFirst(".vip-details"); + + // when + String parking = gumtreeParser.parseParking(detailsElement); + + // then + Assert.assertEquals("Ulica", parking); + } +} diff --git a/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreePetFriendlyOptionsTests.java b/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreePetFriendlyOptionsTests.java new file mode 100644 index 0000000..4a3c29d --- /dev/null +++ b/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreePetFriendlyOptionsTests.java @@ -0,0 +1,45 @@ +package parser.scrapper.options; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import parser.ReaderUtil; +import parser.scrapper.GumtreeAnnouncementParser; + +public class GumtreePetFriendlyOptionsTests { + + private GumtreeAnnouncementParser gumtreeParser; + + @Before + public void setUp() { + gumtreeParser = new GumtreeAnnouncementParser(); + } + + @Test + public void testPetFriendlyYes(){ + // given + Document spyDoc = ReaderUtil.getDocumentToTest("/gumtree/options/pet-friendly/yes.html"); + Element detailsElement = spyDoc.selectFirst(".vip-details"); + + // when + Boolean petFriendly = gumtreeParser.parserPetFriendly(detailsElement); + + // then + Assert.assertTrue(petFriendly); + } + + @Test + public void testPetFriendlyNo(){ + // given + Document spyDoc = ReaderUtil.getDocumentToTest("/gumtree/options/pet-friendly/no.html"); + Element detailsElement = spyDoc.selectFirst(".vip-details"); + + // when + Boolean petFriendly = gumtreeParser.parserPetFriendly(detailsElement); + + // then + Assert.assertFalse(petFriendly); + } +} diff --git a/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreePropertyTypeOptionsTests.java b/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreePropertyTypeOptionsTests.java new file mode 100644 index 0000000..e66f2fc --- /dev/null +++ b/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreePropertyTypeOptionsTests.java @@ -0,0 +1,58 @@ +package parser.scrapper.options; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import parser.ReaderUtil; +import parser.scrapper.GumtreeAnnouncementParser; + +public class GumtreePropertyTypeOptionsTests { + + private GumtreeAnnouncementParser gumtreeParser; + + @Before + public void setUp() { + gumtreeParser = new GumtreeAnnouncementParser(); + } + + @Test + public void testPropertyTypeFlat() { + // given + Document spyDoc = ReaderUtil.getDocumentToTest("/gumtree/options/property-type/flat.html"); + Element detailsElement = spyDoc.selectFirst(".vip-details"); + + // when + String propertyType = gumtreeParser.parsePropertyType(detailsElement); + + // then + Assert.assertEquals("Mieszkanie", propertyType); + } + + @Test + public void testPropertyTypeHouse(){ + // given + Document spyDoc = ReaderUtil.getDocumentToTest("/gumtree/options/property-type/house.html"); + Element detailsElement = spyDoc.selectFirst(".vip-details"); + + // when + String propertyType = gumtreeParser.parsePropertyType(detailsElement); + + // then + Assert.assertEquals("Dom", propertyType); + } + + @Test + public void testPropertyTypeOther(){ + // given + Document spyDoc = ReaderUtil.getDocumentToTest("/gumtree/options/property-type/other.html"); + Element detailsElement = spyDoc.selectFirst(".vip-details"); + + // when + String propertyType = gumtreeParser.parsePropertyType(detailsElement); + + // then + Assert.assertEquals("Inne", propertyType); + } +} diff --git a/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreeRoomAmountOptionsTest.java b/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreeRoomAmountOptionsTest.java new file mode 100644 index 0000000..9158b1d --- /dev/null +++ b/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreeRoomAmountOptionsTest.java @@ -0,0 +1,97 @@ +package parser.scrapper.options; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import parser.ReaderUtil; +import parser.scrapper.GumtreeAnnouncementParser; + +public class GumtreeRoomAmountOptionsTest { + + private GumtreeAnnouncementParser gumtreeParser; + + @Before + public void setUp() { + gumtreeParser = new GumtreeAnnouncementParser(); + } + + @Test + public void testRoomAmountOne(){ + // given + Document spyDoc = ReaderUtil.getDocumentToTest("/gumtree/options/rooms/1.html"); + Element detailsElement = spyDoc.selectFirst(".vip-details"); + + // when + Integer roomAmount = gumtreeParser.parseRoomAmount(detailsElement); + + // then + Assert.assertEquals(Integer.valueOf(1), roomAmount); + } + + @Test + public void testRoomAmountTwo(){ + // given + Document spyDoc = ReaderUtil.getDocumentToTest("/gumtree/options/rooms/2.html"); + Element detailsElement = spyDoc.selectFirst(".vip-details"); + + // when + Integer roomAmount = gumtreeParser.parseRoomAmount(detailsElement); + + // then + Assert.assertEquals(Integer.valueOf(2), roomAmount); + } + + @Test + public void testRoomAmountThree(){ + // given + Document spyDoc = ReaderUtil.getDocumentToTest("/gumtree/options/rooms/3.html"); + Element detailsElement = spyDoc.selectFirst(".vip-details"); + + // when + Integer roomAmount = gumtreeParser.parseRoomAmount(detailsElement); + + // then + Assert.assertEquals(Integer.valueOf(3), roomAmount); + } + + @Test + public void testRoomAmountFour(){ + // given + Document spyDoc = ReaderUtil.getDocumentToTest("/gumtree/options/rooms/4.html"); + Element detailsElement = spyDoc.selectFirst(".vip-details"); + + // when + Integer roomAmount = gumtreeParser.parseRoomAmount(detailsElement); + + // then + Assert.assertEquals(Integer.valueOf(4), roomAmount); + } + + @Test + public void testRoomAmountFive(){ + // given + Document spyDoc = ReaderUtil.getDocumentToTest("/gumtree/options/rooms/5.html"); + Element detailsElement = spyDoc.selectFirst(".vip-details"); + + // when + Integer roomAmount = gumtreeParser.parseRoomAmount(detailsElement); + + // then + Assert.assertEquals(Integer.valueOf(5), roomAmount); + } + + @Test + public void testRoomAmountSixOrMore(){ + // given + Document spyDoc = ReaderUtil.getDocumentToTest("/gumtree/options/rooms/6-or-more.html"); + Element detailsElement = spyDoc.selectFirst(".vip-details"); + + // when + Integer roomAmount = gumtreeParser.parseRoomAmount(detailsElement); + + // then + Assert.assertEquals(Integer.valueOf(6), roomAmount); + } +} diff --git a/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreeSmokerOptionsTests.java b/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreeSmokerOptionsTests.java new file mode 100644 index 0000000..29f6e66 --- /dev/null +++ b/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreeSmokerOptionsTests.java @@ -0,0 +1,45 @@ +package parser.scrapper.options; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import parser.ReaderUtil; +import parser.scrapper.GumtreeAnnouncementParser; + +public class GumtreeSmokerOptionsTests { + + private GumtreeAnnouncementParser gumtreeParser; + + @Before + public void setUp() { + gumtreeParser = new GumtreeAnnouncementParser(); + } + + @Test + public void testSmokersYes(){ + // given + Document spyDoc = ReaderUtil.getDocumentToTest("/gumtree/options/smoker/yes.html"); + Element detailsElement = spyDoc.selectFirst(".vip-details"); + + // when + Boolean smokers = gumtreeParser.parserSmokers(detailsElement); + + // then + Assert.assertTrue(smokers); + } + + @Test + public void testSmokersNo(){ + // given + Document spyDoc = ReaderUtil.getDocumentToTest("/gumtree/options/smoker/no.html"); + Element detailsElement = spyDoc.selectFirst(".vip-details"); + + // when + Boolean smokers = gumtreeParser.parserSmokers(detailsElement); + + // then + Assert.assertFalse(smokers); + } +} diff --git a/announcement-processor-parser/src/test/resources/gumtree/options/baths/1.html b/announcement-processor-parser/src/test/resources/gumtree/options/baths/1.html new file mode 100644 index 0000000..8040001 --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/options/baths/1.html @@ -0,0 +1,10 @@ +
+
    +
  • +
    + Liczba łazienek + 1 łazienka +
    +
    • +
+
\ No newline at end of file diff --git a/announcement-processor-parser/src/test/resources/gumtree/options/baths/2.html b/announcement-processor-parser/src/test/resources/gumtree/options/baths/2.html new file mode 100644 index 0000000..4c497bf --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/options/baths/2.html @@ -0,0 +1,10 @@ +
+
    +
  • +
    + Liczba łazienek + 2 łazienki +
    +
    • +
+
\ No newline at end of file diff --git a/announcement-processor-parser/src/test/resources/gumtree/options/baths/3.html b/announcement-processor-parser/src/test/resources/gumtree/options/baths/3.html new file mode 100644 index 0000000..5b62dbe --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/options/baths/3.html @@ -0,0 +1,10 @@ +
+
    +
  • +
    + Liczba łazienek + 3 łazienki +
    +
    • +
+
\ No newline at end of file diff --git a/announcement-processor-parser/src/test/resources/gumtree/options/baths/4-or-more.html b/announcement-processor-parser/src/test/resources/gumtree/options/baths/4-or-more.html new file mode 100644 index 0000000..31de9e5 --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/options/baths/4-or-more.html @@ -0,0 +1,10 @@ +
+
    +
  • +
    + Liczba łazienek + 4 lub więcej łazienek +
    +
    • +
+
\ No newline at end of file diff --git a/announcement-processor-parser/src/test/resources/gumtree/options/parking/absence.html b/announcement-processor-parser/src/test/resources/gumtree/options/parking/absence.html new file mode 100644 index 0000000..d0cdf87 --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/options/parking/absence.html @@ -0,0 +1,10 @@ +
+
    +
  • +
    + Parking + Brak +
    +
    • +
+
\ No newline at end of file diff --git a/announcement-processor-parser/src/test/resources/gumtree/options/parking/covered.html b/announcement-processor-parser/src/test/resources/gumtree/options/parking/covered.html new file mode 100644 index 0000000..291ba69 --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/options/parking/covered.html @@ -0,0 +1,10 @@ +
+
    +
  • +
    + Parking + Kryty +
    +
    • +
+
\ No newline at end of file diff --git a/announcement-processor-parser/src/test/resources/gumtree/options/parking/garage.html b/announcement-processor-parser/src/test/resources/gumtree/options/parking/garage.html new file mode 100644 index 0000000..e7ec28d --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/options/parking/garage.html @@ -0,0 +1,10 @@ +
+
    +
  • +
    + Parking + Garaż +
    +
    • +
+
\ No newline at end of file diff --git a/announcement-processor-parser/src/test/resources/gumtree/options/parking/street.html b/announcement-processor-parser/src/test/resources/gumtree/options/parking/street.html new file mode 100644 index 0000000..28b83e7 --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/options/parking/street.html @@ -0,0 +1,10 @@ +
+
    +
  • +
    + Parking + Ulica +
    +
    • +
+
\ No newline at end of file diff --git a/announcement-processor-parser/src/test/resources/gumtree/options/pet-friendly/no.html b/announcement-processor-parser/src/test/resources/gumtree/options/pet-friendly/no.html new file mode 100644 index 0000000..8025bab --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/options/pet-friendly/no.html @@ -0,0 +1,10 @@ +
+
    +
  • +
    + Przyjazne zwierzakom + Nie +
    +
    • +
+
\ No newline at end of file diff --git a/announcement-processor-parser/src/test/resources/gumtree/options/pet-friendly/yes.html b/announcement-processor-parser/src/test/resources/gumtree/options/pet-friendly/yes.html new file mode 100644 index 0000000..4cd4535 --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/options/pet-friendly/yes.html @@ -0,0 +1,10 @@ +
+
    +
  • +
    + Przyjazne zwierzakom + Tak +
    +
    • +
+
\ No newline at end of file diff --git a/announcement-processor-parser/src/test/resources/gumtree/options/property-type/flat.html b/announcement-processor-parser/src/test/resources/gumtree/options/property-type/flat.html new file mode 100644 index 0000000..208649e --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/options/property-type/flat.html @@ -0,0 +1,10 @@ +
+
    +
  • +
    + Rodzaj nieruchomości + Mieszkanie +
    +
    • +
+
\ No newline at end of file diff --git a/announcement-processor-parser/src/test/resources/gumtree/options/property-type/house.html b/announcement-processor-parser/src/test/resources/gumtree/options/property-type/house.html new file mode 100644 index 0000000..734ca2b --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/options/property-type/house.html @@ -0,0 +1,10 @@ +
+
    +
  • +
    + Rodzaj nieruchomości + Dom +
    +
    • +
+
\ No newline at end of file diff --git a/announcement-processor-parser/src/test/resources/gumtree/options/property-type/other.html b/announcement-processor-parser/src/test/resources/gumtree/options/property-type/other.html new file mode 100644 index 0000000..fc524bd --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/options/property-type/other.html @@ -0,0 +1,10 @@ +
+
    +
  • +
    + Rodzaj nieruchomości + Inne +
    +
    • +
+
\ No newline at end of file diff --git a/announcement-processor-parser/src/test/resources/gumtree/options/rooms/1.html b/announcement-processor-parser/src/test/resources/gumtree/options/rooms/1.html new file mode 100644 index 0000000..59a4ce6 --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/options/rooms/1.html @@ -0,0 +1,10 @@ +
+
    +
  • +
    + Liczba pokoi + Kawalerka lub garsoniera +
    +
    • +
+
\ No newline at end of file diff --git a/announcement-processor-parser/src/test/resources/gumtree/options/rooms/2.html b/announcement-processor-parser/src/test/resources/gumtree/options/rooms/2.html new file mode 100644 index 0000000..b3b23ae --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/options/rooms/2.html @@ -0,0 +1,10 @@ +
+
    +
  • +
    + Liczba pokoi + 2 pokoje +
    +
    • +
+
\ No newline at end of file diff --git a/announcement-processor-parser/src/test/resources/gumtree/options/rooms/3.html b/announcement-processor-parser/src/test/resources/gumtree/options/rooms/3.html new file mode 100644 index 0000000..f1f0acc --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/options/rooms/3.html @@ -0,0 +1,10 @@ +
+
    +
  • +
    + Liczba pokoi + 3 pokoje +
    +
    • +
+
\ No newline at end of file diff --git a/announcement-processor-parser/src/test/resources/gumtree/options/rooms/4.html b/announcement-processor-parser/src/test/resources/gumtree/options/rooms/4.html new file mode 100644 index 0000000..5682d91 --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/options/rooms/4.html @@ -0,0 +1,10 @@ +
+
    +
  • +
    + Liczba pokoi + 4 pokoje +
    +
    • +
+
\ No newline at end of file diff --git a/announcement-processor-parser/src/test/resources/gumtree/options/rooms/5.html b/announcement-processor-parser/src/test/resources/gumtree/options/rooms/5.html new file mode 100644 index 0000000..4f4fdba --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/options/rooms/5.html @@ -0,0 +1,10 @@ +
+
    +
  • +
    + Liczba pokoi + 5 pokoi +
    +
    • +
+
\ No newline at end of file diff --git a/announcement-processor-parser/src/test/resources/gumtree/options/rooms/6-or-more.html b/announcement-processor-parser/src/test/resources/gumtree/options/rooms/6-or-more.html new file mode 100644 index 0000000..039c6f6 --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/options/rooms/6-or-more.html @@ -0,0 +1,10 @@ +
+
    +
  • +
    + Liczba pokoi + 6 lub więcej pokoi +
    +
    • +
+
\ No newline at end of file diff --git a/announcement-processor-parser/src/test/resources/gumtree/options/smoker/no.html b/announcement-processor-parser/src/test/resources/gumtree/options/smoker/no.html new file mode 100644 index 0000000..1f370e1 --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/options/smoker/no.html @@ -0,0 +1,10 @@ +
+
    +
  • +
    + Palący + Nie +
    +
    • +
+
\ No newline at end of file diff --git a/announcement-processor-parser/src/test/resources/gumtree/options/smoker/yes.html b/announcement-processor-parser/src/test/resources/gumtree/options/smoker/yes.html new file mode 100644 index 0000000..af693b6 --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/options/smoker/yes.html @@ -0,0 +1,10 @@ +
+
    +
  • +
    + Palący + Tak +
    +
    • +
+
\ No newline at end of file From ab19443afb49cd8ebb12cd0525b6f4c4df5fe1b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 18 Nov 2018 20:43:58 +0100 Subject: [PATCH 045/135] Added own exception when property is not valid for provider. #3 --- ...tyNotValidForGumtreeProviderException.java | 8 ++++ .../PropertyNotValidForProviderException.java | 8 ++++ .../scrapper/GumtreeAnnouncementParser.java | 8 ++-- .../GumtreeParserNotAvailableFieldsTest.java | 47 +++++++++++++++++++ 4 files changed, 67 insertions(+), 4 deletions(-) create mode 100644 announcement-processor-parser/src/main/java/parser/exceptions/PropertyNotValidForGumtreeProviderException.java create mode 100644 announcement-processor-parser/src/main/java/parser/exceptions/PropertyNotValidForProviderException.java create mode 100644 announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserNotAvailableFieldsTest.java diff --git a/announcement-processor-parser/src/main/java/parser/exceptions/PropertyNotValidForGumtreeProviderException.java b/announcement-processor-parser/src/main/java/parser/exceptions/PropertyNotValidForGumtreeProviderException.java new file mode 100644 index 0000000..004beb9 --- /dev/null +++ b/announcement-processor-parser/src/main/java/parser/exceptions/PropertyNotValidForGumtreeProviderException.java @@ -0,0 +1,8 @@ +package parser.exceptions; + +public class PropertyNotValidForGumtreeProviderException extends PropertyNotValidForProviderException { + + public PropertyNotValidForGumtreeProviderException(String propertyName) { + super("Gumtree", propertyName); + } +} diff --git a/announcement-processor-parser/src/main/java/parser/exceptions/PropertyNotValidForProviderException.java b/announcement-processor-parser/src/main/java/parser/exceptions/PropertyNotValidForProviderException.java new file mode 100644 index 0000000..b39d73b --- /dev/null +++ b/announcement-processor-parser/src/main/java/parser/exceptions/PropertyNotValidForProviderException.java @@ -0,0 +1,8 @@ +package parser.exceptions; + +class PropertyNotValidForProviderException extends RuntimeException { + + PropertyNotValidForProviderException(String providerName, String propertyName) { + super("Property named: '" + propertyName + "' is not valid for provider: '" + providerName + "'"); + } +} diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java index 30f7c44..d9ebfaa 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java @@ -3,9 +3,9 @@ import lombok.extern.slf4j.Slf4j; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.springframework.cglib.core.Local; import parser.Announcement; import parser.exceptions.GumtreePageParseException; +import parser.exceptions.PropertyNotValidForGumtreeProviderException; import java.io.IOException; import java.math.BigDecimal; @@ -97,16 +97,16 @@ public Boolean parserPetFriendly(Element petFriendlyElement) { @Override public BigDecimal parseAdditionalRentCost(Element additionalRentCostElement) { - return BigDecimal.ZERO; + throw new PropertyNotValidForGumtreeProviderException("additional rent cost"); } @Override public String parseLevel(Element levelElement) { - return ""; + throw new PropertyNotValidForGumtreeProviderException("level"); } @Override public String parseFurnishings(Element furnishingsElement) { - return ""; + throw new PropertyNotValidForGumtreeProviderException("furnishings"); } } diff --git a/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserNotAvailableFieldsTest.java b/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserNotAvailableFieldsTest.java new file mode 100644 index 0000000..7b14813 --- /dev/null +++ b/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserNotAvailableFieldsTest.java @@ -0,0 +1,47 @@ +package parser.scrapper; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.junit.Before; +import org.junit.Test; +import parser.ReaderUtil; +import parser.exceptions.PropertyNotValidForGumtreeProviderException; + +public class GumtreeParserNotAvailableFieldsTest { + private GumtreeAnnouncementParser gumtreeParser; + + @Before + public void setUp() { + gumtreeParser = new GumtreeAnnouncementParser(); + } + + @Test(expected = PropertyNotValidForGumtreeProviderException.class) + public void testAdditionalRentCostsField() { + // given + Document spyDocWithoutPhone = ReaderUtil.getDocumentToTest("/gumtree/sample-announcement-without-phone.html"); + Element element = spyDocWithoutPhone.selectFirst(".vip.vip-contact"); + + // when + gumtreeParser.parseAdditionalRentCost(element); + } + + @Test(expected = PropertyNotValidForGumtreeProviderException.class) + public void testLevelField() { + // given + Document spyDocWithoutPhone = ReaderUtil.getDocumentToTest("/gumtree/sample-announcement-without-phone.html"); + Element element = spyDocWithoutPhone.selectFirst(".vip.vip-contact"); + + // when + gumtreeParser.parseLevel(element); + } + + @Test(expected = PropertyNotValidForGumtreeProviderException.class) + public void testFurnishingsField() { + // given + Document spyDocWithoutPhone = ReaderUtil.getDocumentToTest("/gumtree/sample-announcement-without-phone.html"); + Element element = spyDocWithoutPhone.selectFirst(".vip.vip-contact"); + + // when + gumtreeParser.parseFurnishings(element); + } +} From 738c043799b7577f5c8641942ac3637e29884b62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 18 Nov 2018 20:53:26 +0100 Subject: [PATCH 046/135] Changed package to public scope #3 --- .../src/main/java/parser/Announcement.java | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/announcement-processor-parser/src/main/java/parser/Announcement.java b/announcement-processor-parser/src/main/java/parser/Announcement.java index 8f2af6b..6fd3b2d 100644 --- a/announcement-processor-parser/src/main/java/parser/Announcement.java +++ b/announcement-processor-parser/src/main/java/parser/Announcement.java @@ -54,7 +54,7 @@ private Announcement(AnnouncementBuilder builder) { this.phoneNumber = builder.phoneNumber; } - static AnnouncementBuilder builder() { + public static AnnouncementBuilder builder() { return new AnnouncementBuilder(); } @@ -79,102 +79,102 @@ public static final class AnnouncementBuilder { private String url; private String phoneNumber; - AnnouncementBuilder title(String title) { + public AnnouncementBuilder title(String title) { this.title = title; return this; } - AnnouncementBuilder price(BigDecimal price) { + public AnnouncementBuilder price(BigDecimal price) { this.price = price; return this; } - AnnouncementBuilder creationDate(LocalDateTime creationDate) { + public AnnouncementBuilder creationDate(LocalDateTime creationDate) { this.creationDate = creationDate; return this; } - AnnouncementBuilder lessor(String lessor) { + public AnnouncementBuilder lessor(String lessor) { this.lessor = lessor; return this; } - AnnouncementBuilder propertyType(String propertyType) { + public AnnouncementBuilder propertyType(String propertyType) { this.propertyType = propertyType; return this; } - AnnouncementBuilder flatArea(Double flatArea) { + public AnnouncementBuilder flatArea(Double flatArea) { this.flatArea = flatArea; return this; } - AnnouncementBuilder roomAmount(Integer roomAmount) { + public AnnouncementBuilder roomAmount(Integer roomAmount) { this.roomAmount = roomAmount; return this; } - AnnouncementBuilder bathAmount(Integer bathAmount) { + public AnnouncementBuilder bathAmount(Integer bathAmount) { this.bathAmount = bathAmount; return this; } - AnnouncementBuilder parkingAvailability(String parkingAvailability) { + public AnnouncementBuilder parkingAvailability(String parkingAvailability) { this.parkingAvailability = parkingAvailability; return this; } - AnnouncementBuilder isSmokingAllowed(Boolean isSmokingAllowed) { + public AnnouncementBuilder isSmokingAllowed(Boolean isSmokingAllowed) { this.isSmokingAllowed = isSmokingAllowed; return this; } - AnnouncementBuilder isPetFriendly(Boolean isPetFriendly) { + public AnnouncementBuilder isPetFriendly(Boolean isPetFriendly) { this.isPetFriendly = isPetFriendly; return this; } - AnnouncementBuilder lessorName(String lessorName) { + public AnnouncementBuilder lessorName(String lessorName) { this.lessorName = lessorName; return this; } - AnnouncementBuilder additionalRentCost(BigDecimal additionalRentCost) { + public AnnouncementBuilder additionalRentCost(BigDecimal additionalRentCost) { this.additionalRentCost = additionalRentCost; return this; } - AnnouncementBuilder level(String level) { + public AnnouncementBuilder level(String level) { this.level = level; return this; } - AnnouncementBuilder furnishing(String furnishing) { + public AnnouncementBuilder furnishing(String furnishing) { this.furnishing = furnishing; return this; } - AnnouncementBuilder description(String description) { + public AnnouncementBuilder description(String description) { this.description = description; return this; } - AnnouncementBuilder provider(String provider) { + public AnnouncementBuilder provider(String provider) { this.provider = provider; return this; } - AnnouncementBuilder url(String url) { + public AnnouncementBuilder url(String url) { this.url = url; return this; } - AnnouncementBuilder phoneNumber(String phoneNumber) { + public AnnouncementBuilder phoneNumber(String phoneNumber) { this.phoneNumber = phoneNumber; return this; } - Announcement build() { + public Announcement build() { return new Announcement(this); } } From 81f64f5f50fd490870f9d1382953065614c52ea6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 18 Nov 2018 20:53:42 +0100 Subject: [PATCH 047/135] Title parse implementation. #3 --- .../java/parser/scrapper/GumtreeAnnouncementParser.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java index d9ebfaa..bb245f1 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java @@ -19,15 +19,17 @@ public Announcement parsePage(String url) { Document pageContent = getPageContent(url); log.info("Received content from url: " + url); - return null; + return Announcement.builder() + .title(parseTitle(pageContent.selectFirst(".item-title"))) + .build(); } catch (IOException e) { - throw new GumtreePageParseException("Nie można przetworzyć ogłoszenia o podanym url: " + url); + throw new GumtreePageParseException("Cannot parser announcement from url: " + url); } } @Override public String parseTitle(Element titleElement) { - return ""; + return titleElement.text(); } @Override From 42bc579d54dc9891005a797203a3c169f4d0dd97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 18 Nov 2018 21:01:45 +0100 Subject: [PATCH 048/135] Tests for price. #3 --- .../options/GumtreePriceOptionsTests.java | 60 +++++++++++++++++++ .../gumtree/options/price/exchange.html | 6 ++ .../gumtree/options/price/need-contact.html | 6 ++ .../gumtree/options/price/value.html | 6 ++ 4 files changed, 78 insertions(+) create mode 100644 announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreePriceOptionsTests.java create mode 100644 announcement-processor-parser/src/test/resources/gumtree/options/price/exchange.html create mode 100644 announcement-processor-parser/src/test/resources/gumtree/options/price/need-contact.html create mode 100644 announcement-processor-parser/src/test/resources/gumtree/options/price/value.html diff --git a/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreePriceOptionsTests.java b/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreePriceOptionsTests.java new file mode 100644 index 0000000..41045e8 --- /dev/null +++ b/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreePriceOptionsTests.java @@ -0,0 +1,60 @@ +package parser.scrapper.options; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import parser.ReaderUtil; +import parser.scrapper.GumtreeAnnouncementParser; + +import java.math.BigDecimal; + +public class GumtreePriceOptionsTests { + + private GumtreeAnnouncementParser gumtreeParser; + + @Before + public void setUp() { + gumtreeParser = new GumtreeAnnouncementParser(); + } + + @Test + public void testPriceExchange() { + // given + Document spyDoc = ReaderUtil.getDocumentToTest("/gumtree/options/price/exchange.html"); + Element priceElement = spyDoc.selectFirst(".price"); + + // when + BigDecimal price = gumtreeParser.parsePrice(priceElement); + + // then + Assert.assertEquals(BigDecimal.valueOf(-1), price); + } + + @Test + public void testPriceValue() { + // given + Document spyDoc = ReaderUtil.getDocumentToTest("/gumtree/options/price/value.html"); + Element priceElement = spyDoc.selectFirst(".price"); + + // when + BigDecimal price = gumtreeParser.parsePrice(priceElement); + + // then + Assert.assertEquals(BigDecimal.valueOf(1300), price); + } + + @Test + public void testPriceNeedContact() { + // given + Document spyDoc = ReaderUtil.getDocumentToTest("/gumtree/options/price/need-contact.html"); + Element priceElement = spyDoc.selectFirst(".price"); + + // when + BigDecimal price = gumtreeParser.parsePrice(priceElement); + + // then + Assert.assertNull(price); + } +} diff --git a/announcement-processor-parser/src/test/resources/gumtree/options/price/exchange.html b/announcement-processor-parser/src/test/resources/gumtree/options/price/exchange.html new file mode 100644 index 0000000..c190a53 --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/options/price/exchange.html @@ -0,0 +1,6 @@ +
+

Kawalerka 40m Głowackiego 4 Kraków PRZESTRONNE niskie koszty WIDOK na PANORAMĘ Krakowa POŁUDNIE

+
+ Wymiana/zamiana +
+
\ No newline at end of file diff --git a/announcement-processor-parser/src/test/resources/gumtree/options/price/need-contact.html b/announcement-processor-parser/src/test/resources/gumtree/options/price/need-contact.html new file mode 100644 index 0000000..c7e5a6b --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/options/price/need-contact.html @@ -0,0 +1,6 @@ +
+

Kawalerka 40m Głowackiego 4 Kraków PRZESTRONNE niskie koszty WIDOK na PANORAMĘ Krakowa POŁUDNIE

+
+ Proszę o kontakt +
+
\ No newline at end of file diff --git a/announcement-processor-parser/src/test/resources/gumtree/options/price/value.html b/announcement-processor-parser/src/test/resources/gumtree/options/price/value.html new file mode 100644 index 0000000..1ad916b --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/options/price/value.html @@ -0,0 +1,6 @@ +
+

Kawalerka 40m Głowackiego 4 Kraków PRZESTRONNE niskie koszty WIDOK na PANORAMĘ Krakowa POŁUDNIE

+
+ 1 300 zł +
+
\ No newline at end of file From a7d21acc33db46350b7d63a02c9b2003963d6581 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 18 Nov 2018 21:09:08 +0100 Subject: [PATCH 049/135] Price parse implementation. #3 --- .../scrapper/GumtreeAnnouncementParser.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java index bb245f1..e1f416f 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java @@ -21,6 +21,7 @@ public Announcement parsePage(String url) { return Announcement.builder() .title(parseTitle(pageContent.selectFirst(".item-title"))) + .price(parsePrice(pageContent.selectFirst(".price"))) .build(); } catch (IOException e) { throw new GumtreePageParseException("Cannot parser announcement from url: " + url); @@ -34,7 +35,20 @@ public String parseTitle(Element titleElement) { @Override public BigDecimal parsePrice(Element priceElement) { - return BigDecimal.ZERO; + BigDecimal output = BigDecimal.ZERO; + Element span = priceElement.selectFirst("span"); + String text = span.text(); + + if (text.contains("zł")) { + String number = text.replaceAll("zł", "").replaceAll(" ", ""); + output = new BigDecimal(number); + } else if (text.equals("Proszę o kontakt")) { + output = null; + } else if (text.equals("Wymiana/zamiana")) { + output = new BigDecimal(-1); + } + + return output; } @Override From c6a5e091f569343e33d65eb25209f1275cb2261a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 18 Nov 2018 21:13:29 +0100 Subject: [PATCH 050/135] Fixed typos. #3 --- .../java/parser/scrapper/GumtreeAnnouncementParser.java | 6 +++--- .../src/main/java/parser/scrapper/SinglePageParser.java | 6 +++--- .../scrapper/GumtreeParserNonRequiredFieldsTest.java | 8 ++++---- .../parser/scrapper/GumtreeParserRequiredFieldsTest.java | 2 +- .../scrapper/options/GumtreePetFriendlyOptionsTests.java | 4 ++-- .../scrapper/options/GumtreeSmokerOptionsTests.java | 4 ++-- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java index e1f416f..9fe7bbe 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java @@ -67,7 +67,7 @@ public String parseLessor(Element lessorElement) { } @Override - public String parserLessorName(Element lessorNameElement) { + public String parseLessorName(Element lessorNameElement) { return ""; } @@ -102,12 +102,12 @@ public String parseParking(Element parkingElement) { } @Override - public Boolean parserSmokers(Element smokersElement) { + public Boolean parseSmokers(Element smokersElement) { return false; } @Override - public Boolean parserPetFriendly(Element petFriendlyElement) { + public Boolean parsePetFriendly(Element petFriendlyElement) { return true; } diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/SinglePageParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/SinglePageParser.java index 67eb7ac..3fb47ee 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/SinglePageParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/SinglePageParser.java @@ -15,7 +15,7 @@ default String parseLessor(Element lessorElement) { return null; } - default String parserLessorName(Element lessorNameElement) { + default String parseLessorName(Element lessorNameElement) { return null; } @@ -43,11 +43,11 @@ default String parseParking(Element parkingElement){ return null; } - default Boolean parserSmokers(Element smokersElement){ + default Boolean parseSmokers(Element smokersElement){ return null; } - default Boolean parserPetFriendly(Element petFriendlyElement){ + default Boolean parsePetFriendly(Element petFriendlyElement){ return null; } diff --git a/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserNonRequiredFieldsTest.java b/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserNonRequiredFieldsTest.java index a22f080..8395309 100644 --- a/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserNonRequiredFieldsTest.java +++ b/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserNonRequiredFieldsTest.java @@ -206,7 +206,7 @@ public void testSmokersFieldExists() { Element detailsElement = spyDocWithoutLessor.selectFirst(".vip-details"); // when - Boolean smokers = gumtreeParser.parserSmokers(detailsElement); + Boolean smokers = gumtreeParser.parseSmokers(detailsElement); // then Assert.assertEquals(GumtreeExpectedProperties.expectedSmokers, smokers); @@ -219,7 +219,7 @@ public void testSmokersFieldNotExists() { Element detailsElement = spyDocWithoutLessor.selectFirst(".vip-details"); // when - Boolean smokers = gumtreeParser.parserSmokers(detailsElement); + Boolean smokers = gumtreeParser.parseSmokers(detailsElement); // then Assert.assertNull(smokers); @@ -232,7 +232,7 @@ public void testPetFriendlyFieldExists() { Element detailsElement = spyDocWithoutLessor.selectFirst(".vip-details"); // when - Boolean petFriendly = gumtreeParser.parserPetFriendly(detailsElement); + Boolean petFriendly = gumtreeParser.parsePetFriendly(detailsElement); // then Assert.assertEquals(GumtreeExpectedProperties.expectedPetFriendly, petFriendly); @@ -245,7 +245,7 @@ public void testPetFriendlyFieldNotExists() { Element detailsElement = spyDocWithoutLessor.selectFirst(".vip-details"); // when - Boolean petFriendly = gumtreeParser.parserPetFriendly(detailsElement); + Boolean petFriendly = gumtreeParser.parsePetFriendly(detailsElement); // then Assert.assertNull(petFriendly); diff --git a/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserRequiredFieldsTest.java b/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserRequiredFieldsTest.java index 863f1ff..bf8d577 100644 --- a/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserRequiredFieldsTest.java +++ b/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserRequiredFieldsTest.java @@ -72,7 +72,7 @@ public void testLessorName() { Element lessorNameElement = spyDoc.selectFirst(".username"); // when - String lessorName = gumtreeParser.parserLessorName(lessorNameElement); + String lessorName = gumtreeParser.parseLessorName(lessorNameElement); // then Assert.assertEquals(GumtreeExpectedProperties.expectedLessorName, lessorName); diff --git a/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreePetFriendlyOptionsTests.java b/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreePetFriendlyOptionsTests.java index 4a3c29d..33862bc 100644 --- a/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreePetFriendlyOptionsTests.java +++ b/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreePetFriendlyOptionsTests.java @@ -24,7 +24,7 @@ public void testPetFriendlyYes(){ Element detailsElement = spyDoc.selectFirst(".vip-details"); // when - Boolean petFriendly = gumtreeParser.parserPetFriendly(detailsElement); + Boolean petFriendly = gumtreeParser.parsePetFriendly(detailsElement); // then Assert.assertTrue(petFriendly); @@ -37,7 +37,7 @@ public void testPetFriendlyNo(){ Element detailsElement = spyDoc.selectFirst(".vip-details"); // when - Boolean petFriendly = gumtreeParser.parserPetFriendly(detailsElement); + Boolean petFriendly = gumtreeParser.parsePetFriendly(detailsElement); // then Assert.assertFalse(petFriendly); diff --git a/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreeSmokerOptionsTests.java b/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreeSmokerOptionsTests.java index 29f6e66..881c49d 100644 --- a/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreeSmokerOptionsTests.java +++ b/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreeSmokerOptionsTests.java @@ -24,7 +24,7 @@ public void testSmokersYes(){ Element detailsElement = spyDoc.selectFirst(".vip-details"); // when - Boolean smokers = gumtreeParser.parserSmokers(detailsElement); + Boolean smokers = gumtreeParser.parseSmokers(detailsElement); // then Assert.assertTrue(smokers); @@ -37,7 +37,7 @@ public void testSmokersNo(){ Element detailsElement = spyDoc.selectFirst(".vip-details"); // when - Boolean smokers = gumtreeParser.parserSmokers(detailsElement); + Boolean smokers = gumtreeParser.parseSmokers(detailsElement); // then Assert.assertFalse(smokers); From d5ca45000548217a9c3a3fe0764c64bceac6564b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 18 Nov 2018 21:25:18 +0100 Subject: [PATCH 051/135] Lessor name parse implementation. #3 --- .../parser/scrapper/GumtreeAnnouncementParser.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java index 9fe7bbe..5494f75 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java @@ -22,6 +22,7 @@ public Announcement parsePage(String url) { return Announcement.builder() .title(parseTitle(pageContent.selectFirst(".item-title"))) .price(parsePrice(pageContent.selectFirst(".price"))) + .lessorName(parseLessorName(pageContent.selectFirst(".username"))) .build(); } catch (IOException e) { throw new GumtreePageParseException("Cannot parser announcement from url: " + url); @@ -51,6 +52,13 @@ public BigDecimal parsePrice(Element priceElement) { return output; } + @Override + public String parseLessorName(Element lessorNameElement) { + Element href = lessorNameElement.selectFirst("a"); + String text = href.text(); + return text.replaceAll("\\(Zobacz więcej ogłoszeń\\)", "").trim(); + } + @Override public LocalDateTime parseCreationDate(Element creationDateElement) { return LocalDateTime.MIN; @@ -66,11 +74,6 @@ public String parseLessor(Element lessorElement) { return ""; } - @Override - public String parseLessorName(Element lessorNameElement) { - return ""; - } - @Override public String parsePhoneNumber(Element phoneNumberElement) { return ""; From df38577b52ea25f8c6512525db4bc9280dfac12f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 18 Nov 2018 21:44:03 +0100 Subject: [PATCH 052/135] Creation date parse implementation. #3 --- .../scrapper/GumtreeAnnouncementParser.java | 30 ++++++++++++++++++- .../GumtreeParserRequiredFieldsTest.java | 2 +- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java index 5494f75..3aa950d 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java @@ -3,13 +3,17 @@ import lombok.extern.slf4j.Slf4j; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; import parser.Announcement; import parser.exceptions.GumtreePageParseException; import parser.exceptions.PropertyNotValidForGumtreeProviderException; import java.io.IOException; import java.math.BigDecimal; +import java.time.LocalDate; import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; @Slf4j public class GumtreeAnnouncementParser extends AnnouncementParser implements SinglePageParser { @@ -19,10 +23,13 @@ public Announcement parsePage(String url) { Document pageContent = getPageContent(url); log.info("Received content from url: " + url); + Element details = pageContent.selectFirst(".vip-details"); + return Announcement.builder() .title(parseTitle(pageContent.selectFirst(".item-title"))) .price(parsePrice(pageContent.selectFirst(".price"))) .lessorName(parseLessorName(pageContent.selectFirst(".username"))) + .creationDate(parseCreationDate(details)) .build(); } catch (IOException e) { throw new GumtreePageParseException("Cannot parser announcement from url: " + url); @@ -61,7 +68,16 @@ public String parseLessorName(Element lessorNameElement) { @Override public LocalDateTime parseCreationDate(Element creationDateElement) { - return LocalDateTime.MIN; + LocalDateTime output = null; + + Elements liElements = creationDateElement.select("li"); + String dateInText = getValueForAttributeFromLiElements("Data dodania", liElements); + + if (dateInText != null) { + output = LocalDateTime.of(LocalDate.parse(dateInText, DateTimeFormatter.ofPattern("d/MM/yyyy")), LocalTime.of(0, 0)); + } + + return output; } @Override @@ -128,4 +144,16 @@ public String parseLevel(Element levelElement) { public String parseFurnishings(Element furnishingsElement) { throw new PropertyNotValidForGumtreeProviderException("furnishings"); } + + private String getValueForAttributeFromLiElements(String attributeName, Elements liElements) { + if (liElements != null) { + for (Element singleLiElement : liElements) { + String text = singleLiElement.selectFirst(".attribute > .name").text(); + if (text.equals(attributeName)) { + return singleLiElement.selectFirst(".attribute > .value").text(); + } + } + } + return null; + } } diff --git a/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserRequiredFieldsTest.java b/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserRequiredFieldsTest.java index bf8d577..150f52e 100644 --- a/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserRequiredFieldsTest.java +++ b/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserRequiredFieldsTest.java @@ -82,7 +82,7 @@ public void testLessorName() { @Test public void testCreationDate() { // given - Element detailsElement = spyDoc.selectFirst(".vip.vip-contact"); + Element detailsElement = spyDoc.selectFirst(".vip-details"); // when LocalDateTime dateTime = gumtreeParser.parseCreationDate(detailsElement); From f05343f82a3ab5fd74633fdd80eb54f991f31270 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 18 Nov 2018 22:37:58 +0100 Subject: [PATCH 053/135] Lessor tests and parse implementation. #3 --- .../scrapper/GumtreeAnnouncementParser.java | 5 ++- .../options/GumtreeLessorOptionsTests.java | 45 +++++++++++++++++++ .../gumtree/options/lessor/agency.html | 10 +++++ .../gumtree/options/lessor/owner.html | 10 +++++ 4 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreeLessorOptionsTests.java create mode 100644 announcement-processor-parser/src/test/resources/gumtree/options/lessor/agency.html create mode 100644 announcement-processor-parser/src/test/resources/gumtree/options/lessor/owner.html diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java index 3aa950d..f97a80e 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java @@ -30,6 +30,8 @@ public Announcement parsePage(String url) { .price(parsePrice(pageContent.selectFirst(".price"))) .lessorName(parseLessorName(pageContent.selectFirst(".username"))) .creationDate(parseCreationDate(details)) +// .description(parseDescription(pageContent.selectFirst(".description"))) + .lessor(parseLessor(details)) .build(); } catch (IOException e) { throw new GumtreePageParseException("Cannot parser announcement from url: " + url); @@ -87,7 +89,8 @@ public String parseDescription(Element element) { @Override public String parseLessor(Element lessorElement) { - return ""; + Elements liElements = lessorElement.select("li"); + return getValueForAttributeFromLiElements("Do wynajęcia przez", liElements); } @Override diff --git a/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreeLessorOptionsTests.java b/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreeLessorOptionsTests.java new file mode 100644 index 0000000..48d4a65 --- /dev/null +++ b/announcement-processor-parser/src/test/java/parser/scrapper/options/GumtreeLessorOptionsTests.java @@ -0,0 +1,45 @@ +package parser.scrapper.options; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import parser.ReaderUtil; +import parser.scrapper.GumtreeAnnouncementParser; + +public class GumtreeLessorOptionsTests { + + private GumtreeAnnouncementParser gumtreeParser; + + @Before + public void setUp() { + gumtreeParser = new GumtreeAnnouncementParser(); + } + + @Test + public void testLessorOwner() { + // given + Document spyDoc = ReaderUtil.getDocumentToTest("/gumtree/options/lessor/owner.html"); + Element detailsElement = spyDoc.selectFirst(".vip-details"); + + // when + String lessor = gumtreeParser.parseLessor(detailsElement); + + // then + Assert.assertEquals("Właściciel", lessor); + } + + @Test + public void testLessorAgency() { + // given + Document spyDoc = ReaderUtil.getDocumentToTest("/gumtree/options/lessor/agency.html"); + Element detailsElement = spyDoc.selectFirst(".vip-details"); + + // when + String lessor = gumtreeParser.parseLessor(detailsElement); + + // then + Assert.assertEquals("Agencja", lessor); + } +} diff --git a/announcement-processor-parser/src/test/resources/gumtree/options/lessor/agency.html b/announcement-processor-parser/src/test/resources/gumtree/options/lessor/agency.html new file mode 100644 index 0000000..c344bec --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/options/lessor/agency.html @@ -0,0 +1,10 @@ +
+
    +
  • +
    + Do wynajęcia przez + Agencja +
    +
    • +
+
\ No newline at end of file diff --git a/announcement-processor-parser/src/test/resources/gumtree/options/lessor/owner.html b/announcement-processor-parser/src/test/resources/gumtree/options/lessor/owner.html new file mode 100644 index 0000000..0a4c970 --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/options/lessor/owner.html @@ -0,0 +1,10 @@ +
+
    +
  • +
    + Do wynajęcia przez + Właściciel +
    +
    • +
+
\ No newline at end of file From c3fc484a905396d74263726bb33e2d4eb46107e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 18 Nov 2018 22:43:46 +0100 Subject: [PATCH 054/135] Phone number parse implementation. #3 --- .../java/parser/scrapper/GumtreeAnnouncementParser.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java index f97a80e..aaf2a5a 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java @@ -32,6 +32,7 @@ public Announcement parsePage(String url) { .creationDate(parseCreationDate(details)) // .description(parseDescription(pageContent.selectFirst(".description"))) .lessor(parseLessor(details)) + .phoneNumber(parsePhoneNumber(pageContent.selectFirst(".vip.vip-contact"))) .build(); } catch (IOException e) { throw new GumtreePageParseException("Cannot parser announcement from url: " + url); @@ -95,7 +96,12 @@ public String parseLessor(Element lessorElement) { @Override public String parsePhoneNumber(Element phoneNumberElement) { - return ""; + Element element = phoneNumberElement.selectFirst("div > a"); + String telephone = element.attr("href"); + if(telephone.contains("tel:")){ + return telephone.replaceAll("tel:", ""); + } + return null; } @Override From c77d5466d516f6e1aee8384fdbf5008bcaecc4ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 18 Nov 2018 22:44:54 +0100 Subject: [PATCH 055/135] Property type parse implementation. #3 --- .../main/java/parser/scrapper/GumtreeAnnouncementParser.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java index aaf2a5a..c3c84b7 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java @@ -33,6 +33,7 @@ public Announcement parsePage(String url) { // .description(parseDescription(pageContent.selectFirst(".description"))) .lessor(parseLessor(details)) .phoneNumber(parsePhoneNumber(pageContent.selectFirst(".vip.vip-contact"))) + .propertyType(parsePropertyType(details)) .build(); } catch (IOException e) { throw new GumtreePageParseException("Cannot parser announcement from url: " + url); @@ -106,7 +107,8 @@ public String parsePhoneNumber(Element phoneNumberElement) { @Override public String parsePropertyType(Element propertyTypeElement) { - return ""; + Elements liElements = propertyTypeElement.select("li"); + return getValueForAttributeFromLiElements("Rodzaj nieruchomości", liElements); } @Override From 8306802300a9d8ce312be2615f9f93c175583816 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 18 Nov 2018 22:48:01 +0100 Subject: [PATCH 056/135] Flat area parse implementation. #3 --- .../parser/scrapper/GumtreeAnnouncementParser.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java index c3c84b7..5482f32 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java @@ -34,6 +34,7 @@ public Announcement parsePage(String url) { .lessor(parseLessor(details)) .phoneNumber(parsePhoneNumber(pageContent.selectFirst(".vip.vip-contact"))) .propertyType(parsePropertyType(details)) + .flatArea(parseFlatArea(details)) .build(); } catch (IOException e) { throw new GumtreePageParseException("Cannot parser announcement from url: " + url); @@ -113,7 +114,14 @@ public String parsePropertyType(Element propertyTypeElement) { @Override public Double parseFlatArea(Element flatAreaElement) { - return 0d; + Elements liElements = flatAreaElement.select("li"); + String flatAreaInText = getValueForAttributeFromLiElements("Wielkość (m2)", liElements); + + if(flatAreaInText != null){ + return Double.valueOf(flatAreaInText); + } + + return null; } @Override From 9bab146a5ec72302f6d34809bc58fda1b4136a01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 18 Nov 2018 22:54:59 +0100 Subject: [PATCH 057/135] Room amount parse implementation. #3 --- .../scrapper/GumtreeAnnouncementParser.java | 35 +++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java index 5482f32..2ab4cd1 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java @@ -35,6 +35,7 @@ public Announcement parsePage(String url) { .phoneNumber(parsePhoneNumber(pageContent.selectFirst(".vip.vip-contact"))) .propertyType(parsePropertyType(details)) .flatArea(parseFlatArea(details)) + .roomAmount(parseRoomAmount(details)) .build(); } catch (IOException e) { throw new GumtreePageParseException("Cannot parser announcement from url: " + url); @@ -100,7 +101,7 @@ public String parseLessor(Element lessorElement) { public String parsePhoneNumber(Element phoneNumberElement) { Element element = phoneNumberElement.selectFirst("div > a"); String telephone = element.attr("href"); - if(telephone.contains("tel:")){ + if (telephone.contains("tel:")) { return telephone.replaceAll("tel:", ""); } return null; @@ -117,7 +118,7 @@ public Double parseFlatArea(Element flatAreaElement) { Elements liElements = flatAreaElement.select("li"); String flatAreaInText = getValueForAttributeFromLiElements("Wielkość (m2)", liElements); - if(flatAreaInText != null){ + if (flatAreaInText != null) { return Double.valueOf(flatAreaInText); } @@ -126,7 +127,35 @@ public Double parseFlatArea(Element flatAreaElement) { @Override public Integer parseRoomAmount(Element roomAmountElement) { - return -1; + Elements liElements = roomAmountElement.select("li"); + String roomAmountText = getValueForAttributeFromLiElements("Liczba pokoi", liElements); + + Integer roomAmount = null; + + if (roomAmountText != null) { + switch (roomAmountText) { + case "Kawalerka lub garsoniera": + roomAmount = 1; + break; + case "2 pokoje": + roomAmount = 2; + break; + case "3 pokoje": + roomAmount = 3; + break; + case "4 pokoje": + roomAmount = 4; + break; + case "5 pokoi": + roomAmount = 5; + break; + case "6 lub więcej pokoi": + roomAmount = 6; + break; + } + } + + return roomAmount; } @Override From 1ebf10169936c639cc3c4e8bb2e0d24b342576d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 18 Nov 2018 22:57:11 +0100 Subject: [PATCH 058/135] Bath amount parse implementation. #3 --- .../scrapper/GumtreeAnnouncementParser.java | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java index 2ab4cd1..dc87062 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java @@ -36,6 +36,7 @@ public Announcement parsePage(String url) { .propertyType(parsePropertyType(details)) .flatArea(parseFlatArea(details)) .roomAmount(parseRoomAmount(details)) + .bathAmount(parseBathAmount(details)) .build(); } catch (IOException e) { throw new GumtreePageParseException("Cannot parser announcement from url: " + url); @@ -160,7 +161,29 @@ public Integer parseRoomAmount(Element roomAmountElement) { @Override public Integer parseBathAmount(Element bathAmountElement) { - return -1; + Elements liElements = bathAmountElement.select("li"); + String bathAmountText = getValueForAttributeFromLiElements("Liczba łazienek", liElements); + + Integer bathAmount = null; + + if (bathAmountText != null) { + switch (bathAmountText) { + case "1 łazienka": + bathAmount = 1; + break; + case "2 łazienki": + bathAmount = 2; + break; + case "3 łazienki": + bathAmount = 3; + break; + case "4 lub więcej łazienek": + bathAmount = 4; + break; + } + } + + return bathAmount; } @Override From 4067f4e3faeadeff9e9a07d26199871a4e97660a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 18 Nov 2018 22:59:18 +0100 Subject: [PATCH 059/135] Parking parse implementation. #3 --- .../main/java/parser/scrapper/GumtreeAnnouncementParser.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java index dc87062..2092c91 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java @@ -37,6 +37,7 @@ public Announcement parsePage(String url) { .flatArea(parseFlatArea(details)) .roomAmount(parseRoomAmount(details)) .bathAmount(parseBathAmount(details)) + .parkingAvailability(parseParking(details)) .build(); } catch (IOException e) { throw new GumtreePageParseException("Cannot parser announcement from url: " + url); @@ -188,7 +189,8 @@ public Integer parseBathAmount(Element bathAmountElement) { @Override public String parseParking(Element parkingElement) { - return ""; + Elements liElements = parkingElement.select("li"); + return getValueForAttributeFromLiElements("Parking", liElements); } @Override From 6cee21e6ffd7c29290ce8816a03db70fae52a964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 18 Nov 2018 23:04:21 +0100 Subject: [PATCH 060/135] Smokers parse implementation. #3 --- .../scrapper/GumtreeAnnouncementParser.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java index 2092c91..216f05c 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java @@ -38,6 +38,7 @@ public Announcement parsePage(String url) { .roomAmount(parseRoomAmount(details)) .bathAmount(parseBathAmount(details)) .parkingAvailability(parseParking(details)) + .isSmokingAllowed(parseSmokers(details)) .build(); } catch (IOException e) { throw new GumtreePageParseException("Cannot parser announcement from url: " + url); @@ -195,7 +196,20 @@ public String parseParking(Element parkingElement) { @Override public Boolean parseSmokers(Element smokersElement) { - return false; + Elements liElements = smokersElement.select("li"); + String smokers = getValueForAttributeFromLiElements("Palący", liElements); + + Boolean forSmokers = null; + + if(smokers != null){ + if (smokers.equals("Tak")) { + forSmokers = true; + } else if (smokers.equals("Nie")) { + forSmokers = false; + } + } + + return forSmokers; } @Override From ea5d5d6a3910101d6b29cde6c269bfbe060609e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 18 Nov 2018 23:08:32 +0100 Subject: [PATCH 061/135] Pet friendly parse implementation. #3 --- .../scrapper/GumtreeAnnouncementParser.java | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java index 216f05c..8d8ad74 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java @@ -39,6 +39,7 @@ public Announcement parsePage(String url) { .bathAmount(parseBathAmount(details)) .parkingAvailability(parseParking(details)) .isSmokingAllowed(parseSmokers(details)) + .isPetFriendly(parsePetFriendly(details)) .build(); } catch (IOException e) { throw new GumtreePageParseException("Cannot parser announcement from url: " + url); @@ -199,22 +200,15 @@ public Boolean parseSmokers(Element smokersElement) { Elements liElements = smokersElement.select("li"); String smokers = getValueForAttributeFromLiElements("Palący", liElements); - Boolean forSmokers = null; - - if(smokers != null){ - if (smokers.equals("Tak")) { - forSmokers = true; - } else if (smokers.equals("Nie")) { - forSmokers = false; - } - } - - return forSmokers; + return getBooleanFromYesNoSentence(smokers); } @Override public Boolean parsePetFriendly(Element petFriendlyElement) { - return true; + Elements liElements = petFriendlyElement.select("li"); + String pets = getValueForAttributeFromLiElements("Przyjazne zwierzakom", liElements); + + return getBooleanFromYesNoSentence(pets); } @Override @@ -243,4 +237,15 @@ private String getValueForAttributeFromLiElements(String attributeName, Elements } return null; } + + private Boolean getBooleanFromYesNoSentence(String sentence) { + if (sentence != null) { + if (sentence.equals("Tak")) { + return true; + } else if (sentence.equals("Nie")) { + return false; + } + } + return null; + } } From 038f04d51cf53ae89c5a2e13d09625443192dbe7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Mon, 19 Nov 2018 00:06:20 +0100 Subject: [PATCH 062/135] Description parse implementation. #3 --- .../scrapper/GumtreeAnnouncementParser.java | 19 ++++++++++++++++++- .../scrapper/GumtreeExpectedProperties.java | 8 +++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java index 8d8ad74..de9bfdf 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java @@ -1,6 +1,7 @@ package parser.scrapper; import lombok.extern.slf4j.Slf4j; +import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; @@ -92,7 +93,23 @@ public LocalDateTime parseCreationDate(Element creationDateElement) { @Override public String parseDescription(Element element) { - return ""; + Element span = element.selectFirst("span"); + String textWithoutBoldings = span.html().replaceAll("|", ""); + Document parse = Jsoup.parse(textWithoutBoldings); + + Elements allElements = parse.body().getAllElements(); + StringBuilder sb = new StringBuilder(); + for (Element elem : allElements) { + String ownText = elem.ownText(); + if (!ownText.equals("")) { + sb.append(elem.ownText()).append("\n"); + } else { + if (elem.html().equals("
")) { + sb.append("\n"); + } + } + } + return sb.toString().trim(); } @Override diff --git a/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeExpectedProperties.java b/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeExpectedProperties.java index 3df5cea..1ea85b3 100644 --- a/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeExpectedProperties.java +++ b/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeExpectedProperties.java @@ -47,5 +47,11 @@ class GumtreeExpectedProperties { "\n" + "nioobi (malpa) op (kropka) pl\n" + "\n" + - "Pośrednikom z góry dziękuję za zainteresowanie!\n"; + "\n" + + "Możliwość oglądnięcia mieszkania\n" + + "\n" + + "Palić papierosy można na dużym balkonie ;)\n" + + "\n" + + "\n" + + "Pośrednikom z góry dziękuję za zainteresowanie!"; } From 07e49ed279f0fe3ddf232bdbd9191ca2f5a67a5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Mon, 19 Nov 2018 23:11:00 +0100 Subject: [PATCH 063/135] Build description in announcement. #3 --- .../main/java/parser/scrapper/GumtreeAnnouncementParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java index de9bfdf..e55e488 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java @@ -31,7 +31,7 @@ public Announcement parsePage(String url) { .price(parsePrice(pageContent.selectFirst(".price"))) .lessorName(parseLessorName(pageContent.selectFirst(".username"))) .creationDate(parseCreationDate(details)) -// .description(parseDescription(pageContent.selectFirst(".description"))) + .description(parseDescription(pageContent.selectFirst(".description"))) .lessor(parseLessor(details)) .phoneNumber(parsePhoneNumber(pageContent.selectFirst(".vip.vip-contact"))) .propertyType(parsePropertyType(details)) From 517de1c4f4c965e6d2c5ddd5ec5f3728522006de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Wed, 21 Nov 2018 22:18:55 +0100 Subject: [PATCH 064/135] Added simple Announcement in JSON format (generated like from parser) --- .../resources/announcement-from-parser.json | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 announcement-processor-extractor/src/test/resources/announcement-from-parser.json diff --git a/announcement-processor-extractor/src/test/resources/announcement-from-parser.json b/announcement-processor-extractor/src/test/resources/announcement-from-parser.json new file mode 100644 index 0000000..7512e17 --- /dev/null +++ b/announcement-processor-extractor/src/test/resources/announcement-from-parser.json @@ -0,0 +1,21 @@ +{ + "title":"Kawalerka 40m Głowackiego 4 Kraków PRZESTRONNE niskie koszty WIDOK na PANORAMĘ Krakowa POŁUDNIE", + "price":1300, + "creationDate":"2018-11-17T00:00:00", + "lessor":"Właściciel", + "propertyType":"Mieszkanie", + "flatArea":40.0, + "roomAmount":1, + "bathAmount":1, + "parkingAvailability":"Ulica", + "isSmokingAllowed":true, + "isPetFriendly":false, + "lessorName":"Bartek", + "additionalRentCost":null, + "level":null, + "furnishing":null, + "description":"Proponuję mieszkanie na wynajem (kawalerka) o powierzchni 40m2 zlokalizowane przy ulicy Głowackiego 4 w Krakowie. Dzielnica Krowodrza. Rejon Bronowice.\n\nKawalerka mieście się na 5 piętrze od strony południowej z widokiem na panoramę Krakowa.\n3 minuty pieszo do przystanków tramwajowych. Bliskie sąsiedztwo Uniwersytetu Pedagogicznego, AGH, Wydziału Fizyki i Architektury PK.\n\nBudynek oddany do użytkowania w 2008r. Podłoga z litego drewna, w aneksie kuchennym i w łazience płytki ceramiczne. Mieszkanie wyposażone tak jak na zdjęciach: (lodówka, płyta grzewcza, piekarnik, pralka, stół, krzesła, łóżko 2os., łóżko 1os.)\nMożliwość usunięcia mebli wg upodobań wynajmującego jak i ich dokupienia.\n\nNieruchomość posiada przestronny hol i windę. Na parterze zlokalizowane są sklepy m. in. Lewiatan - można w pantoflach robić zakupy ;)\n\nKoszty najmu (miesięczne): wynajem 1500zł, + czynsz administracyjny 250zł, + media wg zużycia (woda (~60zł/1os), prąd (~60zł/1os) , ogrzewanie, internet (60zł)).\n\nSZACOWANY CAŁKOWITY KOSZT wynajmu i eksploatacji miesięcznej: 1900-2000zł (2 osoby)\n\n2K PLN per month ;)\nZalety to niskie koszty ogrzewania, w zasadzie pomijalne.\n\nINTERNET od UPC. Mieszkanie wyposażone w gniazdo RTV i SAT.\n\nUmowa min. na 9 miesięcy. W razie pytań proszę się nie krępować :)\n\nKontak mailowy lub telefoniczny (w rozsądnych godzinach):\n\n6 0 2 3 2 7 7 0 6\n\nnioobi (malpa) op (kropka) pl\n\n\nMożliwość oglądnięcia mieszkania\n\nPalić papierosy można na dużym balkonie ;)\n\n\nPośrednikom z góry dziękuję za zainteresowanie!", + "provider":"GUMTREE", + "url":"https://www.gumtree.pl/a-mieszkania-i-domy-do-wynajecia/krakow/kawalerka-40m-glowackiego-4-krakow-przestronne-niskie-koszty-widok-na-panorame-krakowa-poludnie/1003555014070910781350209", + "phoneNumber":null +} \ No newline at end of file From 43c25be8d7f0b3980ed5924f7c8df8250cc2e3ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Wed, 28 Nov 2018 11:47:23 +0100 Subject: [PATCH 065/135] Added url and provider when parsing page by Gumtree provider. #34 --- .../main/java/parser/scrapper/GumtreeAnnouncementParser.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java index e55e488..cb1b6c2 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java @@ -41,6 +41,8 @@ public Announcement parsePage(String url) { .parkingAvailability(parseParking(details)) .isSmokingAllowed(parseSmokers(details)) .isPetFriendly(parsePetFriendly(details)) + .url(url) + .provider("Gumtree") .build(); } catch (IOException e) { throw new GumtreePageParseException("Cannot parser announcement from url: " + url); From 454a5e5407bf1df67eda60514fba8983dae9770a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Wed, 28 Nov 2018 11:49:06 +0100 Subject: [PATCH 066/135] Created dev config for parser including database config setup. #34 --- announcement-processor-parser/pom.xml | 27 ++++++++++++++++--- .../main/resources/application-dev.properties | 13 +++++++++ .../src/main/resources/application.properties | 1 + 3 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 announcement-processor-parser/src/main/resources/application-dev.properties diff --git a/announcement-processor-parser/pom.xml b/announcement-processor-parser/pom.xml index e603e00..0d1b3dc 100644 --- a/announcement-processor-parser/pom.xml +++ b/announcement-processor-parser/pom.xml @@ -25,12 +25,12 @@ org.springframework.boot - spring-boot-starter + spring-boot-starter-web org.springframework.boot - spring-boot-starter-test - test + spring-boot-devtools + runtime @@ -48,6 +48,11 @@ + + org.springframework.boot + spring-boot-starter-test + test + org.mockito mockito-all @@ -66,6 +71,22 @@ jsoup 1.11.3 + + + + org.springframework.boot + spring-boot-starter-data-jpa + 2.1.0.RELEASE + + + mysql + mysql-connector-java + + + com.h2database + h2 + 1.4.197 + diff --git a/announcement-processor-parser/src/main/resources/application-dev.properties b/announcement-processor-parser/src/main/resources/application-dev.properties new file mode 100644 index 0000000..bc0481d --- /dev/null +++ b/announcement-processor-parser/src/main/resources/application-dev.properties @@ -0,0 +1,13 @@ +# H2 +spring.h2.console.enabled=true +spring.h2.console.path=/h2 + +# Datasource +spring.datasource.url=jdbc:h2:mem:parser +spring.datasource.username=sa +spring.datasource.password= +spring.datasource.driver-class-name=org.h2.Driver + +spring.jpa.hibernate.ddl-auto=create-drop + +spring.h2.console.settings.web-allow-others=true \ No newline at end of file diff --git a/announcement-processor-parser/src/main/resources/application.properties b/announcement-processor-parser/src/main/resources/application.properties index b988063..94a9b6c 100644 --- a/announcement-processor-parser/src/main/resources/application.properties +++ b/announcement-processor-parser/src/main/resources/application.properties @@ -1 +1,2 @@ +spring.profiles.default=dev server.port = 8081 \ No newline at end of file From c0b941fe8e81b3631f6c35706afb40b848ed11d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Wed, 28 Nov 2018 13:03:09 +0100 Subject: [PATCH 067/135] ParsingInfoRegistry table mapping. #34 --- .../java/parser/registry/ParsingInfo.java | 30 +++++++++++++++++++ .../registry/ParsingInfoRepository.java | 8 +++++ .../parser/registry/ParsingInfoService.java | 14 +++++++++ 3 files changed, 52 insertions(+) create mode 100644 announcement-processor-parser/src/main/java/parser/registry/ParsingInfo.java create mode 100644 announcement-processor-parser/src/main/java/parser/registry/ParsingInfoRepository.java create mode 100644 announcement-processor-parser/src/main/java/parser/registry/ParsingInfoService.java diff --git a/announcement-processor-parser/src/main/java/parser/registry/ParsingInfo.java b/announcement-processor-parser/src/main/java/parser/registry/ParsingInfo.java new file mode 100644 index 0000000..43edf8b --- /dev/null +++ b/announcement-processor-parser/src/main/java/parser/registry/ParsingInfo.java @@ -0,0 +1,30 @@ +package parser.registry; + +import lombok.Getter; +import lombok.Setter; + +import javax.persistence.*; +import java.time.LocalDate; + +@Getter +@Setter +@Entity +@Table(name = "PARSED_INFO_REGISTRY") +public class ParsingInfo { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "ID") + Integer id; + + @Column(name = "URL") + String url; + + @Column(name = "PAGE_HASH") + String pageHash; + + @Column(name = "DATE") + LocalDate date; + + @Column(name = "COUNTER_PER_DATE") + Integer counterPerDate; +} diff --git a/announcement-processor-parser/src/main/java/parser/registry/ParsingInfoRepository.java b/announcement-processor-parser/src/main/java/parser/registry/ParsingInfoRepository.java new file mode 100644 index 0000000..69d4365 --- /dev/null +++ b/announcement-processor-parser/src/main/java/parser/registry/ParsingInfoRepository.java @@ -0,0 +1,8 @@ +package parser.registry; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface ParsingInfoRepository extends JpaRepository { +} diff --git a/announcement-processor-parser/src/main/java/parser/registry/ParsingInfoService.java b/announcement-processor-parser/src/main/java/parser/registry/ParsingInfoService.java new file mode 100644 index 0000000..a4734da --- /dev/null +++ b/announcement-processor-parser/src/main/java/parser/registry/ParsingInfoService.java @@ -0,0 +1,14 @@ +package parser.registry; + +import org.springframework.stereotype.Service; + +@Service +public class ParsingInfoService { + + private ParsingInfoRepository parsingInfoRepository; + + public ParsingInfoService(ParsingInfoRepository parsingInfoRepository) { + this.parsingInfoRepository = parsingInfoRepository; + } + +} From 4d7b1d6190c80a4d838108530709e05790376348 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Wed, 28 Nov 2018 19:17:43 +0100 Subject: [PATCH 068/135] Added provider as a column. #34 --- .../src/main/java/parser/registry/ParsingInfo.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/announcement-processor-parser/src/main/java/parser/registry/ParsingInfo.java b/announcement-processor-parser/src/main/java/parser/registry/ParsingInfo.java index 43edf8b..04ee68c 100644 --- a/announcement-processor-parser/src/main/java/parser/registry/ParsingInfo.java +++ b/announcement-processor-parser/src/main/java/parser/registry/ParsingInfo.java @@ -27,4 +27,7 @@ public class ParsingInfo { @Column(name = "COUNTER_PER_DATE") Integer counterPerDate; + + @Column(name = "PROVIDER") + String provider; } From 80eb0bc51d8a2ab1fac2c4b12625a3d4e45751fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Thu, 29 Nov 2018 22:17:56 +0100 Subject: [PATCH 069/135] Added walker files structure. #34 --- .../parser/registry/ParsingInfoService.java | 1 - .../java/parser/walker/GumtreePageWalker.java | 19 ++++++++++++++++++ .../main/java/parser/walker/PageWalker.java | 5 +++++ .../parser/walker/helpers/GumtreeHelper.java | 13 ++++++++++++ .../parser/walker/helpers/ProviderHelper.java | 14 +++++++++++++ .../walker/states/FetchRegistryState.java | 20 +++++++++++++++++++ .../parser/walker/states/FindUrlState.java | 20 +++++++++++++++++++ .../parser/walker/states/NewerPageState.java | 20 +++++++++++++++++++ .../walker/states/ProcessPageState.java | 20 +++++++++++++++++++ .../java/parser/walker/states/StopState.java | 20 +++++++++++++++++++ .../parser/walker/states/TopElementState.java | 20 +++++++++++++++++++ .../states/VisibleAfterReloadState.java | 20 +++++++++++++++++++ .../parser/walker/states/WalkerState.java | 15 ++++++++++++++ 13 files changed, 206 insertions(+), 1 deletion(-) create mode 100644 announcement-processor-parser/src/main/java/parser/walker/GumtreePageWalker.java create mode 100644 announcement-processor-parser/src/main/java/parser/walker/PageWalker.java create mode 100644 announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java create mode 100644 announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java create mode 100644 announcement-processor-parser/src/main/java/parser/walker/states/FetchRegistryState.java create mode 100644 announcement-processor-parser/src/main/java/parser/walker/states/FindUrlState.java create mode 100644 announcement-processor-parser/src/main/java/parser/walker/states/NewerPageState.java create mode 100644 announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java create mode 100644 announcement-processor-parser/src/main/java/parser/walker/states/StopState.java create mode 100644 announcement-processor-parser/src/main/java/parser/walker/states/TopElementState.java create mode 100644 announcement-processor-parser/src/main/java/parser/walker/states/VisibleAfterReloadState.java create mode 100644 announcement-processor-parser/src/main/java/parser/walker/states/WalkerState.java diff --git a/announcement-processor-parser/src/main/java/parser/registry/ParsingInfoService.java b/announcement-processor-parser/src/main/java/parser/registry/ParsingInfoService.java index a4734da..2a91319 100644 --- a/announcement-processor-parser/src/main/java/parser/registry/ParsingInfoService.java +++ b/announcement-processor-parser/src/main/java/parser/registry/ParsingInfoService.java @@ -10,5 +10,4 @@ public class ParsingInfoService { public ParsingInfoService(ParsingInfoRepository parsingInfoRepository) { this.parsingInfoRepository = parsingInfoRepository; } - } diff --git a/announcement-processor-parser/src/main/java/parser/walker/GumtreePageWalker.java b/announcement-processor-parser/src/main/java/parser/walker/GumtreePageWalker.java new file mode 100644 index 0000000..052f12c --- /dev/null +++ b/announcement-processor-parser/src/main/java/parser/walker/GumtreePageWalker.java @@ -0,0 +1,19 @@ +package parser.walker; + +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; +import parser.walker.states.WalkerState; + +@Service +public class GumtreePageWalker extends PageWalker { + + private WalkerState walkerState; + + public GumtreePageWalker(@Qualifier("fetchRegistryState") WalkerState walkerState) { + this.walkerState = walkerState; + } + + @Override + public void walk(String startPageUrl) { + } +} diff --git a/announcement-processor-parser/src/main/java/parser/walker/PageWalker.java b/announcement-processor-parser/src/main/java/parser/walker/PageWalker.java new file mode 100644 index 0000000..9c95742 --- /dev/null +++ b/announcement-processor-parser/src/main/java/parser/walker/PageWalker.java @@ -0,0 +1,5 @@ +package parser.walker; + +public abstract class PageWalker { + public abstract void walk(String startPageUrl); +} diff --git a/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java b/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java new file mode 100644 index 0000000..68a2626 --- /dev/null +++ b/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java @@ -0,0 +1,13 @@ +package parser.walker.helpers; + +import org.springframework.stereotype.Component; +import parser.registry.ParsingInfoService; + +@Component +public class GumtreeHelper extends ProviderHelper { + + public GumtreeHelper(ParsingInfoService parsingInfoService) { + super(parsingInfoService); + } + +} diff --git a/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java b/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java new file mode 100644 index 0000000..0565cec --- /dev/null +++ b/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java @@ -0,0 +1,14 @@ +package parser.walker.helpers; + +import org.springframework.stereotype.Component; +import parser.registry.ParsingInfoService; + +@Component +public abstract class ProviderHelper { + ParsingInfoService parsingInfoService; + + public ProviderHelper(ParsingInfoService parsingInfoService) { + this.parsingInfoService = parsingInfoService; + } + +} diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/FetchRegistryState.java b/announcement-processor-parser/src/main/java/parser/walker/states/FetchRegistryState.java new file mode 100644 index 0000000..7281375 --- /dev/null +++ b/announcement-processor-parser/src/main/java/parser/walker/states/FetchRegistryState.java @@ -0,0 +1,20 @@ +package parser.walker.states; + +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; +import parser.walker.helpers.ProviderHelper; + +@Component +@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class FetchRegistryState extends WalkerState { + + public FetchRegistryState(ProviderHelper providerHelper) { + super(providerHelper); + } + + @Override + public WalkerState run() { + return null; + } +} diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/FindUrlState.java b/announcement-processor-parser/src/main/java/parser/walker/states/FindUrlState.java new file mode 100644 index 0000000..a855e6c --- /dev/null +++ b/announcement-processor-parser/src/main/java/parser/walker/states/FindUrlState.java @@ -0,0 +1,20 @@ +package parser.walker.states; + +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; +import parser.walker.helpers.ProviderHelper; + +@Component +@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class FindUrlState extends WalkerState { + + public FindUrlState(ProviderHelper providerHelper) { + super(providerHelper); + } + + @Override + public WalkerState run() { + return null; + } +} diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/NewerPageState.java b/announcement-processor-parser/src/main/java/parser/walker/states/NewerPageState.java new file mode 100644 index 0000000..6d4ffb8 --- /dev/null +++ b/announcement-processor-parser/src/main/java/parser/walker/states/NewerPageState.java @@ -0,0 +1,20 @@ +package parser.walker.states; + +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; +import parser.walker.helpers.ProviderHelper; + +@Component +@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class NewerPageState extends WalkerState { + + public NewerPageState(ProviderHelper providerHelper) { + super(providerHelper); + } + + @Override + public WalkerState run() { + return null; + } +} diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java b/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java new file mode 100644 index 0000000..48d6bed --- /dev/null +++ b/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java @@ -0,0 +1,20 @@ +package parser.walker.states; + +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; +import parser.walker.helpers.ProviderHelper; + +@Component +@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class ProcessPageState extends WalkerState { + + public ProcessPageState(ProviderHelper providerHelper) { + super(providerHelper); + } + + @Override + public WalkerState run() { + return null; + } +} diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/StopState.java b/announcement-processor-parser/src/main/java/parser/walker/states/StopState.java new file mode 100644 index 0000000..477475d --- /dev/null +++ b/announcement-processor-parser/src/main/java/parser/walker/states/StopState.java @@ -0,0 +1,20 @@ +package parser.walker.states; + +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; +import parser.walker.helpers.ProviderHelper; + +@Component +@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class StopState extends WalkerState { + + public StopState(ProviderHelper providerHelper) { + super(providerHelper); + } + + @Override + public WalkerState run() { + return null; + } +} diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/TopElementState.java b/announcement-processor-parser/src/main/java/parser/walker/states/TopElementState.java new file mode 100644 index 0000000..b021bee --- /dev/null +++ b/announcement-processor-parser/src/main/java/parser/walker/states/TopElementState.java @@ -0,0 +1,20 @@ +package parser.walker.states; + +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; +import parser.walker.helpers.ProviderHelper; + +@Component +@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class TopElementState extends WalkerState { + + public TopElementState(ProviderHelper providerHelper) { + super(providerHelper); + } + + @Override + public WalkerState run() { + return null; + } +} diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/VisibleAfterReloadState.java b/announcement-processor-parser/src/main/java/parser/walker/states/VisibleAfterReloadState.java new file mode 100644 index 0000000..31d137c --- /dev/null +++ b/announcement-processor-parser/src/main/java/parser/walker/states/VisibleAfterReloadState.java @@ -0,0 +1,20 @@ +package parser.walker.states; + +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; +import parser.walker.helpers.ProviderHelper; + +@Component +@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class VisibleAfterReloadState extends WalkerState { + + public VisibleAfterReloadState(ProviderHelper providerHelper) { + super(providerHelper); + } + + @Override + public WalkerState run() { + return null; + } +} diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/WalkerState.java b/announcement-processor-parser/src/main/java/parser/walker/states/WalkerState.java new file mode 100644 index 0000000..b87fc35 --- /dev/null +++ b/announcement-processor-parser/src/main/java/parser/walker/states/WalkerState.java @@ -0,0 +1,15 @@ +package parser.walker.states; + +import org.springframework.stereotype.Component; +import parser.walker.helpers.ProviderHelper; + +@Component +public abstract class WalkerState { + ProviderHelper providerHelper; + + public WalkerState(ProviderHelper providerHelper) { + this.providerHelper = providerHelper; + } + + public abstract WalkerState run(); +} From e4703cca35d71b03beac75c8003f328c56902839 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Thu, 29 Nov 2018 22:19:33 +0100 Subject: [PATCH 070/135] gitignore updated. #34 --- announcement-processor-parser/.gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/announcement-processor-parser/.gitignore b/announcement-processor-parser/.gitignore index c4702dd..20e9cad 100644 --- a/announcement-processor-parser/.gitignore +++ b/announcement-processor-parser/.gitignore @@ -3,6 +3,8 @@ mvnw mvnw.cmd +src/main/resources/application-prod.properties + ### STS ### .apt_generated .classpath From d4e21c0d5c890a7ef81a3b0f74bae34843651a25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 2 Dec 2018 00:05:18 +0100 Subject: [PATCH 071/135] Implementation of FetchRegistryState. #10 --- .../src/main/java/parser/walker/GumtreePageWalker.java | 4 ++++ .../main/java/parser/walker/helpers/GumtreeHelper.java | 9 +++++++++ .../java/parser/walker/helpers/ProviderHelper.java | 6 ++++++ .../java/parser/walker/states/FetchRegistryState.java | 10 ++++++++-- 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/announcement-processor-parser/src/main/java/parser/walker/GumtreePageWalker.java b/announcement-processor-parser/src/main/java/parser/walker/GumtreePageWalker.java index 052f12c..d2f76d9 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/GumtreePageWalker.java +++ b/announcement-processor-parser/src/main/java/parser/walker/GumtreePageWalker.java @@ -2,6 +2,7 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; +import parser.walker.states.StopState; import parser.walker.states.WalkerState; @Service @@ -15,5 +16,8 @@ public GumtreePageWalker(@Qualifier("fetchRegistryState") WalkerState walkerStat @Override public void walk(String startPageUrl) { + while(walkerState != null && !(walkerState instanceof StopState)){ + walkerState = walkerState.run(); + } } } diff --git a/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java b/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java index 68a2626..8c90b26 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java +++ b/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java @@ -1,8 +1,11 @@ package parser.walker.helpers; import org.springframework.stereotype.Component; +import parser.registry.ParsingInfo; import parser.registry.ParsingInfoService; +import java.util.Optional; + @Component public class GumtreeHelper extends ProviderHelper { @@ -10,4 +13,10 @@ public GumtreeHelper(ParsingInfoService parsingInfoService) { super(parsingInfoService); } + @Override + public Optional fetchLatestRecord() { + Optional info = parsingInfoService.fetchLastRecordForProvider("GUMTREE", lastParsedAnnouncement); + lastParsedAnnouncement = info.orElse(null); + return info; + } } diff --git a/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java b/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java index 0565cec..10ce70d 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java +++ b/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java @@ -1,14 +1,20 @@ package parser.walker.helpers; import org.springframework.stereotype.Component; +import parser.registry.ParsingInfo; import parser.registry.ParsingInfoService; +import java.util.Optional; + @Component public abstract class ProviderHelper { ParsingInfoService parsingInfoService; + ParsingInfo lastParsedAnnouncement; public ProviderHelper(ParsingInfoService parsingInfoService) { this.parsingInfoService = parsingInfoService; } + public abstract Optional fetchLatestRecord(); + } diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/FetchRegistryState.java b/announcement-processor-parser/src/main/java/parser/walker/states/FetchRegistryState.java index 7281375..3974047 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/states/FetchRegistryState.java +++ b/announcement-processor-parser/src/main/java/parser/walker/states/FetchRegistryState.java @@ -3,8 +3,11 @@ import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; +import parser.registry.ParsingInfo; import parser.walker.helpers.ProviderHelper; +import java.util.Optional; + @Component @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) public class FetchRegistryState extends WalkerState { @@ -15,6 +18,9 @@ public FetchRegistryState(ProviderHelper providerHelper) { @Override public WalkerState run() { - return null; + Optional parsingInfoToFind = providerHelper.fetchLatestRecord(); + return parsingInfoToFind + .map(parsingInfo -> new FindUrlState(providerHelper, parsingInfo)) + .orElseGet(() -> new FindUrlState(providerHelper, null)); } -} +} \ No newline at end of file From 75ee00eb44c8ec4c8c9f17361bc118b8c41d9750 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 2 Dec 2018 13:37:44 +0100 Subject: [PATCH 072/135] Added implementation for finding url on provider pages. #10 --- .../parser/walker/helpers/GumtreeHelper.java | 92 +++++++++++++++++++ .../parser/walker/helpers/ProviderHelper.java | 18 ++++ .../parser/walker/states/FindUrlState.java | 22 +++-- 3 files changed, 125 insertions(+), 7 deletions(-) diff --git a/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java b/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java index 8c90b26..4902adf 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java +++ b/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java @@ -1,14 +1,26 @@ package parser.walker.helpers; +import lombok.extern.slf4j.Slf4j; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; import org.springframework.stereotype.Component; import parser.registry.ParsingInfo; import parser.registry.ParsingInfoService; +import java.io.IOException; +import java.util.HashMap; import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +@Slf4j @Component public class GumtreeHelper extends ProviderHelper { + private final String BASE_ANNOUNCEMENTS_URL = "https://www.gumtree.pl/s-mieszkania-i-domy-do-wynajecia/krakow/v1c9008l3200208p1"; + private int divElementWithAnnouncementsNumber = 1; + public GumtreeHelper(ParsingInfoService parsingInfoService) { super(parsingInfoService); } @@ -19,4 +31,84 @@ public Optional fetchLatestRecord() { lastParsedAnnouncement = info.orElse(null); return info; } + + @Override + public Optional> findPageWithAnnouncement(ParsingInfo parsingInfoToFind) { + HashMap returnValues = null; + + if (parsingInfoToFind != null) { + try { + String urlToScan = null; + HashMap pageToBeFound; + int scannedPageNumber = 0; + int totalPage; + + do { + scannedPageNumber++; + urlToScan = getNextPageUrl(urlToScan, scannedPageNumber); + Document scannedPage = getPageAsDocumentFromUrl(urlToScan); + pageToBeFound = findAnnouncementUrlOnPage(scannedPage, parsingInfoToFind.getUrl()); + totalPage = getNumberOfTotalPages(scannedPage); + } while (pageToBeFound == null && scannedPageNumber <= totalPage); + + if (pageToBeFound != null) { + returnValues = new HashMap<>(); + returnValues.put("url", urlToScan); + returnValues.put("pageDocument", pageToBeFound.get("pageDocument")); + returnValues.put("divNumber", pageToBeFound.get("divNumber")); + returnValues.put("scannedPageNumber", scannedPageNumber); + } + } catch (IOException e) { + log.error("Error during finding page with announcement with url: " + parsingInfoToFind.getUrl()); + } + } + return Optional.ofNullable(returnValues); + } + + @Override + String getPageUrlFromElement(Element element) { + return element.selectFirst(".title > a").attr("abs:href"); + } + + @Override + HashMap findAnnouncementUrlOnPage(Document page, String url) { + Elements liElementsWithData = page + .select(".view").get(divElementWithAnnouncementsNumber) + .selectFirst("ul") + .select("li"); + for (int i = 0; i < liElementsWithData.size(); ++i) { + String elementUrl = getPageUrlFromElement(liElementsWithData.get(i)); + if (elementUrl.equals(url)) { + HashMap returnMap = new HashMap<>(); + returnMap.put("pageDocument", page); + returnMap.put("divNumber", i); + return returnMap; + } + } + return null; + } + + @Override + int getNumberOfTotalPages(Document scannedPage) { + String attr = scannedPage.selectFirst(".last.follows").attr("abs:href"); + Pattern pattern = Pattern.compile("page-[0-9]+"); + Matcher matcher = pattern.matcher(attr); + if (matcher.find()) { + String pageNumber = matcher.group().split("-")[1]; + return Integer.parseInt(pageNumber); + } + return 0; + } + + @Override + String getNextPageUrl(String previousUrl, int nextPageNumber) { + if (previousUrl == null) { + return BASE_ANNOUNCEMENTS_URL; + } + String[] splitted = previousUrl.split("(krakow/)(page-[0-9]+/)*"); + for (String s : splitted) { + System.out.println(s); + } + return splitted[0] + "krakow/page-" + nextPageNumber + "/" + splitted[1].replaceAll("p[0-9]", "p" + nextPageNumber); + } } diff --git a/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java b/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java index 10ce70d..0ae84be 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java +++ b/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java @@ -1,9 +1,14 @@ package parser.walker.helpers; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; import org.springframework.stereotype.Component; import parser.registry.ParsingInfo; import parser.registry.ParsingInfoService; +import java.io.IOException; +import java.util.HashMap; import java.util.Optional; @Component @@ -15,6 +20,19 @@ public ProviderHelper(ParsingInfoService parsingInfoService) { this.parsingInfoService = parsingInfoService; } + Document getPageAsDocumentFromUrl(String url) throws IOException { + return Jsoup.connect(url).get(); + } + public abstract Optional fetchLatestRecord(); + public abstract Optional> findPageWithAnnouncement(ParsingInfo parsingInfoToFind); + + abstract HashMap findAnnouncementUrlOnPage(Document page, String url); + + abstract String getPageUrlFromElement(Element element); + + abstract String getNextPageUrl(String previousUrl, int nextPageNumber); + + abstract int getNumberOfTotalPages(Document scannedPage); } diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/FindUrlState.java b/announcement-processor-parser/src/main/java/parser/walker/states/FindUrlState.java index a855e6c..87f8a2a 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/states/FindUrlState.java +++ b/announcement-processor-parser/src/main/java/parser/walker/states/FindUrlState.java @@ -1,20 +1,28 @@ package parser.walker.states; -import org.springframework.beans.factory.config.ConfigurableBeanFactory; -import org.springframework.context.annotation.Scope; -import org.springframework.stereotype.Component; +import parser.registry.ParsingInfo; import parser.walker.helpers.ProviderHelper; -@Component -@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) +import java.util.HashMap; +import java.util.Optional; + public class FindUrlState extends WalkerState { - public FindUrlState(ProviderHelper providerHelper) { + private ParsingInfo parsingInfoToFind; + + FindUrlState(ProviderHelper providerHelper, ParsingInfo parsingInfoToFind) { super(providerHelper); + this.parsingInfoToFind = parsingInfoToFind; } @Override public WalkerState run() { - return null; + Optional> pageWithAnnouncement = providerHelper.findPageWithAnnouncement(parsingInfoToFind); + + if (pageWithAnnouncement.isPresent()) { + return new ProcessPageState(providerHelper); + } else { + return new FetchRegistryState(providerHelper); + } } } From 925bcf0dc8070e79c5aa8f4dc1571dec3553ceec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sat, 1 Dec 2018 23:38:33 +0100 Subject: [PATCH 073/135] Implemented logic of fetching last record from db. #10 --- .../registry/ParsingInfoRepository.java | 6 + .../parser/registry/ParsingInfoService.java | 22 ++++ .../registry/ParsingInfoServiceTest.java | 107 ++++++++++++++++++ 3 files changed, 135 insertions(+) create mode 100644 announcement-processor-parser/src/test/java/parser/registry/ParsingInfoServiceTest.java diff --git a/announcement-processor-parser/src/main/java/parser/registry/ParsingInfoRepository.java b/announcement-processor-parser/src/main/java/parser/registry/ParsingInfoRepository.java index 69d4365..d641be0 100644 --- a/announcement-processor-parser/src/main/java/parser/registry/ParsingInfoRepository.java +++ b/announcement-processor-parser/src/main/java/parser/registry/ParsingInfoRepository.java @@ -3,6 +3,12 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; +import java.time.LocalDate; +import java.util.Optional; + @Repository public interface ParsingInfoRepository extends JpaRepository { + Optional findFirstByProviderOrderByDateDescCounterPerDateDesc(String provider); + Optional findFirstByProviderAndDateEqualsAndCounterPerDateLessThanOrderByCounterPerDateDesc(String provider, LocalDate date, Integer counterPerDate); + Optional findFirstByProviderAndDateLessThanOrderByDateDescCounterPerDateDesc(String provider, LocalDate date); } diff --git a/announcement-processor-parser/src/main/java/parser/registry/ParsingInfoService.java b/announcement-processor-parser/src/main/java/parser/registry/ParsingInfoService.java index 2a91319..505bbe3 100644 --- a/announcement-processor-parser/src/main/java/parser/registry/ParsingInfoService.java +++ b/announcement-processor-parser/src/main/java/parser/registry/ParsingInfoService.java @@ -2,6 +2,8 @@ import org.springframework.stereotype.Service; +import java.util.Optional; + @Service public class ParsingInfoService { @@ -10,4 +12,24 @@ public class ParsingInfoService { public ParsingInfoService(ParsingInfoRepository parsingInfoRepository) { this.parsingInfoRepository = parsingInfoRepository; } + + public Optional fetchLastRecordForProvider(String provider, ParsingInfo lastParsedAnnouncement) { + Optional announcement; + if (lastParsedAnnouncement == null) { + announcement = parsingInfoRepository.findFirstByProviderOrderByDateDescCounterPerDateDesc(provider); + } else { + announcement = parsingInfoRepository.findFirstByProviderAndDateEqualsAndCounterPerDateLessThanOrderByCounterPerDateDesc( + provider, + lastParsedAnnouncement.getDate(), + lastParsedAnnouncement.getCounterPerDate() + ); + if (!announcement.isPresent()) { + announcement = parsingInfoRepository.findFirstByProviderAndDateLessThanOrderByDateDescCounterPerDateDesc( + provider, + lastParsedAnnouncement.getDate() + ); + } + } + return announcement; + } } diff --git a/announcement-processor-parser/src/test/java/parser/registry/ParsingInfoServiceTest.java b/announcement-processor-parser/src/test/java/parser/registry/ParsingInfoServiceTest.java new file mode 100644 index 0000000..5988481 --- /dev/null +++ b/announcement-processor-parser/src/test/java/parser/registry/ParsingInfoServiceTest.java @@ -0,0 +1,107 @@ +package parser.registry; + + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +import java.time.LocalDate; +import java.util.Optional; + + +public class ParsingInfoServiceTest { + + private ParsingInfoService parsingInfoService; + + @Mock + public ParsingInfoRepository parsingInfoRepository; + @Rule + public MockitoRule mockitoRule = MockitoJUnit.rule(); + + @Before + public void setUp() { + parsingInfoService = new ParsingInfoService(parsingInfoRepository); + } + + @Test + public void testFetchingLastRecordOnFirstCall() { + // given + String provider = "GUMTREE"; + ParsingInfo lastParsedAnnouncement = null; + + // when + parsingInfoService.fetchLastRecordForProvider(provider, lastParsedAnnouncement); + + // then + Mockito.verify(parsingInfoRepository, Mockito.times(1)) + .findFirstByProviderOrderByDateDescCounterPerDateDesc(provider); + } + + @Test + public void testFetchingLastRecordWhenNoMoreRecordsFromDay() { + // given + String provider = "GUMTREE"; + ParsingInfo lastParsedAnnouncement = new ParsingInfo(); + lastParsedAnnouncement.setDate(LocalDate.now()); + lastParsedAnnouncement.setCounterPerDate(1); + + // when + Mockito.when( + parsingInfoRepository.findFirstByProviderAndDateEqualsAndCounterPerDateLessThanOrderByCounterPerDateDesc( + provider, + lastParsedAnnouncement.getDate(), + lastParsedAnnouncement.getCounterPerDate() + ) + ).thenReturn(Optional.empty()); + parsingInfoService.fetchLastRecordForProvider(provider, lastParsedAnnouncement); + + // then + Mockito.verify(parsingInfoRepository, Mockito.times(1)) + .findFirstByProviderAndDateEqualsAndCounterPerDateLessThanOrderByCounterPerDateDesc( + Mockito.anyString(), + Mockito.any(), + Mockito.anyInt() + ); + Mockito.verify(parsingInfoRepository, Mockito.times(1)) + .findFirstByProviderAndDateLessThanOrderByDateDescCounterPerDateDesc( + Mockito.anyString(), + Mockito.any() + ); + } + + @Test + public void testFetchingLastRecordWhenMoreRecordsFromDay() { + // given + String provider = "GUMTREE"; + ParsingInfo lastParsedAnnouncement = new ParsingInfo(); + lastParsedAnnouncement.setDate(LocalDate.now()); + lastParsedAnnouncement.setCounterPerDate(1); + + // when + Mockito.when( + parsingInfoRepository.findFirstByProviderAndDateEqualsAndCounterPerDateLessThanOrderByCounterPerDateDesc( + provider, + lastParsedAnnouncement.getDate(), + lastParsedAnnouncement.getCounterPerDate() + ) + ).thenReturn(Optional.of(new ParsingInfo())); + parsingInfoService.fetchLastRecordForProvider(provider, lastParsedAnnouncement); + + // then + Mockito.verify(parsingInfoRepository, Mockito.times(1)) + .findFirstByProviderAndDateEqualsAndCounterPerDateLessThanOrderByCounterPerDateDesc( + Mockito.anyString(), + Mockito.any(), + Mockito.anyInt() + ); + Mockito.verify(parsingInfoRepository, Mockito.times(0)) + .findFirstByProviderAndDateLessThanOrderByDateDescCounterPerDateDesc( + Mockito.anyString(), + Mockito.any() + ); + } +} \ No newline at end of file From b4353c0f4d99d8ae12c78aaae7bf38d283447e0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 2 Dec 2018 00:05:18 +0100 Subject: [PATCH 074/135] Fix for function which generates next url. #10 --- .../src/main/java/parser/walker/helpers/GumtreeHelper.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java b/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java index 4902adf..2b76a0e 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java +++ b/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java @@ -52,6 +52,7 @@ public Optional> findPageWithAnnouncement(ParsingInfo pa } while (pageToBeFound == null && scannedPageNumber <= totalPage); if (pageToBeFound != null) { + log.debug("\nURL: " + parsingInfoToFind.getUrl() + "\n\t at page: " + scannedPageNumber); returnValues = new HashMap<>(); returnValues.put("url", urlToScan); returnValues.put("pageDocument", pageToBeFound.get("pageDocument")); @@ -106,9 +107,6 @@ String getNextPageUrl(String previousUrl, int nextPageNumber) { return BASE_ANNOUNCEMENTS_URL; } String[] splitted = previousUrl.split("(krakow/)(page-[0-9]+/)*"); - for (String s : splitted) { - System.out.println(s); - } - return splitted[0] + "krakow/page-" + nextPageNumber + "/" + splitted[1].replaceAll("p[0-9]", "p" + nextPageNumber); + return splitted[0] + "krakow/page-" + nextPageNumber + "/" + splitted[1].replaceAll("p[0-9]+", "p" + nextPageNumber); } } From 41717f5a24bb6af4c46295f33c09d2e1f46bbb3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 2 Dec 2018 18:58:37 +0100 Subject: [PATCH 075/135] Added breakpoint when date is too old comparing to parsingInfoDate. #10 --- .../parser/walker/helpers/GumtreeHelper.java | 60 +++++++++++++++---- .../parser/walker/helpers/ProviderHelper.java | 3 + 2 files changed, 53 insertions(+), 10 deletions(-) diff --git a/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java b/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java index 2b76a0e..d990ab5 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java +++ b/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java @@ -9,6 +9,9 @@ import parser.registry.ParsingInfoService; import java.io.IOException; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.Comparator; import java.util.HashMap; import java.util.Optional; import java.util.regex.Matcher; @@ -39,20 +42,24 @@ public Optional> findPageWithAnnouncement(ParsingInfo pa if (parsingInfoToFind != null) { try { String urlToScan = null; - HashMap pageToBeFound; + HashMap pageToBeFound = null; int scannedPageNumber = 0; - int totalPage; + int totalPage = -1; + LocalDate date; do { scannedPageNumber++; urlToScan = getNextPageUrl(urlToScan, scannedPageNumber); Document scannedPage = getPageAsDocumentFromUrl(urlToScan); - pageToBeFound = findAnnouncementUrlOnPage(scannedPage, parsingInfoToFind.getUrl()); - totalPage = getNumberOfTotalPages(scannedPage); - } while (pageToBeFound == null && scannedPageNumber <= totalPage); + date = getEarliestDateOnAnnouncementPage(scannedPage); + if (date != null && !date.isBefore(parsingInfoToFind.getDate())) { + pageToBeFound = findAnnouncementUrlOnPage(scannedPage, parsingInfoToFind.getUrl()); + totalPage = getNumberOfTotalPages(scannedPage); + } + } + while (pageToBeFound == null && date != null && !date.isBefore(parsingInfoToFind.getDate()) && scannedPageNumber <= totalPage); if (pageToBeFound != null) { - log.debug("\nURL: " + parsingInfoToFind.getUrl() + "\n\t at page: " + scannedPageNumber); returnValues = new HashMap<>(); returnValues.put("url", urlToScan); returnValues.put("pageDocument", pageToBeFound.get("pageDocument")); @@ -66,6 +73,42 @@ public Optional> findPageWithAnnouncement(ParsingInfo pa return Optional.ofNullable(returnValues); } + private LocalDate getEarliestDateOnAnnouncementPage(Document scannedPage) { + Elements elementsWithData = getElementsWithDataFromPage(scannedPage); + Optional first = elementsWithData.stream() + .map(element -> element.selectFirst(".creation-date").text()) + .map(this::convertStringToLocalDate) + .min(Comparator.reverseOrder()); + return first.orElse(null); + } + + private LocalDate convertStringToLocalDate(String date) { + LocalDateTime actualDateTime = LocalDateTime.now(); + LocalDate toReturn; + + if (date.contains("temu")) { + String[] partsOfDate = date.split(" "); + if (partsOfDate[1].equals("min")) { + actualDateTime = actualDateTime.minusMinutes(Integer.parseInt(partsOfDate[0])); + } else { + actualDateTime = actualDateTime.minusHours(Integer.parseInt(partsOfDate[0]) + 1); + } + toReturn = actualDateTime.toLocalDate(); + } else { + String[] dayMonth = date.split("-"); + toReturn = LocalDate.of(actualDateTime.getYear(), Integer.parseInt(dayMonth[1]), Integer.parseInt(dayMonth[0])); + } + return toReturn; + } + + @Override + Elements getElementsWithDataFromPage(Document page) { + return page + .select(".view").get(divElementWithAnnouncementsNumber) + .selectFirst("ul") + .select("li"); + } + @Override String getPageUrlFromElement(Element element) { return element.selectFirst(".title > a").attr("abs:href"); @@ -73,10 +116,7 @@ String getPageUrlFromElement(Element element) { @Override HashMap findAnnouncementUrlOnPage(Document page, String url) { - Elements liElementsWithData = page - .select(".view").get(divElementWithAnnouncementsNumber) - .selectFirst("ul") - .select("li"); + Elements liElementsWithData = getElementsWithDataFromPage(page); for (int i = 0; i < liElementsWithData.size(); ++i) { String elementUrl = getPageUrlFromElement(liElementsWithData.get(i)); if (elementUrl.equals(url)) { diff --git a/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java b/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java index 0ae84be..4ea03d9 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java +++ b/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java @@ -3,6 +3,7 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; import org.springframework.stereotype.Component; import parser.registry.ParsingInfo; import parser.registry.ParsingInfoService; @@ -30,6 +31,8 @@ Document getPageAsDocumentFromUrl(String url) throws IOException { abstract HashMap findAnnouncementUrlOnPage(Document page, String url); + abstract Elements getElementsWithDataFromPage(Document page); + abstract String getPageUrlFromElement(Element element); abstract String getNextPageUrl(String previousUrl, int nextPageNumber); From 143dd0a3524cbe222bd2ca8efbfbdada138f1a16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Wed, 5 Dec 2018 17:20:28 +0100 Subject: [PATCH 076/135] Fixed methods startsWith parse and added test coverage for it. #40 --- .../scrapper/GumtreeAnnouncementParser.java | 251 +++++++++++------- .../GumtreeParserNonRequiredFieldsTest.java | 82 ++++++ .../GumtreeParserRequiredFieldsTest.java | 47 ++++ 3 files changed, 286 insertions(+), 94 deletions(-) diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java index cb1b6c2..352ed88 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java @@ -51,22 +51,31 @@ public Announcement parsePage(String url) { @Override public String parseTitle(Element titleElement) { - return titleElement.text(); + String title = null; + + if (titleElement != null) { + title = titleElement.text(); + } + + return title; } @Override public BigDecimal parsePrice(Element priceElement) { BigDecimal output = BigDecimal.ZERO; - Element span = priceElement.selectFirst("span"); - String text = span.text(); - - if (text.contains("zł")) { - String number = text.replaceAll("zł", "").replaceAll(" ", ""); - output = new BigDecimal(number); - } else if (text.equals("Proszę o kontakt")) { - output = null; - } else if (text.equals("Wymiana/zamiana")) { - output = new BigDecimal(-1); + + if (priceElement != null) { + Element span = priceElement.selectFirst("span"); + String text = span.text(); + + if (text.contains("zł")) { + String number = text.replaceAll("zł", "").replaceAll(" ", ""); + output = new BigDecimal(number); + } else if (text.equals("Proszę o kontakt")) { + output = null; + } else if (text.equals("Wymiana/zamiana")) { + output = new BigDecimal(-1); + } } return output; @@ -74,20 +83,28 @@ public BigDecimal parsePrice(Element priceElement) { @Override public String parseLessorName(Element lessorNameElement) { - Element href = lessorNameElement.selectFirst("a"); - String text = href.text(); - return text.replaceAll("\\(Zobacz więcej ogłoszeń\\)", "").trim(); + String lessorName = null; + + if (lessorNameElement != null) { + Element href = lessorNameElement.selectFirst("a"); + String text = href.text(); + lessorName = text.replaceAll("\\(Zobacz więcej ogłoszeń\\)", "").trim(); + } + + return lessorName; } @Override public LocalDateTime parseCreationDate(Element creationDateElement) { LocalDateTime output = null; - Elements liElements = creationDateElement.select("li"); - String dateInText = getValueForAttributeFromLiElements("Data dodania", liElements); + if (creationDateElement != null) { + Elements liElements = creationDateElement.select("li"); + String dateInText = getValueForAttributeFromLiElements("Data dodania", liElements); - if (dateInText != null) { - output = LocalDateTime.of(LocalDate.parse(dateInText, DateTimeFormatter.ofPattern("d/MM/yyyy")), LocalTime.of(0, 0)); + if (dateInText != null) { + output = LocalDateTime.of(LocalDate.parse(dateInText, DateTimeFormatter.ofPattern("d/MM/yyyy")), LocalTime.of(0, 0)); + } } return output; @@ -95,86 +112,114 @@ public LocalDateTime parseCreationDate(Element creationDateElement) { @Override public String parseDescription(Element element) { - Element span = element.selectFirst("span"); - String textWithoutBoldings = span.html().replaceAll("|", ""); - Document parse = Jsoup.parse(textWithoutBoldings); - - Elements allElements = parse.body().getAllElements(); - StringBuilder sb = new StringBuilder(); - for (Element elem : allElements) { - String ownText = elem.ownText(); - if (!ownText.equals("")) { - sb.append(elem.ownText()).append("\n"); - } else { - if (elem.html().equals("
")) { - sb.append("\n"); + String description = null; + + if (element != null) { + Element span = element.selectFirst("span"); + String textWithoutBoldings = span.html().replaceAll("|", ""); + Document parse = Jsoup.parse(textWithoutBoldings); + + Elements allElements = parse.body().getAllElements(); + StringBuilder sb = new StringBuilder(); + for (Element elem : allElements) { + String ownText = elem.ownText(); + if (!ownText.equals("")) { + sb.append(elem.ownText()).append("\n"); + } else { + if (elem.html().equals("
")) { + sb.append("\n"); + } } } + description = sb.toString().trim(); } - return sb.toString().trim(); + + return description; } @Override public String parseLessor(Element lessorElement) { - Elements liElements = lessorElement.select("li"); - return getValueForAttributeFromLiElements("Do wynajęcia przez", liElements); + String lessor = null; + + if (lessorElement != null) { + Elements liElements = lessorElement.select("li"); + lessor = getValueForAttributeFromLiElements("Do wynajęcia przez", liElements); + } + + return lessor; } @Override public String parsePhoneNumber(Element phoneNumberElement) { - Element element = phoneNumberElement.selectFirst("div > a"); - String telephone = element.attr("href"); - if (telephone.contains("tel:")) { - return telephone.replaceAll("tel:", ""); + String phoneNumber = null; + + if (phoneNumberElement != null) { + Element element = phoneNumberElement.selectFirst("div > a"); + String telephone = element.attr("href"); + if (telephone.contains("tel:")) { + phoneNumber = telephone.replaceAll("tel:", ""); + } } - return null; + return phoneNumber; } @Override public String parsePropertyType(Element propertyTypeElement) { - Elements liElements = propertyTypeElement.select("li"); - return getValueForAttributeFromLiElements("Rodzaj nieruchomości", liElements); + String propertyType = null; + + if (propertyTypeElement != null) { + Elements liElements = propertyTypeElement.select("li"); + propertyType = getValueForAttributeFromLiElements("Rodzaj nieruchomości", liElements); + } + + return propertyType; } @Override public Double parseFlatArea(Element flatAreaElement) { - Elements liElements = flatAreaElement.select("li"); - String flatAreaInText = getValueForAttributeFromLiElements("Wielkość (m2)", liElements); + Double flatArea = null; - if (flatAreaInText != null) { - return Double.valueOf(flatAreaInText); + if (flatAreaElement != null) { + Elements liElements = flatAreaElement.select("li"); + String flatAreaInText = getValueForAttributeFromLiElements("Wielkość (m2)", liElements); + + if (flatAreaInText != null) { + flatArea = Double.valueOf(flatAreaInText); + } } - return null; + return flatArea; } @Override public Integer parseRoomAmount(Element roomAmountElement) { - Elements liElements = roomAmountElement.select("li"); - String roomAmountText = getValueForAttributeFromLiElements("Liczba pokoi", liElements); - Integer roomAmount = null; - if (roomAmountText != null) { - switch (roomAmountText) { - case "Kawalerka lub garsoniera": - roomAmount = 1; - break; - case "2 pokoje": - roomAmount = 2; - break; - case "3 pokoje": - roomAmount = 3; - break; - case "4 pokoje": - roomAmount = 4; - break; - case "5 pokoi": - roomAmount = 5; - break; - case "6 lub więcej pokoi": - roomAmount = 6; - break; + if (roomAmountElement != null) { + Elements liElements = roomAmountElement.select("li"); + String roomAmountText = getValueForAttributeFromLiElements("Liczba pokoi", liElements); + + if (roomAmountText != null) { + switch (roomAmountText) { + case "Kawalerka lub garsoniera": + roomAmount = 1; + break; + case "2 pokoje": + roomAmount = 2; + break; + case "3 pokoje": + roomAmount = 3; + break; + case "4 pokoje": + roomAmount = 4; + break; + case "5 pokoi": + roomAmount = 5; + break; + case "6 lub więcej pokoi": + roomAmount = 6; + break; + } } } @@ -183,25 +228,27 @@ public Integer parseRoomAmount(Element roomAmountElement) { @Override public Integer parseBathAmount(Element bathAmountElement) { - Elements liElements = bathAmountElement.select("li"); - String bathAmountText = getValueForAttributeFromLiElements("Liczba łazienek", liElements); - Integer bathAmount = null; - if (bathAmountText != null) { - switch (bathAmountText) { - case "1 łazienka": - bathAmount = 1; - break; - case "2 łazienki": - bathAmount = 2; - break; - case "3 łazienki": - bathAmount = 3; - break; - case "4 lub więcej łazienek": - bathAmount = 4; - break; + if (bathAmountElement != null) { + Elements liElements = bathAmountElement.select("li"); + String bathAmountText = getValueForAttributeFromLiElements("Liczba łazienek", liElements); + + if (bathAmountText != null) { + switch (bathAmountText) { + case "1 łazienka": + bathAmount = 1; + break; + case "2 łazienki": + bathAmount = 2; + break; + case "3 łazienki": + bathAmount = 3; + break; + case "4 lub więcej łazienek": + bathAmount = 4; + break; + } } } @@ -210,24 +257,40 @@ public Integer parseBathAmount(Element bathAmountElement) { @Override public String parseParking(Element parkingElement) { - Elements liElements = parkingElement.select("li"); - return getValueForAttributeFromLiElements("Parking", liElements); + String parking = null; + + if (parkingElement != null) { + Elements liElements = parkingElement.select("li"); + parking = getValueForAttributeFromLiElements("Parking", liElements); + } + + return parking; } @Override public Boolean parseSmokers(Element smokersElement) { - Elements liElements = smokersElement.select("li"); - String smokers = getValueForAttributeFromLiElements("Palący", liElements); + Boolean isForSmokers = null; - return getBooleanFromYesNoSentence(smokers); + if (smokersElement != null) { + Elements liElements = smokersElement.select("li"); + String smokers = getValueForAttributeFromLiElements("Palący", liElements); + isForSmokers = getBooleanFromYesNoSentence(smokers); + } + + return isForSmokers; } @Override public Boolean parsePetFriendly(Element petFriendlyElement) { - Elements liElements = petFriendlyElement.select("li"); - String pets = getValueForAttributeFromLiElements("Przyjazne zwierzakom", liElements); + Boolean isPetFriendly = null; + + if (petFriendlyElement != null) { + Elements liElements = petFriendlyElement.select("li"); + String pets = getValueForAttributeFromLiElements("Przyjazne zwierzakom", liElements); + isPetFriendly = getBooleanFromYesNoSentence(pets); + } - return getBooleanFromYesNoSentence(pets); + return isPetFriendly; } @Override diff --git a/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserNonRequiredFieldsTest.java b/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserNonRequiredFieldsTest.java index 8395309..4867e80 100644 --- a/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserNonRequiredFieldsTest.java +++ b/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserNonRequiredFieldsTest.java @@ -43,6 +43,15 @@ public void testPhoneNumberFieldExists() { Assert.assertEquals(GumtreeExpectedProperties.expectedPhoneNumberExists, phoneNumber); } + @Test + public void testPhoneNumberElementNull() { + // when + String phoneNumber = gumtreeParser.parsePhoneNumber(null); + + // then + Assert.assertNull(phoneNumber); + } + @Test public void testLessorFieldExists() { // given @@ -69,6 +78,15 @@ public void testLessorFieldNotExists() { Assert.assertNull(lessor); } + @Test + public void testLessorFieldElementNull() { + // when + String lessor = gumtreeParser.parseLessor(null); + + // then + Assert.assertNull(lessor); + } + @Test public void testPropertyTypeFieldExists() { // given @@ -95,6 +113,15 @@ public void testPropertyTypeFieldNotExists() { Assert.assertNull(propertyType); } + @Test + public void testPropertyTypeElementNull() { + // when + String propertyType = gumtreeParser.parsePropertyType(null); + + // then + Assert.assertNull(propertyType); + } + @Test public void testFlatAreaFieldExists() { // given @@ -121,6 +148,15 @@ public void testFlatAreaFieldNotExists() { Assert.assertNull(flatArea); } + @Test + public void testFlatAreaElementNull() { + // when + Double flatArea = gumtreeParser.parseFlatArea(null); + + // then + Assert.assertNull(flatArea); + } + @Test public void testRoomAmountFieldExists() { // given @@ -147,6 +183,15 @@ public void testRoomAmountFieldNotExists() { Assert.assertNull(roomAmount); } + @Test + public void testRoomAmountElementNull() { + // when + Integer roomAmount = gumtreeParser.parseRoomAmount(null); + + // then + Assert.assertNull(roomAmount); + } + @Test public void testBathAmountFieldExists() { // given @@ -173,6 +218,15 @@ public void testBathAmountFieldNotExists() { Assert.assertNull(bathAmount); } + @Test + public void testBathAmountElementNull() { + // when + Integer bathAmount = gumtreeParser.parseBathAmount(null); + + // then + Assert.assertNull(bathAmount); + } + @Test public void testParkingFieldExists() { // given @@ -199,6 +253,15 @@ public void testParkingFieldNotExists() { Assert.assertNull(parking); } + @Test + public void testParkingFieldElementNull() { + // when + String parking = gumtreeParser.parseParking(null); + + // then + Assert.assertNull(parking); + } + @Test public void testSmokersFieldExists() { // given @@ -225,6 +288,15 @@ public void testSmokersFieldNotExists() { Assert.assertNull(smokers); } + @Test + public void testSmokersFieldElementNull() { + // when + Boolean isForSmokers = gumtreeParser.parseSmokers(null); + + // then + Assert.assertNull(isForSmokers); + } + @Test public void testPetFriendlyFieldExists() { // given @@ -250,4 +322,14 @@ public void testPetFriendlyFieldNotExists() { // then Assert.assertNull(petFriendly); } + + + @Test + public void testPetFriendlyElementNull() { + // when + Boolean isPetFriendly = gumtreeParser.parsePetFriendly(null); + + // then + Assert.assertNull(isPetFriendly); + } } \ No newline at end of file diff --git a/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserRequiredFieldsTest.java b/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserRequiredFieldsTest.java index 150f52e..d2d0308 100644 --- a/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserRequiredFieldsTest.java +++ b/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserRequiredFieldsTest.java @@ -54,6 +54,18 @@ public void testParseTitle() { Assert.assertEquals(GumtreeExpectedProperties.expectedTitle, title); } + @Test + public void testParseTitleNull() { + // given + Element titleElement = null; + + // when + String title = gumtreeParser.parseTitle(titleElement); + + // then + Assert.assertNull(title); + } + @Test public void testParsePrice() { // given @@ -66,6 +78,15 @@ public void testParsePrice() { Assert.assertEquals(GumtreeExpectedProperties.expectedPrice, price); } + @Test + public void testParsePriceNull() { + // when + BigDecimal price = gumtreeParser.parsePrice(null); + + // then + Assert.assertEquals(BigDecimal.ZERO, price); + } + @Test public void testLessorName() { // given @@ -78,6 +99,14 @@ public void testLessorName() { Assert.assertEquals(GumtreeExpectedProperties.expectedLessorName, lessorName); } + @Test + public void testLessorNameNull() { + // when + String lessorName = gumtreeParser.parseLessorName(null); + + // then + Assert.assertNull(lessorName); + } @Test public void testCreationDate() { @@ -91,6 +120,15 @@ public void testCreationDate() { Assert.assertEquals(GumtreeExpectedProperties.expectedCreationDateTime, dateTime); } + @Test + public void testCreationDateNull() { + // when + LocalDateTime dateTime = gumtreeParser.parseCreationDate(null); + + // then + Assert.assertNull(dateTime); + } + @Test public void testDescription() { // given @@ -102,4 +140,13 @@ public void testDescription() { // then Assert.assertEquals(GumtreeExpectedProperties.expectedDescription, description); } + + @Test + public void testDescriptionNull() { + // when + String description = gumtreeParser.parseDescription(null); + + // then + Assert.assertNull(description); + } } From e46de3569a843f195653efcdf6bea15d35a6c94e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Wed, 5 Dec 2018 17:20:28 +0100 Subject: [PATCH 077/135] Fixed methods startsWith parse and added test coverage for it. #40 --- .../scrapper/GumtreeAnnouncementParser.java | 251 +++++++++++------- .../GumtreeParserNonRequiredFieldsTest.java | 82 ++++++ .../GumtreeParserRequiredFieldsTest.java | 47 ++++ 3 files changed, 286 insertions(+), 94 deletions(-) diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java index cb1b6c2..352ed88 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java @@ -51,22 +51,31 @@ public Announcement parsePage(String url) { @Override public String parseTitle(Element titleElement) { - return titleElement.text(); + String title = null; + + if (titleElement != null) { + title = titleElement.text(); + } + + return title; } @Override public BigDecimal parsePrice(Element priceElement) { BigDecimal output = BigDecimal.ZERO; - Element span = priceElement.selectFirst("span"); - String text = span.text(); - - if (text.contains("zł")) { - String number = text.replaceAll("zł", "").replaceAll(" ", ""); - output = new BigDecimal(number); - } else if (text.equals("Proszę o kontakt")) { - output = null; - } else if (text.equals("Wymiana/zamiana")) { - output = new BigDecimal(-1); + + if (priceElement != null) { + Element span = priceElement.selectFirst("span"); + String text = span.text(); + + if (text.contains("zł")) { + String number = text.replaceAll("zł", "").replaceAll(" ", ""); + output = new BigDecimal(number); + } else if (text.equals("Proszę o kontakt")) { + output = null; + } else if (text.equals("Wymiana/zamiana")) { + output = new BigDecimal(-1); + } } return output; @@ -74,20 +83,28 @@ public BigDecimal parsePrice(Element priceElement) { @Override public String parseLessorName(Element lessorNameElement) { - Element href = lessorNameElement.selectFirst("a"); - String text = href.text(); - return text.replaceAll("\\(Zobacz więcej ogłoszeń\\)", "").trim(); + String lessorName = null; + + if (lessorNameElement != null) { + Element href = lessorNameElement.selectFirst("a"); + String text = href.text(); + lessorName = text.replaceAll("\\(Zobacz więcej ogłoszeń\\)", "").trim(); + } + + return lessorName; } @Override public LocalDateTime parseCreationDate(Element creationDateElement) { LocalDateTime output = null; - Elements liElements = creationDateElement.select("li"); - String dateInText = getValueForAttributeFromLiElements("Data dodania", liElements); + if (creationDateElement != null) { + Elements liElements = creationDateElement.select("li"); + String dateInText = getValueForAttributeFromLiElements("Data dodania", liElements); - if (dateInText != null) { - output = LocalDateTime.of(LocalDate.parse(dateInText, DateTimeFormatter.ofPattern("d/MM/yyyy")), LocalTime.of(0, 0)); + if (dateInText != null) { + output = LocalDateTime.of(LocalDate.parse(dateInText, DateTimeFormatter.ofPattern("d/MM/yyyy")), LocalTime.of(0, 0)); + } } return output; @@ -95,86 +112,114 @@ public LocalDateTime parseCreationDate(Element creationDateElement) { @Override public String parseDescription(Element element) { - Element span = element.selectFirst("span"); - String textWithoutBoldings = span.html().replaceAll("|", ""); - Document parse = Jsoup.parse(textWithoutBoldings); - - Elements allElements = parse.body().getAllElements(); - StringBuilder sb = new StringBuilder(); - for (Element elem : allElements) { - String ownText = elem.ownText(); - if (!ownText.equals("")) { - sb.append(elem.ownText()).append("\n"); - } else { - if (elem.html().equals("
")) { - sb.append("\n"); + String description = null; + + if (element != null) { + Element span = element.selectFirst("span"); + String textWithoutBoldings = span.html().replaceAll("|", ""); + Document parse = Jsoup.parse(textWithoutBoldings); + + Elements allElements = parse.body().getAllElements(); + StringBuilder sb = new StringBuilder(); + for (Element elem : allElements) { + String ownText = elem.ownText(); + if (!ownText.equals("")) { + sb.append(elem.ownText()).append("\n"); + } else { + if (elem.html().equals("
")) { + sb.append("\n"); + } } } + description = sb.toString().trim(); } - return sb.toString().trim(); + + return description; } @Override public String parseLessor(Element lessorElement) { - Elements liElements = lessorElement.select("li"); - return getValueForAttributeFromLiElements("Do wynajęcia przez", liElements); + String lessor = null; + + if (lessorElement != null) { + Elements liElements = lessorElement.select("li"); + lessor = getValueForAttributeFromLiElements("Do wynajęcia przez", liElements); + } + + return lessor; } @Override public String parsePhoneNumber(Element phoneNumberElement) { - Element element = phoneNumberElement.selectFirst("div > a"); - String telephone = element.attr("href"); - if (telephone.contains("tel:")) { - return telephone.replaceAll("tel:", ""); + String phoneNumber = null; + + if (phoneNumberElement != null) { + Element element = phoneNumberElement.selectFirst("div > a"); + String telephone = element.attr("href"); + if (telephone.contains("tel:")) { + phoneNumber = telephone.replaceAll("tel:", ""); + } } - return null; + return phoneNumber; } @Override public String parsePropertyType(Element propertyTypeElement) { - Elements liElements = propertyTypeElement.select("li"); - return getValueForAttributeFromLiElements("Rodzaj nieruchomości", liElements); + String propertyType = null; + + if (propertyTypeElement != null) { + Elements liElements = propertyTypeElement.select("li"); + propertyType = getValueForAttributeFromLiElements("Rodzaj nieruchomości", liElements); + } + + return propertyType; } @Override public Double parseFlatArea(Element flatAreaElement) { - Elements liElements = flatAreaElement.select("li"); - String flatAreaInText = getValueForAttributeFromLiElements("Wielkość (m2)", liElements); + Double flatArea = null; - if (flatAreaInText != null) { - return Double.valueOf(flatAreaInText); + if (flatAreaElement != null) { + Elements liElements = flatAreaElement.select("li"); + String flatAreaInText = getValueForAttributeFromLiElements("Wielkość (m2)", liElements); + + if (flatAreaInText != null) { + flatArea = Double.valueOf(flatAreaInText); + } } - return null; + return flatArea; } @Override public Integer parseRoomAmount(Element roomAmountElement) { - Elements liElements = roomAmountElement.select("li"); - String roomAmountText = getValueForAttributeFromLiElements("Liczba pokoi", liElements); - Integer roomAmount = null; - if (roomAmountText != null) { - switch (roomAmountText) { - case "Kawalerka lub garsoniera": - roomAmount = 1; - break; - case "2 pokoje": - roomAmount = 2; - break; - case "3 pokoje": - roomAmount = 3; - break; - case "4 pokoje": - roomAmount = 4; - break; - case "5 pokoi": - roomAmount = 5; - break; - case "6 lub więcej pokoi": - roomAmount = 6; - break; + if (roomAmountElement != null) { + Elements liElements = roomAmountElement.select("li"); + String roomAmountText = getValueForAttributeFromLiElements("Liczba pokoi", liElements); + + if (roomAmountText != null) { + switch (roomAmountText) { + case "Kawalerka lub garsoniera": + roomAmount = 1; + break; + case "2 pokoje": + roomAmount = 2; + break; + case "3 pokoje": + roomAmount = 3; + break; + case "4 pokoje": + roomAmount = 4; + break; + case "5 pokoi": + roomAmount = 5; + break; + case "6 lub więcej pokoi": + roomAmount = 6; + break; + } } } @@ -183,25 +228,27 @@ public Integer parseRoomAmount(Element roomAmountElement) { @Override public Integer parseBathAmount(Element bathAmountElement) { - Elements liElements = bathAmountElement.select("li"); - String bathAmountText = getValueForAttributeFromLiElements("Liczba łazienek", liElements); - Integer bathAmount = null; - if (bathAmountText != null) { - switch (bathAmountText) { - case "1 łazienka": - bathAmount = 1; - break; - case "2 łazienki": - bathAmount = 2; - break; - case "3 łazienki": - bathAmount = 3; - break; - case "4 lub więcej łazienek": - bathAmount = 4; - break; + if (bathAmountElement != null) { + Elements liElements = bathAmountElement.select("li"); + String bathAmountText = getValueForAttributeFromLiElements("Liczba łazienek", liElements); + + if (bathAmountText != null) { + switch (bathAmountText) { + case "1 łazienka": + bathAmount = 1; + break; + case "2 łazienki": + bathAmount = 2; + break; + case "3 łazienki": + bathAmount = 3; + break; + case "4 lub więcej łazienek": + bathAmount = 4; + break; + } } } @@ -210,24 +257,40 @@ public Integer parseBathAmount(Element bathAmountElement) { @Override public String parseParking(Element parkingElement) { - Elements liElements = parkingElement.select("li"); - return getValueForAttributeFromLiElements("Parking", liElements); + String parking = null; + + if (parkingElement != null) { + Elements liElements = parkingElement.select("li"); + parking = getValueForAttributeFromLiElements("Parking", liElements); + } + + return parking; } @Override public Boolean parseSmokers(Element smokersElement) { - Elements liElements = smokersElement.select("li"); - String smokers = getValueForAttributeFromLiElements("Palący", liElements); + Boolean isForSmokers = null; - return getBooleanFromYesNoSentence(smokers); + if (smokersElement != null) { + Elements liElements = smokersElement.select("li"); + String smokers = getValueForAttributeFromLiElements("Palący", liElements); + isForSmokers = getBooleanFromYesNoSentence(smokers); + } + + return isForSmokers; } @Override public Boolean parsePetFriendly(Element petFriendlyElement) { - Elements liElements = petFriendlyElement.select("li"); - String pets = getValueForAttributeFromLiElements("Przyjazne zwierzakom", liElements); + Boolean isPetFriendly = null; + + if (petFriendlyElement != null) { + Elements liElements = petFriendlyElement.select("li"); + String pets = getValueForAttributeFromLiElements("Przyjazne zwierzakom", liElements); + isPetFriendly = getBooleanFromYesNoSentence(pets); + } - return getBooleanFromYesNoSentence(pets); + return isPetFriendly; } @Override diff --git a/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserNonRequiredFieldsTest.java b/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserNonRequiredFieldsTest.java index 8395309..4867e80 100644 --- a/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserNonRequiredFieldsTest.java +++ b/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserNonRequiredFieldsTest.java @@ -43,6 +43,15 @@ public void testPhoneNumberFieldExists() { Assert.assertEquals(GumtreeExpectedProperties.expectedPhoneNumberExists, phoneNumber); } + @Test + public void testPhoneNumberElementNull() { + // when + String phoneNumber = gumtreeParser.parsePhoneNumber(null); + + // then + Assert.assertNull(phoneNumber); + } + @Test public void testLessorFieldExists() { // given @@ -69,6 +78,15 @@ public void testLessorFieldNotExists() { Assert.assertNull(lessor); } + @Test + public void testLessorFieldElementNull() { + // when + String lessor = gumtreeParser.parseLessor(null); + + // then + Assert.assertNull(lessor); + } + @Test public void testPropertyTypeFieldExists() { // given @@ -95,6 +113,15 @@ public void testPropertyTypeFieldNotExists() { Assert.assertNull(propertyType); } + @Test + public void testPropertyTypeElementNull() { + // when + String propertyType = gumtreeParser.parsePropertyType(null); + + // then + Assert.assertNull(propertyType); + } + @Test public void testFlatAreaFieldExists() { // given @@ -121,6 +148,15 @@ public void testFlatAreaFieldNotExists() { Assert.assertNull(flatArea); } + @Test + public void testFlatAreaElementNull() { + // when + Double flatArea = gumtreeParser.parseFlatArea(null); + + // then + Assert.assertNull(flatArea); + } + @Test public void testRoomAmountFieldExists() { // given @@ -147,6 +183,15 @@ public void testRoomAmountFieldNotExists() { Assert.assertNull(roomAmount); } + @Test + public void testRoomAmountElementNull() { + // when + Integer roomAmount = gumtreeParser.parseRoomAmount(null); + + // then + Assert.assertNull(roomAmount); + } + @Test public void testBathAmountFieldExists() { // given @@ -173,6 +218,15 @@ public void testBathAmountFieldNotExists() { Assert.assertNull(bathAmount); } + @Test + public void testBathAmountElementNull() { + // when + Integer bathAmount = gumtreeParser.parseBathAmount(null); + + // then + Assert.assertNull(bathAmount); + } + @Test public void testParkingFieldExists() { // given @@ -199,6 +253,15 @@ public void testParkingFieldNotExists() { Assert.assertNull(parking); } + @Test + public void testParkingFieldElementNull() { + // when + String parking = gumtreeParser.parseParking(null); + + // then + Assert.assertNull(parking); + } + @Test public void testSmokersFieldExists() { // given @@ -225,6 +288,15 @@ public void testSmokersFieldNotExists() { Assert.assertNull(smokers); } + @Test + public void testSmokersFieldElementNull() { + // when + Boolean isForSmokers = gumtreeParser.parseSmokers(null); + + // then + Assert.assertNull(isForSmokers); + } + @Test public void testPetFriendlyFieldExists() { // given @@ -250,4 +322,14 @@ public void testPetFriendlyFieldNotExists() { // then Assert.assertNull(petFriendly); } + + + @Test + public void testPetFriendlyElementNull() { + // when + Boolean isPetFriendly = gumtreeParser.parsePetFriendly(null); + + // then + Assert.assertNull(isPetFriendly); + } } \ No newline at end of file diff --git a/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserRequiredFieldsTest.java b/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserRequiredFieldsTest.java index 150f52e..d2d0308 100644 --- a/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserRequiredFieldsTest.java +++ b/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserRequiredFieldsTest.java @@ -54,6 +54,18 @@ public void testParseTitle() { Assert.assertEquals(GumtreeExpectedProperties.expectedTitle, title); } + @Test + public void testParseTitleNull() { + // given + Element titleElement = null; + + // when + String title = gumtreeParser.parseTitle(titleElement); + + // then + Assert.assertNull(title); + } + @Test public void testParsePrice() { // given @@ -66,6 +78,15 @@ public void testParsePrice() { Assert.assertEquals(GumtreeExpectedProperties.expectedPrice, price); } + @Test + public void testParsePriceNull() { + // when + BigDecimal price = gumtreeParser.parsePrice(null); + + // then + Assert.assertEquals(BigDecimal.ZERO, price); + } + @Test public void testLessorName() { // given @@ -78,6 +99,14 @@ public void testLessorName() { Assert.assertEquals(GumtreeExpectedProperties.expectedLessorName, lessorName); } + @Test + public void testLessorNameNull() { + // when + String lessorName = gumtreeParser.parseLessorName(null); + + // then + Assert.assertNull(lessorName); + } @Test public void testCreationDate() { @@ -91,6 +120,15 @@ public void testCreationDate() { Assert.assertEquals(GumtreeExpectedProperties.expectedCreationDateTime, dateTime); } + @Test + public void testCreationDateNull() { + // when + LocalDateTime dateTime = gumtreeParser.parseCreationDate(null); + + // then + Assert.assertNull(dateTime); + } + @Test public void testDescription() { // given @@ -102,4 +140,13 @@ public void testDescription() { // then Assert.assertEquals(GumtreeExpectedProperties.expectedDescription, description); } + + @Test + public void testDescriptionNull() { + // when + String description = gumtreeParser.parseDescription(null); + + // then + Assert.assertNull(description); + } } From 145968b032d98b8dfbbcb39eacb4efd413d1a20f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Tue, 4 Dec 2018 11:31:08 +0100 Subject: [PATCH 078/135] Refactoring of return fields for method findPageWithAnnouncement. #10 --- .../main/java/parser/walker/helpers/GumtreeHelper.java | 4 ++-- .../main/java/parser/walker/helpers/ProviderHelper.java | 2 ++ .../src/main/java/parser/walker/states/FindUrlState.java | 8 +++----- .../main/java/parser/walker/states/ProcessPageState.java | 7 ++++++- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java b/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java index d990ab5..59c0d17 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java +++ b/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java @@ -61,10 +61,10 @@ public Optional> findPageWithAnnouncement(ParsingInfo pa if (pageToBeFound != null) { returnValues = new HashMap<>(); - returnValues.put("url", urlToScan); + actualPageURL = urlToScan; + actualPageURLNumber = scannedPageNumber; returnValues.put("pageDocument", pageToBeFound.get("pageDocument")); returnValues.put("divNumber", pageToBeFound.get("divNumber")); - returnValues.put("scannedPageNumber", scannedPageNumber); } } catch (IOException e) { log.error("Error during finding page with announcement with url: " + parsingInfoToFind.getUrl()); diff --git a/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java b/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java index 4ea03d9..18db303 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java +++ b/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java @@ -15,6 +15,8 @@ @Component public abstract class ProviderHelper { ParsingInfoService parsingInfoService; + String actualPageURL; + int actualPageURLNumber; ParsingInfo lastParsedAnnouncement; public ProviderHelper(ParsingInfoService parsingInfoService) { diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/FindUrlState.java b/announcement-processor-parser/src/main/java/parser/walker/states/FindUrlState.java index 87f8a2a..a3b7ec1 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/states/FindUrlState.java +++ b/announcement-processor-parser/src/main/java/parser/walker/states/FindUrlState.java @@ -19,10 +19,8 @@ public class FindUrlState extends WalkerState { public WalkerState run() { Optional> pageWithAnnouncement = providerHelper.findPageWithAnnouncement(parsingInfoToFind); - if (pageWithAnnouncement.isPresent()) { - return new ProcessPageState(providerHelper); - } else { - return new FetchRegistryState(providerHelper); - } + return pageWithAnnouncement + .map(stringObjectHashMap -> new ProcessPageState(providerHelper, stringObjectHashMap)) + .orElseGet(() -> new FetchRegistryState(providerHelper)); } } diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java b/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java index 48d6bed..b05a313 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java +++ b/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java @@ -5,12 +5,17 @@ import org.springframework.stereotype.Component; import parser.walker.helpers.ProviderHelper; +import java.util.HashMap; + @Component @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) public class ProcessPageState extends WalkerState { - public ProcessPageState(ProviderHelper providerHelper) { + private HashMap pageWithAnnouncement; + + public ProcessPageState(ProviderHelper providerHelper, HashMap pageWithAnnouncement) { super(providerHelper); + this.pageWithAnnouncement = pageWithAnnouncement; } @Override From 90650332bbdfac5c1205ff2da5dcdc6b6a335c18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Tue, 4 Dec 2018 12:31:30 +0100 Subject: [PATCH 079/135] Added GumtreeAnnouncementParser to GumtreePageWalker and set it to WalkerState. #10 --- .../java/parser/scrapper/GumtreeAnnouncementParser.java | 2 ++ .../src/main/java/parser/walker/GumtreePageWalker.java | 9 +++++++-- .../src/main/java/parser/walker/states/WalkerState.java | 7 +++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java index 352ed88..f6af709 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java @@ -5,6 +5,7 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; +import org.springframework.stereotype.Component; import parser.Announcement; import parser.exceptions.GumtreePageParseException; import parser.exceptions.PropertyNotValidForGumtreeProviderException; @@ -17,6 +18,7 @@ import java.time.format.DateTimeFormatter; @Slf4j +@Component public class GumtreeAnnouncementParser extends AnnouncementParser implements SinglePageParser { @Override public Announcement parsePage(String url) { diff --git a/announcement-processor-parser/src/main/java/parser/walker/GumtreePageWalker.java b/announcement-processor-parser/src/main/java/parser/walker/GumtreePageWalker.java index d2f76d9..5bd7a35 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/GumtreePageWalker.java +++ b/announcement-processor-parser/src/main/java/parser/walker/GumtreePageWalker.java @@ -2,16 +2,21 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; +import parser.scrapper.AnnouncementParser; +import parser.scrapper.GumtreeAnnouncementParser; import parser.walker.states.StopState; import parser.walker.states.WalkerState; @Service public class GumtreePageWalker extends PageWalker { - + private AnnouncementParser announcementParser; private WalkerState walkerState; - public GumtreePageWalker(@Qualifier("fetchRegistryState") WalkerState walkerState) { + public GumtreePageWalker(@Qualifier("fetchRegistryState") WalkerState walkerState, + GumtreeAnnouncementParser gumtreeAnnouncementParser) { this.walkerState = walkerState; + this.announcementParser = gumtreeAnnouncementParser; + walkerState.setAnnouncementParser(gumtreeAnnouncementParser); } @Override diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/WalkerState.java b/announcement-processor-parser/src/main/java/parser/walker/states/WalkerState.java index b87fc35..3b8a10a 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/states/WalkerState.java +++ b/announcement-processor-parser/src/main/java/parser/walker/states/WalkerState.java @@ -1,15 +1,22 @@ package parser.walker.states; import org.springframework.stereotype.Component; +import parser.scrapper.AnnouncementParser; +import parser.scrapper.GumtreeAnnouncementParser; import parser.walker.helpers.ProviderHelper; @Component public abstract class WalkerState { ProviderHelper providerHelper; + AnnouncementParser announcementParser; public WalkerState(ProviderHelper providerHelper) { this.providerHelper = providerHelper; } public abstract WalkerState run(); + + public void setAnnouncementParser(GumtreeAnnouncementParser gumtreeAnnouncementParser){ + this.announcementParser = gumtreeAnnouncementParser; + }; } From 44113e4059c6003a66af18c2ecc4db501091de23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Thu, 6 Dec 2018 22:25:36 +0100 Subject: [PATCH 080/135] Changed getValueForAttributeFromLiElements function. #10 --- .../java/parser/scrapper/GumtreeAnnouncementParser.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java index f6af709..bd58308 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java @@ -312,10 +312,11 @@ public String parseFurnishings(Element furnishingsElement) { private String getValueForAttributeFromLiElements(String attributeName, Elements liElements) { if (liElements != null) { - for (Element singleLiElement : liElements) { - String text = singleLiElement.selectFirst(".attribute > .name").text(); + Elements attributes = liElements.select(".attribute"); + for (Element singleAttributeElement : attributes) { + String text = singleAttributeElement.selectFirst(".name").text(); if (text.equals(attributeName)) { - return singleLiElement.selectFirst(".attribute > .value").text(); + return singleAttributeElement.selectFirst(".value").text(); } } } From 9e3db9411069138b3b3aac935bf9e5798a8bf085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 9 Dec 2018 23:35:54 +0100 Subject: [PATCH 081/135] Implementation for simple page parsing, update of loggers. #10 --- ...nnouncementProcessorParserApplication.java | 21 +++++++--------- .../scrapper/GumtreeAnnouncementParser.java | 4 ++-- .../java/parser/walker/GumtreePageWalker.java | 3 +++ .../parser/walker/helpers/GumtreeHelper.java | 11 +++++++++ .../parser/walker/helpers/ProviderHelper.java | 15 ++++++++++++ .../walker/states/FetchRegistryState.java | 18 ++++++++++---- .../parser/walker/states/FindUrlState.java | 17 +++++++++---- .../walker/states/ProcessPageState.java | 24 +++++++++++++++---- .../parser/walker/states/WalkerState.java | 7 +++++- 9 files changed, 93 insertions(+), 27 deletions(-) diff --git a/announcement-processor-parser/src/main/java/parser/AnnouncementProcessorParserApplication.java b/announcement-processor-parser/src/main/java/parser/AnnouncementProcessorParserApplication.java index 2218fbd..f16e55f 100644 --- a/announcement-processor-parser/src/main/java/parser/AnnouncementProcessorParserApplication.java +++ b/announcement-processor-parser/src/main/java/parser/AnnouncementProcessorParserApplication.java @@ -1,21 +1,18 @@ package parser; -import lombok.extern.java.Log; -import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import parser.walker.PageWalker; -@Log +@EnableJpaRepositories @SpringBootApplication -@EnableScheduling -public class AnnouncementProcessorParserApplication implements CommandLineRunner { - public static void main(String[] args) { - SpringApplication app = new SpringApplication(AnnouncementProcessorParserApplication.class); - app.run(args); - } +public class AnnouncementProcessorParserApplication { - @Override - public void run(String... args) { + public static void main(String[] args) { + ConfigurableApplicationContext run = SpringApplication.run(AnnouncementProcessorParserApplication.class, args); + PageWalker pageWalker = (PageWalker) run.getBean("gumtreePageWalker"); + pageWalker.walk("www.example.com"); } } diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java index bd58308..d5ad9a0 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java @@ -24,8 +24,6 @@ public class GumtreeAnnouncementParser extends AnnouncementParser implements Sin public Announcement parsePage(String url) { try { Document pageContent = getPageContent(url); - log.info("Received content from url: " + url); - Element details = pageContent.selectFirst(".vip-details"); return Announcement.builder() @@ -48,6 +46,8 @@ public Announcement parsePage(String url) { .build(); } catch (IOException e) { throw new GumtreePageParseException("Cannot parser announcement from url: " + url); + } finally { + log.info("Announcement from page: " + url + " has been parsed"); } } diff --git a/announcement-processor-parser/src/main/java/parser/walker/GumtreePageWalker.java b/announcement-processor-parser/src/main/java/parser/walker/GumtreePageWalker.java index 5bd7a35..d2fa7d4 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/GumtreePageWalker.java +++ b/announcement-processor-parser/src/main/java/parser/walker/GumtreePageWalker.java @@ -1,5 +1,6 @@ package parser.walker; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import parser.scrapper.AnnouncementParser; @@ -7,6 +8,7 @@ import parser.walker.states.StopState; import parser.walker.states.WalkerState; +@Slf4j @Service public class GumtreePageWalker extends PageWalker { private AnnouncementParser announcementParser; @@ -24,5 +26,6 @@ public void walk(String startPageUrl) { while(walkerState != null && !(walkerState instanceof StopState)){ walkerState = walkerState.run(); } + log.info("GumtreePageWalker has parsed all pages"); } } diff --git a/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java b/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java index 59c0d17..de5e4f5 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java +++ b/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java @@ -13,9 +13,11 @@ import java.time.LocalDateTime; import java.util.Comparator; import java.util.HashMap; +import java.util.List; import java.util.Optional; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; @Slf4j @Component @@ -73,6 +75,15 @@ public Optional> findPageWithAnnouncement(ParsingInfo pa return Optional.ofNullable(returnValues); } + @Override + public List getUrlsToParse(Document document, int divNumber) { + log.info("Geeting urls to parse"); + return getElementsWithDataFromPage(document) + .stream() + .limit(divNumber).map(this::getPageUrlFromElement) + .collect(Collectors.toList()); + } + private LocalDate getEarliestDateOnAnnouncementPage(Document scannedPage) { Elements elementsWithData = getElementsWithDataFromPage(scannedPage); Optional first = elementsWithData.stream() diff --git a/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java b/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java index 18db303..c1a4cbd 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java +++ b/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java @@ -10,6 +10,7 @@ import java.io.IOException; import java.util.HashMap; +import java.util.List; import java.util.Optional; @Component @@ -31,6 +32,8 @@ Document getPageAsDocumentFromUrl(String url) throws IOException { public abstract Optional> findPageWithAnnouncement(ParsingInfo parsingInfoToFind); + public abstract List getUrlsToParse(Document document, int divNumber); + abstract HashMap findAnnouncementUrlOnPage(Document page, String url); abstract Elements getElementsWithDataFromPage(Document page); @@ -40,4 +43,16 @@ Document getPageAsDocumentFromUrl(String url) throws IOException { abstract String getNextPageUrl(String previousUrl, int nextPageNumber); abstract int getNumberOfTotalPages(Document scannedPage); + + public String getActualPageURL() { + return actualPageURL; + } + + public int getActualPageURLNumber() { + return actualPageURLNumber; + } + + public ParsingInfo getLastParsedAnnouncement() { + return lastParsedAnnouncement; + } } diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/FetchRegistryState.java b/announcement-processor-parser/src/main/java/parser/walker/states/FetchRegistryState.java index 3974047..89e9cf0 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/states/FetchRegistryState.java +++ b/announcement-processor-parser/src/main/java/parser/walker/states/FetchRegistryState.java @@ -1,26 +1,36 @@ package parser.walker.states; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; import parser.registry.ParsingInfo; +import parser.scrapper.AnnouncementParser; import parser.walker.helpers.ProviderHelper; import java.util.Optional; +@Slf4j @Component @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) public class FetchRegistryState extends WalkerState { - public FetchRegistryState(ProviderHelper providerHelper) { - super(providerHelper); + public FetchRegistryState(ProviderHelper providerHelper, AnnouncementParser announcementParser) { + super(providerHelper, announcementParser); } @Override public WalkerState run() { Optional parsingInfoToFind = providerHelper.fetchLatestRecord(); + + if (parsingInfoToFind.isPresent()) { + log.info("Url fetched: " + parsingInfoToFind.get().getUrl()); + } else { + log.info("Url cannot be fetched."); + } + return parsingInfoToFind - .map(parsingInfo -> new FindUrlState(providerHelper, parsingInfo)) - .orElseGet(() -> new FindUrlState(providerHelper, null)); + .map(parsingInfo -> new FindUrlState(providerHelper, announcementParser, parsingInfo)) + .orElseGet(() -> new FindUrlState(providerHelper, announcementParser, null)); } } \ No newline at end of file diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/FindUrlState.java b/announcement-processor-parser/src/main/java/parser/walker/states/FindUrlState.java index a3b7ec1..5334e4d 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/states/FindUrlState.java +++ b/announcement-processor-parser/src/main/java/parser/walker/states/FindUrlState.java @@ -1,17 +1,20 @@ package parser.walker.states; +import lombok.extern.slf4j.Slf4j; import parser.registry.ParsingInfo; +import parser.scrapper.AnnouncementParser; import parser.walker.helpers.ProviderHelper; import java.util.HashMap; import java.util.Optional; +@Slf4j public class FindUrlState extends WalkerState { private ParsingInfo parsingInfoToFind; - FindUrlState(ProviderHelper providerHelper, ParsingInfo parsingInfoToFind) { - super(providerHelper); + FindUrlState(ProviderHelper providerHelper, AnnouncementParser announcementParser, ParsingInfo parsingInfoToFind) { + super(providerHelper, announcementParser); this.parsingInfoToFind = parsingInfoToFind; } @@ -19,8 +22,14 @@ public class FindUrlState extends WalkerState { public WalkerState run() { Optional> pageWithAnnouncement = providerHelper.findPageWithAnnouncement(parsingInfoToFind); + if (pageWithAnnouncement.isPresent()) { + log.info("Page has been found on page number: " + providerHelper.getActualPageURLNumber() + ". Go to ProcessPageState"); + } else { + log.info("Page not found. Go to FetchRegistryState"); + } + return pageWithAnnouncement - .map(stringObjectHashMap -> new ProcessPageState(providerHelper, stringObjectHashMap)) - .orElseGet(() -> new FetchRegistryState(providerHelper)); + .map(stringObjectHashMap -> new ProcessPageState(providerHelper, announcementParser, stringObjectHashMap)) + .orElseGet(() -> new FetchRegistryState(providerHelper, announcementParser)); } } diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java b/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java index b05a313..5a145d6 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java +++ b/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java @@ -1,25 +1,41 @@ package parser.walker.states; +import lombok.extern.slf4j.Slf4j; +import org.jsoup.nodes.Document; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; +import parser.Announcement; +import parser.scrapper.AnnouncementParser; import parser.walker.helpers.ProviderHelper; import java.util.HashMap; +import java.util.List; +@Slf4j @Component @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) public class ProcessPageState extends WalkerState { - private HashMap pageWithAnnouncement; - public ProcessPageState(ProviderHelper providerHelper, HashMap pageWithAnnouncement) { - super(providerHelper); + public ProcessPageState(ProviderHelper providerHelper, AnnouncementParser announcementParser, HashMap pageWithAnnouncement) { + super(providerHelper, announcementParser); this.pageWithAnnouncement = pageWithAnnouncement; } @Override public WalkerState run() { - return null; + Document pageDocument = (Document) pageWithAnnouncement.get("pageDocument"); + int divNumber = (int) pageWithAnnouncement.get("divNumber"); + + List urlsToParse = providerHelper.getUrlsToParse(pageDocument, divNumber); + for (String urlToParse : urlsToParse) { + Announcement announcement = announcementParser.parsePage(urlToParse); + // TODO: Save record to DB_PARSER with valid counter + } + + log.info("Parsing urls from page " + providerHelper.getActualPageURLNumber() + " done. Go to VisibleAfterReloadState"); + + return new VisibleAfterReloadState(providerHelper); } } diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/WalkerState.java b/announcement-processor-parser/src/main/java/parser/walker/states/WalkerState.java index 3b8a10a..55d0587 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/states/WalkerState.java +++ b/announcement-processor-parser/src/main/java/parser/walker/states/WalkerState.java @@ -14,9 +14,14 @@ public WalkerState(ProviderHelper providerHelper) { this.providerHelper = providerHelper; } + public WalkerState(ProviderHelper providerHelper, AnnouncementParser announcementParser) { + this.providerHelper = providerHelper; + this.announcementParser = announcementParser; + } + public abstract WalkerState run(); public void setAnnouncementParser(GumtreeAnnouncementParser gumtreeAnnouncementParser){ this.announcementParser = gumtreeAnnouncementParser; - }; + } } From 27f4c8cb1c96eeb2742e861043cabdceb0818f26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Mon, 10 Dec 2018 14:05:00 +0100 Subject: [PATCH 082/135] Test for GumtreeHelper. #10 --- .../parser/walker/helpers/GumtreeHelper.java | 8 +- .../src/test/java/parser/ReaderUtil.java | 12 + .../walker/helpers/GumtreeHelperTest.java | 252 + .../gumtree/walker/announcements-list.html | 5847 +++++++++++++++++ 4 files changed, 6115 insertions(+), 4 deletions(-) create mode 100644 announcement-processor-parser/src/test/java/parser/walker/helpers/GumtreeHelperTest.java create mode 100644 announcement-processor-parser/src/test/resources/gumtree/walker/announcements-list.html diff --git a/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java b/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java index de5e4f5..5dd4244 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java +++ b/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java @@ -84,17 +84,17 @@ public List getUrlsToParse(Document document, int divNumber) { .collect(Collectors.toList()); } - private LocalDate getEarliestDateOnAnnouncementPage(Document scannedPage) { + LocalDate getEarliestDateOnAnnouncementPage(Document scannedPage) { Elements elementsWithData = getElementsWithDataFromPage(scannedPage); + LocalDateTime actualDateTime = LocalDateTime.now(); Optional first = elementsWithData.stream() .map(element -> element.selectFirst(".creation-date").text()) - .map(this::convertStringToLocalDate) + .map(date -> convertStringToLocalDate(date, actualDateTime)) .min(Comparator.reverseOrder()); return first.orElse(null); } - private LocalDate convertStringToLocalDate(String date) { - LocalDateTime actualDateTime = LocalDateTime.now(); + LocalDate convertStringToLocalDate(String date, LocalDateTime actualDateTime) { LocalDate toReturn; if (date.contains("temu")) { diff --git a/announcement-processor-parser/src/test/java/parser/ReaderUtil.java b/announcement-processor-parser/src/test/java/parser/ReaderUtil.java index 55c4530..84bd89f 100644 --- a/announcement-processor-parser/src/test/java/parser/ReaderUtil.java +++ b/announcement-processor-parser/src/test/java/parser/ReaderUtil.java @@ -24,6 +24,18 @@ public static Document getDocumentToTest(String resourceName) { return spyDoc; } + public static Document getDocumentToTestWithBaseUrl(String resourceName, String baseUrl){ + Document spyDoc = null; + try { + String path = Paths.get(ReaderUtil.class.getResource(resourceName).toURI()).toString(); + String fileContent = ReaderUtil.readAllLinesFromFile(path); + spyDoc = Jsoup.parse(fileContent, baseUrl); + } catch (URISyntaxException e) { + e.printStackTrace(); + } + return spyDoc; + } + private static String readAllLinesFromFile(String filePath) { StringBuilder contentBuilder = new StringBuilder(); try (Stream stream = Files.lines(Paths.get(filePath), StandardCharsets.UTF_8)) { diff --git a/announcement-processor-parser/src/test/java/parser/walker/helpers/GumtreeHelperTest.java b/announcement-processor-parser/src/test/java/parser/walker/helpers/GumtreeHelperTest.java new file mode 100644 index 0000000..f790085 --- /dev/null +++ b/announcement-processor-parser/src/test/java/parser/walker/helpers/GumtreeHelperTest.java @@ -0,0 +1,252 @@ +package parser.walker.helpers; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import parser.ReaderUtil; +import parser.registry.ParsingInfoService; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.Month; +import java.util.HashMap; +import java.util.List; +import java.util.Optional; + +public class GumtreeHelperTest { + private GumtreeHelper gumtreeHelper; + + @Mock + private ParsingInfoService parsingInfoService; + + @Rule + public MockitoRule mockitoRule = MockitoJUnit.rule(); + + @Before + public void setUp() { + gumtreeHelper = new GumtreeHelper(parsingInfoService); + } + + @Test + public void findPageWithAnnouncementWhenParsingInfoIsNull() { + // given + + // when + Optional> pageWithAnnouncement = gumtreeHelper.findPageWithAnnouncement(null); + + // then + Assert.assertEquals(Optional.empty(), pageWithAnnouncement); + } + + @Test + public void findAnnouncementUrlOnPageIsNotNull() { + // given + String baseUrl = "https://www.gumtree.pl/s-mieszkania-i-domy-do-wynajecia/krakow/v1c9008l3200208p1"; + String resourcePath = "/gumtree/walker/announcements-list.html"; + Document documentToTest = ReaderUtil.getDocumentToTestWithBaseUrl(resourcePath, baseUrl); + + // when + HashMap announcementUrlOnPage = gumtreeHelper.findAnnouncementUrlOnPage(documentToTest, + "https://www.gumtree.pl/a-mieszkania-i-domy-do-wynajecia/krakow/przytulna-kawalerka-do-wynaj%C4%99cia/1003754341000911379548809"); + + // then + Assert.assertNotNull(announcementUrlOnPage); + } + + @Test + public void findAnnouncementUrlOnPageIsNull() { + // given + String baseUrl = "https://www.gumtree.pl/s-mieszkania-i-domy-do-wynajecia/krakow/v1c9008l3200208p1"; + String resourcePath = "/gumtree/walker/announcements-list.html"; + Document documentToTest = ReaderUtil.getDocumentToTestWithBaseUrl(resourcePath, baseUrl); + + // when + HashMap announcementUrlOnPage = gumtreeHelper.findAnnouncementUrlOnPage(documentToTest, + "https://www.gumtree.pl/a-mieszkania-i-domy-do-wynajecia/krakow/123213213dsadasdsa9"); + + // then + Assert.assertNull(announcementUrlOnPage); + } + + @Test + public void getUrlsToParse() { + // given + String resourcePath = "/gumtree/walker/announcements-list.html"; + Document documentToTest = ReaderUtil.getDocumentToTest(resourcePath); + + // when + List urlsToParse = gumtreeHelper.getUrlsToParse(documentToTest, 4); + + // then + Assert.assertEquals(4, urlsToParse.size()); + } + + @Test + public void getEarliestDateOnAnnouncementPage() { + // given + String resourcePath = "/gumtree/walker/announcements-list.html"; + Document documentToTest = ReaderUtil.getDocumentToTest(resourcePath); + + // when + LocalDate returnedDate = gumtreeHelper.getEarliestDateOnAnnouncementPage(documentToTest); + + // then + Assert.assertEquals(Month.DECEMBER, returnedDate.getMonth()); + Assert.assertEquals(10, returnedDate.getDayOfMonth()); + } + + @Test + public void fetchLatestRecord() { + // given + + // when + gumtreeHelper.fetchLatestRecord(); + + // then + Mockito.verify(parsingInfoService, Mockito.times(1)) + .fetchLastRecordForProvider(Mockito.anyString(), Mockito.any()); + } + + @Test + public void convertStringToLocalDateWhenMinutesAgoWithDayBreak() { + // given + LocalDateTime localDateTime = LocalDateTime.of(2018, 12, 10, 0, 20); + String date = "30 min temu"; + + // when + LocalDate returnedDate = gumtreeHelper.convertStringToLocalDate(date, localDateTime); + + // then + Assert.assertEquals(LocalDate.of(2018, 12, 9), returnedDate); + } + + @Test + public void convertStringToLocalDateWhenMinutesAgoWithoutDayBreak() { + // given + LocalDateTime localDateTime = LocalDateTime.of(2018, 12, 10, 10, 48); + String date = "11 min temu"; + + // when + LocalDate returnedDate = gumtreeHelper.convertStringToLocalDate(date, localDateTime); + + // then + Assert.assertEquals(LocalDate.of(2018, 12, 10), returnedDate); + } + + @Test + public void convertStringToLocalDateWhenHoursAgoWithDayBreak() { + // given + LocalDateTime localDateTime = LocalDateTime.of(2018, 12, 10, 7, 30); + String date = "7 godz temu"; + + // when + LocalDate returnedDate = gumtreeHelper.convertStringToLocalDate(date, localDateTime); + + // then + Assert.assertEquals(LocalDate.of(2018, 12, 9), returnedDate); + } + + @Test + public void convertStringToLocalDateWhenHoursAgoWithoutDayBreak() { + // given + LocalDateTime localDateTime = LocalDateTime.of(2018, 12, 10, 8, 30); + String date = "7 godz temu"; + + // when + LocalDate returnedDate = gumtreeHelper.convertStringToLocalDate(date, localDateTime); + + // then + Assert.assertEquals(LocalDate.of(2018, 12, 10), returnedDate); + } + + @Test + public void convertStringToLocalDateWhenDateSpecified() { + // given + LocalDateTime localDateTime = LocalDateTime.of(2018, 12, 10, 8, 30); + String date = "4-12"; + + // when + LocalDate returnedDate = gumtreeHelper.convertStringToLocalDate(date, localDateTime); + + // then + Assert.assertEquals(LocalDate.of(2018, 12, 4), returnedDate); + } + + @Test + public void getPageUrlFromElement() { + // given + String html = "
\n" + "\t\t\t\t\t\t\t\n" + + "\t\t\t\t\t\t\t Nowe kawalerka Dębniki bezpośrednio (english / deutsch)\n" + + "\t\t\t\t\t\t\t\n" + "\t\t\t\t\t\t
"; + String baseUrl = "https://www.gumtree.pl/s-mieszkania-i-domy-do-wynajecia/krakow/v1c9008l3200208p1"; + Element element = Jsoup.parse(html, baseUrl).selectFirst("div"); + String expectedPageUrl = "https://www.gumtree.pl/a-mieszkania-i-domy-do-wynajecia/krakow/nowe-kawalerka-d%C4%99bniki-bezpo%C5%9Brednio-english-deutsch/1003692394800910479699309"; + + // when + String returnedPageUrl = gumtreeHelper.getPageUrlFromElement(element); + + // then + Assert.assertEquals(expectedPageUrl, returnedPageUrl); + } + + @Test + public void getNumberOfTotalPagesWithGoodPattern() { + // given + String html = ""; + String baseUrl = "https://www.gumtree.pl/s-mieszkania-i-domy-do-wynajecia/krakow/page-2/v1c9008l3200208p2"; + Document parsedHtml = Jsoup.parse(html, baseUrl); + + // when + int numberOfTotalPages = gumtreeHelper.getNumberOfTotalPages(parsedHtml); + + // then + Assert.assertEquals(85, numberOfTotalPages); + } + + @Test + public void getNumberOfTotalPagesIfNoPattern() { + // given + String html = ""; + String baseUrl = "https://www.gumtree.pl/s-mieszkania-i-domy-do-wynajecia/krakow/v1c9008l3200208p1"; + Document parsedHtml = Jsoup.parse(html, baseUrl); + + // when + int numberOfTotalPages = gumtreeHelper.getNumberOfTotalPages(parsedHtml); + + // then + Assert.assertEquals(0, numberOfTotalPages); + } + + @Test + public void getNextPageUrlWhenPreviousUrlNull() { + // given + String BASE_ANNOUNCEMENTS_URL = "https://www.gumtree.pl/s-mieszkania-i-domy-do-wynajecia/krakow/v1c9008l3200208p1"; + + // when + String returnedUrl = gumtreeHelper.getNextPageUrl(null, 2); + + // then + Assert.assertEquals(BASE_ANNOUNCEMENTS_URL, returnedUrl); + } + + @Test + public void getNextPageUrlWhenPreviousUrlNotNull() { + // given + String previousUrl = "https://www.gumtree.pl/s-mieszkania-i-domy-do-wynajecia/krakow/page-2/v1c9008l3200208p2"; + String expectedUrl = "https://www.gumtree.pl/s-mieszkania-i-domy-do-wynajecia/krakow/page-3/v1c9008l3200208p3"; + + // when + String returnedUrl = gumtreeHelper.getNextPageUrl(previousUrl, 3); + + // then + Assert.assertEquals(expectedUrl, returnedUrl); + } +} diff --git a/announcement-processor-parser/src/test/resources/gumtree/walker/announcements-list.html b/announcement-processor-parser/src/test/resources/gumtree/walker/announcements-list.html new file mode 100644 index 0000000..5ba5ccd --- /dev/null +++ b/announcement-processor-parser/src/test/resources/gumtree/walker/announcements-list.html @@ -0,0 +1,5847 @@ + + Mieszkania do wynajęcia – domy do wynajęcia – Kraków | Gumtree + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+
+ + + + + + + +
+
+ + + + + + + +
+ + + + Dodaj ogłoszenie + + + + +
+ + + + + + + + + +
+ +
+ + + + + + + + + + + +
+
+
+
+
+ + + +
+
+
+
+ + Dodaj ogłoszenie + +
+
+
+
+
+ + + +
+
+
+ + +
+
+ + + +
+ +
+ + + + +
+ + + + + + + + + +
+ + + + + +
+ + + + + + + +
+ + + + + +
+ +
+ + + +
+ + + + + + + + + + + + + + + +
+ + +
+
+
+ + + + + + + + + + + +
+ + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + + + + + + Małopolskie + + + + + + + + + + + + + + + + + + Nieruchomości + + + + + + + + + + + + + +

+ + + + + mieszkania i domy do wynajęcia + + | Kraków + + + + + + +

+ + + + | + + + + + + Aktywnych ogłoszeń: 6 635 + + +
+ +
+ + + + + + + +
+
+
+ + + + + + + + + + + + + + + + + + + + + +
+ + +
+
+ + + Zmień filtry + + + + +
+
+ + + + +
+ +
+ + Lista + + + Galeria + +
+ +
+ + +
    +
  • +
    + + Sortuj według + +
    Najnowsze + + +
    + +
    + +
      + +
    • + + Najnowsze +
      • +
    • + + Najniższa cena +
      • +
    • + + Najwyższa cena +
      • + +
    +
    • +
+ +
+ + + 0 + + +
+
+ Zachowane ogłoszenia + (0) + + Pokaż wszystkie + | + Usuń wszystkie + +
+ +
    +
+
+
+
+ + + Kliknij gwiazdkę w ulubionym ogłoszeniu, aby dodać ofertę do listy Zachowanych ogłoszeń + + lub, l Zaloguj się Aby zobaczyć oferty dodane do swojej listy Zachowanych ogłoszeń + + +
+
+
+ + + Zapisz + + + + +
+
+
+
+
+ +
+
+ + + + + +
+ +
    +
  • +
    + + Sortuj według + +
    Najnowsze + + +
    + +
    + +
      + +
    • + + Najnowsze +
      • +
    • + + Najniższa cena +
      • +
    • + + Najwyższa cena +
      • + +
    +
    • +
+ + +
+ + + + + + + + + + + + + +
    + + + + + + +
  • +
    + Kategoria + + mieszkania i domy do wynajęcia + + +
    + + +
    • + + + + + + + + +
  • +
    + Lokalizacja + + Kraków + + +
    + + +
    • + + + + + + + + + + + + +
  • +
    + + + - + + + +
    +
    • + + + + + + + + + +
  • +
    + Rodzaj oferty + + Wszystkie + + +
    +
      + + + +
    • + + Cena + + 6424 +
      • + + + + + + + + + +
    • + + Proszę o kontakt + + 210 +
      • + + + + + +
    • + + Wymiana/zamiana + + 1 +
      • + + + + + + + + +
    • + + + Pokaż więcej + Pokaż mniej +
      • + +
    +
    • + + + + + + + + + + + + + + + + + +
  • +
    + Rodzaj nieruchomości + + Wszystkie + + +
    +
      + + + +
    • + + Mieszkanie + + 6212 +
      • + + + + + +
    • + + Dom + + 347 +
      • + + + + + +
    • + + Inne + + 23 +
      • + + + + +
    • + + + Pokaż więcej + Pokaż mniej +
      • + +
    +
    • + + + + + + + + + + + + + + + + +
  • +
    + Liczba pokoi + + Wszystkie + + +
    +
      + + + +
    • + + Kawalerka lub garsoniera + + 1211 +
      • + + + + + +
    • + + 2 pokoje + + 3239 +
      • + + + + + +
    • + + 3 pokoje + + 1241 +
      • + + + + + +
    • + + 4 pokoje + + 348 +
      • + + + + + +
    • + + 5 pokoi + + 145 +
      • + + + + + +
    • + + 6 lub więcej pokoi + + 184 +
      • + + + + +
    • + + + Pokaż więcej + Pokaż mniej +
      • + +
    +
    • + + + + + + + + + + + + + + +
  • +
    + Liczba łazienek + + Wszystkie + + +
    +
      + + + +
    • + + 1 łazienka + + 3297 +
      • + + + + + +
    • + + 2 łazienki + + 344 +
      • + + + + + +
    • + + 3 łazienki + + 47 +
      • + + + + + +
    • + + 4 lub więcej łazienek + + 27 +
      • + + + + +
    • + + + Pokaż więcej + Pokaż mniej +
      • + +
    +
    • + + + + + + + + + + + + + + + +
  • +
    + Parking + + Wszystkie + + +
    +
      + + + +
    • + + Garaż + + 632 +
      • + + + + + +
    • + + Kryty + + 288 +
      • + + + + + +
    • + + Ulica + + 1651 +
      • + + + + + +
    • + + Brak + + 121 +
      • + + + + +
    • + + + Pokaż więcej + Pokaż mniej +
      • + +
    +
    • + + + + + + + + + + + + + + + +
  • +
    + Palący + + Wszystkie + + +
    +
      + + + +
    • + + Tak + + 221 +
      • + + + + + +
    • + + Nie + + 1109 +
      • + + + + +
    • + + + Pokaż więcej + Pokaż mniej +
      • + +
    +
    • + + + + + + + + + + + + + + +
  • +
    + Przyjazne zwierzakom + + Wszystkie + + +
    +
      + + + +
    • + + Tak + + 614 +
      • + + + + + +
    • + + Nie + + 852 +
      • + + + + +
    • + + + Pokaż więcej + Pokaż mniej +
      • + +
    +
    • + + + + + + + + + + + + + + + + +
  • +
    + Do wynajęcia przez + + Wszystkie + + +
    +
      + + + +
    • + + Właściciel + + 1690 +
      • + + + + + +
    • + + Agencja + + 4886 +
      • + + + + +
    • + + + Pokaż więcej + Pokaż mniej +
      • + +
    +
    • + + + + + + + + + + + + + + + + + + +
+
+ + + + + + + + + + + + + + + +
+
+ + + + + Powiadomienia Gumtree +
+ +
+

Otrzymuj codziennie e-mail z najnowszymi ogłoszeniami, które Cię interesują.

+
+ +
+
+ Filtry powiadomienia: + + + + + mieszkania i domy do wynajęcia, Kraków + + + + +
+ + + + +
+ Klikając “Dodaj powiadomienie”, wyrażasz zgodę na nasze Zasady korzystania i Politykę prywatności oraz zgadzasz się na otrzymywanie naszych newsletterów i ofert promocyjnych. +
+
+ + +
+ + +
+ + + + + + +
+
+
+ + + + + +
+ + + +
Linki sponsorowane
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Ogłoszenia wyróżnione + + Zobacz wszystkie + +
+ +
+
    + + + +
  • +
    +
    + +
    + + Klimatyzowana kawalerka - Nowe budownictwo - Bajeczna Kraków z Krakow, zobacz zdjęcie + + +
    Zdjęć: 7
    + +
    + +
    +
    +
    + + Klimatyzowana kawalerka - Nowe budownictwo - Bajeczna Kraków + +
    + + +
    Budynek znajduje się na zamkniętym, moniotorowanym osiedlu. Mieszkanie znajduje +się na 3 piętrze bloku, w którym znajdują się 2 windy. + + + +Mieszkanie typu Studio znajduje się w pięciokondygnacyjnym +nowoczesnym budynku na 3 piętrze. Salon połączony jest z aneksem kuchennym i +małym pokojem oraz osobną łazienką. + +Kuchnia: Wyposażona w sprzęt AGD o podwyższony standardzie. W +aneksie kuchennym ...…
    + + +
    + + + + + +
    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + 1 700 zł + + + + + + + + + + + + +
    + + +
    + + 9-11 +
    + +
    + +
    + + mieszkania i domy do wynajęcia , Kraków + +
    + + +
    + + + +
    +
    + +
    + + +
    + +
    + +
    +
    + + +
    +
    • + + + + + +
  • +
    +
    + +
    + + Sudio Krakowskie do wynajęcia. Nowy, luksusowy, klimatyzowany mikro apartament w Wegierska Residence z Krakow, zobacz zdjęcie + + +
    Zdjęć: 12
    + +
    + +
    +
    +
    + + Sudio Krakowskie do wynajęcia. Nowy, luksusowy, klimatyzowany mikro apartament w Wegierska Residence + +
    + + +
    Zamieszkaj w prestiżowej Węgierska Residence w sercu Starego +Podgórza.Do wynajęcia bez pośrednictwa, praktycznie nowe, +klimatyzowane studio na Starym Podgórzu przy ul. Węgierskiej tuż obok Rynku +Podgórskiego. Doskonała propozycja dla osób ceniących sobie wysoki standard +oraz mieszkanie blisko centrum.Mini apartament " Studio Krakowskie "Studio na 2 piętrze ( winda ) składa się z: pokoju z anek ...…
    + + +
    + + + + + +
    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + 1 800 zł + + + + + + + + + + + + +
    + + +
    + + 30-11 +
    + +
    + +
    + + mieszkania i domy do wynajęcia , Kraków + +
    + + +
    + + + +
    +
    + +
    + + +
    + +
    + +
    +
    + + +
    +
    • + + + + + +
  • +
    +
    + +
    + + Wynajmę mieszkanie tarasy verona z Krakow, zobacz zdjęcie + + +
    Zdjęć: 9
    + +
    + +
    +
    +
    + + Wynajmę mieszkanie tarasy verona + +
    + + +
    Wynajmę od zaraz w pełni wyposażone mieszkanie znajdujące się przy ulicy kołowej w Krakowie na drugim pietrze. Jest to jedna z inwestycji TARASY VERONA. + +Obok budynku znajdują się biurowce Galileo, Newton, Edison, park Jordana oraz krakowskie błonia, a także kompleks akademików studenckich AGH. W okolicy znajdują się również uczelnie wyższe takie jak np. WSZiB oraz AGH. + +Apartament o powier ...…
    + + +
    + + + + + +
    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + 2 500 zł + + + + + + + + + + + + +
    + + +
    + + 18-11 +
    + +
    + +
    + + mieszkania i domy do wynajęcia , Kraków + +
    + + +
    + + + +
    +
    + +
    + + +
    + +
    + +
    +
    + + +
    +
    • + + + +
+
+ + + + + + +
+
+
+ + + + + + +
+ + + + Ogłoszenia + + + +
+ +
+
    + + + +
  • +
    +
    + +
    + + Mieszkanie Kraków 33m2 (nr: 10319) z Krakow, zobacz zdjęcie + + +
    Zdjęć: 9
    + +
    + +
    +
    +
    + + Mieszkanie Kraków 33m2 (nr: 10319) + +
    + + +
    Apartament typu studio w centrum KrakowaMINT PROPERTY prezentuje na wynajem jeden z kilku apartamentów typu studio o pow. 33m2 położonych w nowoczesnym budynku z 2002, wyposażonym w windę, 24-godzinną ochronę oraz parking podziemny.Apartament tej wielkości składa się z pokoju połączonego z aneksem kuchennym, łazienki z kabiną prysznicową oraz pralką.Mieszkanie jest w pełni wyposażone, umeblowane, ...…
    + + +
    + + + + + +
    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + 1 800 zł + + + + + + + + + + + + +
    + + +
    + + 3 min temu +
    + +
    + +
    + + mieszkania i domy do wynajęcia , Kraków + +
    + + +
    + + + +
    +
    + +
    + + +
    + +
    + +
    +
    + + +
    +
    • + + + + + + + + + + + + +
  • +
    +
    + +
    + + 57 M2 *** UL. BIAŁOPRĄDNICKA *** PRĄDNIK BIAŁY *** 2 POKOJE z Krakow, zobacz zdjęcie + + +
    Zdjęć: 10
    + +
    + +
    +
    +
    + + 57 M2 *** UL. BIAŁOPRĄDNICKA *** PRĄDNIK BIAŁY *** 2 POKOJE + +
    + + +
    DWUPOKOJOWE! *** Do wynajęcia mieszkanie na Prądniku Białym przy ul. Białoprądnickiej, znajduje się ono na 2 piętrze. Mieszkanie o powierzchni 57 m2, znajdują się w nim: dwa odrębne pokoje, kuchnia, łazienka, przedpokój, dwa balkony. Do mieszkania przynależy jedno miejsce parkingowe ogrodzone z bramą otwieraną na pilota. +Mieszkanie wyposażone w :lodówkę, pralkę, zmywarkę, piekarnik, płytę grzewcz ...…
    + + +
    + + + + + +
    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + 1 600 zł + + + + + + + + + + + + +
    + + +
    + + 5 min temu +
    + +
    + +
    + + mieszkania i domy do wynajęcia , Kraków + +
    + + +
    + + + +
    +
    + +
    + + +
    + +
    + +
    +
    + + +
    +
    • + + + + + + + + + + + + +
  • +
    +
    + +
    + + 77 M2 *** UL. BUJWIDA *** GRZEGÓRZKI *** 2 POKOJE z Krakow, zobacz zdjęcie + + +
    Zdjęć: 12
    + +
    + +
    +
    +
    + + 77 M2 *** UL. BUJWIDA *** GRZEGÓRZKI *** 2 POKOJE + +
    + + +
    DWUPOKOJOWE! *** Do wynajęcia mieszkanie ok. 77 m2 w centrum Krakowa - Ul. Bujwida, wysoki parter.Wspaniała lokalizacja:- ok. 10 min spacerem do rynku- 15 minut piechotą na Kazimierz.- 3 min do przystanku Hala Targowa (tramwaje i autobusy),- plac targowy- bardzo blisko Uniwersytet Ekonomiczny oraz szpitale.W mieszkaniu 2 pokoje, kuchnia z balkonem, osobno łazienka i WC. Mieszkanie umeblowane i wyp ...…
    + + +
    + + + + + +
    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + 2 200 zł + + + + + + + + + + + + +
    + + +
    + + 7 min temu +
    + +
    + +
    + + mieszkania i domy do wynajęcia , Kraków + +
    + + +
    + + + +
    +
    + +
    + + +
    + +
    + +
    +
    + + +
    +
    • + + + + + + + + + + + + +
  • +
    +
    + +
    + + Mieszkanie Kraków 40m2 (nr: 10309) z Krakow, zobacz zdjęcie + + +
    Zdjęć: 8
    + +
    + +
    +
    +
    + + Mieszkanie Kraków 40m2 (nr: 10309) + +
    + + +
    Mint Property ma przyjemność zaprezentować mieszkanie na wynajem przy ulicy Prądnickiej. Mieszkanie znajduje się na trzecim piętrze jedenastopiętrowego nowoczesnego budynku z windą.Na powierzchni 40 m2 znajduję się: Salon z aneksem kuchennym SypialniaSypialnia II Łazienka BalkonMieszkanie jest w pełni umeblowane, cały teren jest monitorowany w modernistycznym budynku z miejscem postojowym w garażu ...…
    + + +
    + + + + + +
    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + 2 400 zł + + + + + + + + + + + + +
    + + +
    + + 7 min temu +
    + +
    + +
    + + mieszkania i domy do wynajęcia , Kraków + +
    + + +
    + + + +
    +
    + +
    + + +
    + +
    + +
    +
    + + +
    +
    • + + + + + + + + + + + + +
  • +
    +
    + +
    + + Piękne mieszkanie na wynajem, Apartamenty Novum! z Krakow, zobacz zdjęcie + + + +
    Zdjęć: 15
    + +
    + +
    +
    +
    + + Piękne mieszkanie na wynajem, Apartamenty Novum! + +
    + + +
    English version below. + +Do wynajęcia od zaraz mieszkanie w prestiżowej inwestycji - Apartamenty Novum. +Apartamenty Novum to nowoczesny kompleks mieszkalny, z świetnie zagospodarowanymi częściami wspólnymi. Znajdują się tu zielone alejki, oczka wodne, ławki, plac zabaw dla dzieci. + + +LOKALIZACJA +Ulica Rakowicka znajduje się w samym sercu Krakowa. Lokalizacja zapewnia doskonałą komunikację. W pob ...…
    + + +
    + + + + + +
    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + 2 300 zł + + + + + + + + + + + + +
    + + +
    + + 8 min temu +
    + +
    + +
    + + mieszkania i domy do wynajęcia , Kraków + +
    + + +
    + + + +
    +
    + +
    + + +
    + +
    + +
    +
    + + +
    +
    • + + + + + + + + + + + + +
  • +
    +
    + +
    + + UL. SZAFRANA ^^^ PODGÓRZE ^^^ 44 M2 +++ 2 POKOJE z Krakow, zobacz zdjęcie + + + +
    Zdjęć: 8
    + +
    + +
    +
    +
    + + UL. SZAFRANA ^^^ PODGÓRZE ^^^ 44 M2 +++ 2 POKOJE + +
    + + +
    UL. SZAFRANA *** Dwupokojowe mieszkanie o powierzchni 44 m2 zlokalizowane przy ulicy Szafrana na Podgórzu. +Apartament: +- bardzo jasny, z dużymi oknami +- 2 piętro (winda w budynku) +- 44 mkw +- salon z aneksem kuchennym, sypialnia, łazienka, balkon +- wysoki standard wykończenia +- pralka i zmywarka w mieszkaniu OPŁATY: 2000 PLN + 400 PLN (czynsz) + prądTEL ...…
    + + +
    + + + + + +
    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + 2 000 zł + + + + + + + + + + + + +
    + + +
    + + 9 min temu +
    + +
    + +
    + + mieszkania i domy do wynajęcia , Kraków + +
    + + +
    + + + +
    +
    + +
    + + +
    + +
    + +
    +
    + + +
    +
    • + + + + + + + + + + + + +
  • +
    +
    + +
    + + 3 pokojowe mieszkanie w okolicy Kleparza - 80m2 z Krakow, zobacz zdjęcie + + + +
    Zdjęć: 15
    + +
    + +
    +
    +
    + + 3 pokojowe mieszkanie w okolicy Kleparza - 80m2 + +
    + + +
    English version below. + +Do wynajęcia przestronne 3 pokojowe mieszkanie położone przy ulicy Wrocławskiej w dzielnicy Krowodrza. + +LOKALIZACJA: +Krowodrza to dzielnica, która ciągle się rozwija, a jej mieszkańcy mogą docenić szeroki wachlarz usług blisko domu. Inwestycja mieści się zaledwie 2 km od Rynku Głównego. W pobliżu można znaleźć m.in. najważniejsze ośrodki akademickie Krakowa, centra bizne ...…
    + + +
    + + + + + +
    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + 2 600 zł + + + + + + + + + + + + +
    + + +
    + + 9 min temu +
    + +
    + +
    + + mieszkania i domy do wynajęcia , Kraków + +
    + + +
    + + + +
    +
    + +
    + + +
    + +
    + +
    +
    + + +
    +
    • + + + + + + + + + + + + +
  • +
    +
    + +
    + + Mieszkanie Kraków 44m2 (nr: 10306) z Krakow, zobacz zdjęcie + + + +
    Zdjęć: 6
    + +
    + +
    +
    +
    + + Mieszkanie Kraków 44m2 (nr: 10306) + +
    + + +
    Mint Property ma przyjemność zaprezentować atrakcyjny dwupokojowy apartament w odrestaurowanej kamienicy przy ul. Borowickiej w Krakowie.Na mieszkanie o powierzchni 44 m2 mieszczące się na 3 piętrze 4-ro kondygnacyjnego budynku składa się sypialnia, salon z aneksem kuchennym i wyjściem na balkon, łazienka z prysznicem i obszerny korytarz. Właściciel wyraża chęć doposażenia mieszkania wg sugestii w ...…
    + + +
    + + + + + +
    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + 2 500 zł + + + + + + + + + + + + +
    + + +
    + + 9 min temu +
    + +
    + +
    + + mieszkania i domy do wynajęcia , Kraków + +
    + + +
    + + + +
    +
    + +
    + + +
    + +
    + +
    +
    + + +
    +
    • + + + + + + + + + + + + +
  • +
    +
    + +
    + + Dom 180 m2 - Centrum Zielonek - 100 m od Krakowa z Krakow, zobacz zdjęcie + + + +
    Zdjęć: 14
    + +
    + +
    +
    +
    + + Dom 180 m2 - Centrum Zielonek - 100 m od Krakowa + +
    + + +
    English version below. + +Do wynajęcie przestronny dom w zabudowie bliźniaczej w cichej okolicy nieobodal centrum Zielonek. + +LOKALIZACJA: +Dom znajduje się w centrum Zielonek, 3 minuty od granicy Krakowa. W najbliższej odległości wiele punków usługowych oraz handlowych. Kilka minut od nieruchomości przystanek autobusowy MPK. + + + +DOM: +Na 180 m2 powierzchni składają się: 5 sypialni, 2 łazienki, wc, s ...…
    + + +
    + + + + + +
    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + 3 000 zł + + + + + + + + + + + + +
    + + +
    + + 10 min temu +
    + +
    + +
    + + mieszkania i domy do wynajęcia , Kraków + +
    + + +
    + + + +
    +
    + +
    + + +
    + +
    + +
    +
    + + +
    +
    • + + + + + + + + +
    + + + + + + + + + + + +
  • +
    +
    + +
    + + przytulna kawalerka do wynajęcia z Krakow, zobacz zdjęcie + + + +
    Zdjęć: 5
    + +
    + +
    +
    +
    + + przytulna kawalerka do wynajęcia + +
    + + +
    Wynajmę słoneczną, umeblowaną +kawalerkę zlokalizowaną na osiedlu Azory idealną dla pary. +Aneks kuchenny w przedpokoju (nowe +szafki, blat, zlew, nowa płyta gazowa, nowa lodówka). + +Blok znajduję się zaraz przy pętli +Azory – 15 minut autobusem do centrum. +W mieszkaniu nowe okna, parkiet, +pralka, ciepła woda z MPEC. Możliwość oglądania mieszkania od +przyszłego tygodnia. Możliwość sporządzen ...…
    + + +
    + + + + + +
    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + 1 150 zł + + + + + + + + + + + + +
    + + +
    + + 11 min temu +
    + +
    + +
    + + mieszkania i domy do wynajęcia , Kraków + +
    + + +
    + + + +
    +
    + +
    + + +
    + +
    + +
    +
    + + +
    +
    • + + + + + + + + +
    +
    +
    + + + + + + + + + +
  • +
    +
    + +
    + + Krowodrza. Dwa nieprzechodnie pokoje z widokiem na Stare Miasto. z Krakow, zobacz zdjęcie + + + +
    Zdjęć: 6
    + +
    + +
    +
    +
    + + Krowodrza. Dwa nieprzechodnie pokoje z widokiem na Stare Miasto. + +
    + + +
    Prezentujemy Państwu do wynajęcia 2-pokojowe mieszkanie o powierzchni 37m2, składające się z:przedpokoju,2 oddzielnych pokoi,osobnej, jasnej kuchni,łazienki,balkonu typu loggia.Mieszkanie o powierzchni użytkowej 37m2 położone na ostatnim piętrze w ośmiopiętrowym bloku z lat 70.Stan mieszkania: Dobry – w całym mieszkaniu okna PCV. Ciepła woda z sieci miejskiej MPEC.Stan budynku: Dobry – ośmiopiętro ...…
    + + +
    + + + + + +
    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + 1 250 zł + + + + + + + + + + + + +
    + + +
    + + 11 min temu +
    + +
    + +
    + + mieszkania i domy do wynajęcia , Kraków + +
    + + +
    + + + +
    +
    + +
    + + +
    + +
    + +
    +
    + + +
    +
    • + + + + + + + + + + + + +
  • +
    +
    + +
    + + GARSONIERA 16 m2 MISTRZEJOWICE z Krakow, zobacz zdjęcie + + + +
    Zdjęć: 4
    + +
    + +
    +
    +
    + + GARSONIERA 16 m2 MISTRZEJOWICE + +
    + + +
    OFERTA BIURA NIERUCHOMOŚCIDo wynajęcia w pełni umeblowana garsoniera w bezczynszowym, prywatnym bloku w Mistrzejowicach (w pobliżu ul. Dobrego Pasterza -Multikina i Aqua Parku). Mieszkanie składa się z pokoju, jasnej kuchni, łazienki z WC , kabiną prysznicową oraz przedpokoju. Powierzchnia mieszkania ok. 16 m2. II piętro w budynku dwupiętrowym. +Mieszkanie jest bardzo przytulne, czyste i w peł ...…
    + + +
    + + + + + +
    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + 800 zł + + + + + + + + + + + + +
    + + +
    + + 12 min temu +
    + +
    + +
    + + mieszkania i domy do wynajęcia , Kraków + +
    + + +
    + + + +
    +
    + +
    + + +
    + +
    + +
    +
    + + +
    +
    • + + + + + + + + + + + + +
  • +
    +
    + +
    + + Apartamenty przy Bulwarze -dwa pokoje 2400 zł ! z Krakow, zobacz zdjęcie + + + +
    Zdjęć: 11
    + +
    + +
    +
    +
    + + Apartamenty przy Bulwarze -dwa pokoje 2400 zł ! + +
    + + +
    English version below. + +Do wynajęcia przestronny w świetnej lokalizacji apartament – Apartamenty przy Bulwarze, świetna okazja ! + +LOKALIZACJA: + +Apartament zlokalizowany jest w dzielnicy Grzegórzki, przy ulicy Masarskiej 15 w kompleksie mieszkaniowym Apartamenty Przy Bulwarze. Budynek mieści się w centrum Krakowa na styku dzielnic Kazimierza i Grzegórzki, 15 minut od Rynku Głównego, 2 km od dwor ...…
    + + +
    + + + + + +
    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + 2 400 zł + + + + + + + + + + + + +
    + + +
    + + 12 min temu +
    + +
    + +
    + + mieszkania i domy do wynajęcia , Kraków + +
    + + +
    + + + +
    +
    + +
    + + +
    + +
    + +
    +
    + + +
    +
    • + + + + + + + + + + + + +
  • +
    +
    + +
    + + Apartament 59 m2 - 2 pok. Rondo Mogilskie z Krakow, zobacz zdjęcie + + + +
    Zdjęć: 11
    + +
    + +
    +
    +
    + + Apartament 59 m2 - 2 pok. Rondo Mogilskie + +
    + + +
    English version below. + +Apartament do wynajęcia, nowocześnie urządzony z pięknym widokiem na centrum miasta - 7 piętro + +LOKALIZACJA: + +Apartament znajduje się w doskonałej lokalizacji w nowej inwestycji. Teren wokół osiedla strzeżony z łatwym dostępem do centrum miasta (do 5 minut), 3 minuty spacerem do przystanków komunikacji miejskiej, co gwarantuje świetny dojazd w każdy inny zakątek Krakowa. ...…
    + + +
    + + + + + +
    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + 2 700 zł + + + + + + + + + + + + +
    + + +
    + + 12 min temu +
    + +
    + +
    + + mieszkania i domy do wynajęcia , Kraków + +
    + + +
    + + + +
    +
    + +
    + + +
    + +
    + +
    +
    + + +
    +
    • + + + + + + + + + + + + +
  • +
    +
    + +
    + + Do wynajęcia Apartament 37 m2 - Rondo Mogilskie z Krakow, zobacz zdjęcie + + + +
    Zdjęć: 8
    + +
    + +
    +
    +
    + + Do wynajęcia Apartament 37 m2 - Rondo Mogilskie + +
    + + +
    English version below. + +Na wynajem Apartament 37 m2 w świetnej lokalizacji + +LOKALIZACJA: +Mieszkanie znajduje się w doskonałej lokalizacji w nowej inwestycji. łatwym dostępem do centrum miasta (do 5 minut), 3 minuty spacerem do przystanków komunikacji miejskiej, co gwarantuje świetny dojazd w każdy inny zakątek Krakowa. Wysoko rozwinięta infrastruktura handlowo – usługowa, sklepy, restauracje cz ...…
    + + +
    + + + + + +
    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + 2 100 zł + + + + + + + + + + + + +
    + + +
    + + 12 min temu +
    + +
    + +
    + + mieszkania i domy do wynajęcia , Kraków + +
    + + +
    + + + +
    +
    + +
    + + +
    + +
    + +
    +
    + + +
    +
    • + + + + + + + + + + + + + + +
  • +
    +
    + +
    + + Apartament z widokiem na Wawel - do wynajęcia 47m2 z Krakow, zobacz zdjęcie + + + +
    Zdjęć: 12
    + +
    + +
    +
    +
    + + Apartament z widokiem na Wawel - do wynajęcia 47m2 + +
    + + +
    English version below. + +Do wynajęcia Apartament 47 m2 - 2 pokojowe w doskonałej lokalizacji - Kapelanka + +LOKALIZACJA: + +Doskonała lokalizacja, która umożliwia bardzo szybkie dostanie się do centrum miasta (do 5 -10 minut, spacer na Stare Miasto do 20-25 minut). Kilka minut do tramwaju i autobusów co gwarantuje świetny dojazd w każdy zakątek Krakowa. Tuż obok centra biznesowe i biurowce takich f ...…
    + + +
    + + + + + +
    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + 2 200 zł + + + + + + + + + + + + +
    + + +
    + + 12 min temu +
    + +
    + +
    + + mieszkania i domy do wynajęcia , Kraków + +
    + + +
    + + + +
    +
    + +
    + + +
    + +
    + +
    +
    + + +
    +
    • + + + + + + + + + + + + +
  • +
    +
    + +
    + + Apartament w centrum - Wiślane Tarasy z Krakow, zobacz zdjęcie + + + +
    Zdjęć: 12
    + +
    + +
    +
    +
    + + Apartament w centrum - Wiślane Tarasy + +
    + + +
    English version below. + +OKAZJA! + +Do wynajęcia luksusowo wykończony apartament w inwestycji Wiślane Tarasy. + +LOKALIZACJA: + +Ul. Grzegórzecka na której znajduje się prezentowany apartament znajduje się nieopodal bulwarów wiślanych. W pobliżu znajdziemy nowo powstające biurowce, Galerię Kazimierz a także coraz bardziej popularny teatr Variete. Spacer na Kazimierz zajmuje 10 min. Pomimo bliskości r ...…
    + + +
    + + + + + +
    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + 4 100 zł + + + + + + + + + + + + +
    + + +
    + + 12 min temu +
    + +
    + +
    + + mieszkania i domy do wynajęcia , Kraków + +
    + + +
    + + + +
    +
    + +
    + + +
    + +
    + +
    +
    + + +
    +
    • + + + + + + + + + + + + +
  • +
    +
    + +
    + + Unikatowy apartament z pięknym tarasem-Eldorado z Krakow, zobacz zdjęcie + + + +
    Zdjęć: 14
    + +
    + +
    +
    +
    + + Unikatowy apartament z pięknym tarasem-Eldorado + +
    + + +
    English version below. + +OKAZJA! + +3-pok. apartament w inwestycji Eldorado + + +LOKALIZACJA: + +Apartament zlokalizowany jest w nowoczesnym ogrodzonym osiedlu z monitoringiem i stałą ochroną. W najbliższym zasięgu znajduje się Wydział Mechaniczny Politechniki Krakowskiej, AWF, główna siedziba firmy COMARCH, Muzeum Lotnictwa Polskiego, supermarket LIDL. Świetne połączenie komunikacji miejskiej. W odleg ...…
    + + +
    + + + + + +
    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + 3 900 zł + + + + + + + + + + + + +
    + + +
    + + 12 min temu +
    + +
    + +
    + + mieszkania i domy do wynajęcia , Kraków + +
    + + +
    + + + +
    +
    + +
    + + +
    + +
    + +
    +
    + + +
    +
    • + + + + + + + + + + + + +
  • +
    +
    + +
    + + Apartament tuż przy Wawelu z Krakow, zobacz zdjęcie + + + +
    Zdjęć: 15
    + +
    + +
    +
    +
    + + Apartament tuż przy Wawelu + +
    + + +
    English version below. + +Do wynajęcia apartament z widokiem na Planty. + +LOKALIZACJA: + +Oferowana na sprzedaż nieruchomość znajduje się w Krakowie w sąsiedztwie ulicy Zwierzynieckiej. W 5 minut spacerem znajdują się liczne atrakcje turystyczne Krakowa, w tym Rynek Główny, różnorodne restauracje, kluby oraz bary, centrum fitness, przystanek autobusowy i tramwajowy. W dalszej odległości – około 20 m ...…
    + + +
    + + + + + +
    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + 3 200 zł + + + + + + + + + + + + +
    + + +
    + + 12 min temu +
    + +
    + +
    + + mieszkania i domy do wynajęcia , Kraków + +
    + + +
    + + + +
    +
    + +
    + + +
    + +
    + +
    +
    + + +
    +
    • + + + + + + + + + + + + +
  • +
    +
    + +
    + + Do wynajęcia piękny dom 140 m2 - Borek Fałęcki z Krakow, zobacz zdjęcie + + + +
    Zdjęć: 15
    + +
    + +
    +
    +
    + + Do wynajęcia piękny dom 140 m2 - Borek Fałęcki + +
    + + +
    English version below. + +Do wynajęcia 140 m2 dom z ogrodem znajdująca się w południowej części Krakowa – Borek Fałęcki + +LOKALIZACJA: + +Dom znajduje się w cichej, spokojnej i przytulnej okolicy tuż obok parku. Lokalizacja zapewnia świetny dojazd w każdy zakątek Krakowa – pętla Borek Fałęcki. W pobliżu bogata infrastruktura handlowo – usługowa, sklepy, restauracje, Carrefour, Lidl, centrum handlowe ...…
    + + +
    + + + + + +
    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + 4 600 zł + + + + + + + + + + + + +
    + + +
    + + 12 min temu +
    + +
    + +
    + + mieszkania i domy do wynajęcia , Kraków + +
    + + +
    + + + +
    +
    + +
    + + +
    + +
    + +
    +
    + + +
    +
    • + + + + + + + + + + + + + + +
+ + + + + + + +
+ + + + + + + + + +
+ + + + +
Linki sponsorowane
+ + + + + + + +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ Klikając “Dodaj powiadomienie”, wyrażasz zgodę na nasze Zasady korzystania i Politykę prywatności oraz zgadzasz się na otrzymywanie naszych newsletterów i ofert promocyjnych. +
+
+ + + + + + + + +
+ + + + + + + + +
+
+
+ + + + + +
+ + + + + +
+ + + + + + +
+
+ Proponowane +
+ +
+ + +
+ + + + + + +
+
+
+ + + +
+
+
+
+
+
+
+
+ + + + + + +
+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + +
+
+
+
+
+ + + + + + + + + + + +
+
+
+ Popularne : +
+ + + + +
+
+
+ + + + + + + + + + + + + +
+
+
+ Najnowsze : +
+ + + + +
+
+
+ + + + + +
+ + + + +
+ +
+ + + +
+ + + +
+
+
+
+
+ +
+
+
+ + + Post Free Ad + + + +
+
+
+
+
+ + +
+ +
+ + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From b649266e8b0000f31e66be3b745b5dabbab4edb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Sroka?= Date: Sun, 2 Dec 2018 17:02:13 +0100 Subject: [PATCH 083/135] Announcement class and components structures --- .../java/extractor/entity/Announcement.java | 26 +++++++++++++++++++ .../main/java/extractor/entity/Lessor.java | 14 ++++++++++ .../main/java/extractor/entity/Location.java | 22 ++++++++++++++++ .../src/main/java/extractor/entity/Price.java | 22 ++++++++++++++++ .../java/extractor/entity/PropertyData.java | 20 ++++++++++++++ 5 files changed, 104 insertions(+) create mode 100644 announcement-processor-extractor/src/main/java/extractor/entity/Announcement.java create mode 100644 announcement-processor-extractor/src/main/java/extractor/entity/Lessor.java create mode 100644 announcement-processor-extractor/src/main/java/extractor/entity/Location.java create mode 100644 announcement-processor-extractor/src/main/java/extractor/entity/Price.java create mode 100644 announcement-processor-extractor/src/main/java/extractor/entity/PropertyData.java diff --git a/announcement-processor-extractor/src/main/java/extractor/entity/Announcement.java b/announcement-processor-extractor/src/main/java/extractor/entity/Announcement.java new file mode 100644 index 0000000..de7d2c1 --- /dev/null +++ b/announcement-processor-extractor/src/main/java/extractor/entity/Announcement.java @@ -0,0 +1,26 @@ +package extractor.entity; + +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.time.LocalDate; + +/* +* Contains extracted data which is sourced by the parser service. +* Groups the data into categories +* +* */ +@AllArgsConstructor +@Data +public class Announcement { + + private String title; + private Price price; + private Location location; + private PropertyData propertyData; + private Lessor lessor; + private LocalDate creationDate; + private String url; + private String description; + private String provider; // to enum +} diff --git a/announcement-processor-extractor/src/main/java/extractor/entity/Lessor.java b/announcement-processor-extractor/src/main/java/extractor/entity/Lessor.java new file mode 100644 index 0000000..75a8a4c --- /dev/null +++ b/announcement-processor-extractor/src/main/java/extractor/entity/Lessor.java @@ -0,0 +1,14 @@ +package extractor.entity; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class Lessor { + + String name; + String lessorType; // to enum + String phoneNumber; + String email; +} diff --git a/announcement-processor-extractor/src/main/java/extractor/entity/Location.java b/announcement-processor-extractor/src/main/java/extractor/entity/Location.java new file mode 100644 index 0000000..5d172ac --- /dev/null +++ b/announcement-processor-extractor/src/main/java/extractor/entity/Location.java @@ -0,0 +1,22 @@ +package extractor.entity; + +import lombok.AllArgsConstructor; +import lombok.Data; + +/* +* Consists information about the offered propoerty location. +* */ +@Data +@AllArgsConstructor +public class Location { + + private String country; + private String city; + private String street; + private String zipCode; + private String buildingNumber; + private String flatNumber; + private String district; + + +} diff --git a/announcement-processor-extractor/src/main/java/extractor/entity/Price.java b/announcement-processor-extractor/src/main/java/extractor/entity/Price.java new file mode 100644 index 0000000..cd75724 --- /dev/null +++ b/announcement-processor-extractor/src/main/java/extractor/entity/Price.java @@ -0,0 +1,22 @@ +package extractor.entity; + +import java.util.Map; + +/* + * Consists all the prices extracted from the announcement. + * It tries to evaluate an overall cost basing on the data contained in all the fields of the announcement. + * */ +public class Price { + + // Integers may be changed to a custom Money type + private Integer basePrice; + Map additionalPrices; + + public Integer getSummedPrice() { + return basePrice + additionalPrices.values() + .stream() + .mapToInt(i -> i) + .sum(); + } + +} diff --git a/announcement-processor-extractor/src/main/java/extractor/entity/PropertyData.java b/announcement-processor-extractor/src/main/java/extractor/entity/PropertyData.java new file mode 100644 index 0000000..50f7e0a --- /dev/null +++ b/announcement-processor-extractor/src/main/java/extractor/entity/PropertyData.java @@ -0,0 +1,20 @@ +package extractor.entity; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class PropertyData { + + String propertyType; // to enum? + Float area; + Boolean isSmokingAllowed; + Boolean isPerFriendly; + Integer roomNumber; + Integer bathroomNumber; + String parkingAvailability; + Integer level; + String furnishing; + +} From 2500ec4ded7157863b831fc722393dc8839a2fff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Sroka?= Date: Sun, 2 Dec 2018 17:31:55 +0100 Subject: [PATCH 084/135] Add AnnouncementDto --- announcement-processor-extractor/pom.xml | 8 ++ .../java/extractor/dto/AnnouncementDto.java | 73 +++++++++++++++++++ .../java/extractor/entity/Announcement.java | 4 +- 3 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 announcement-processor-extractor/src/main/java/extractor/dto/AnnouncementDto.java diff --git a/announcement-processor-extractor/pom.xml b/announcement-processor-extractor/pom.xml index f54eb5b..c16151c 100644 --- a/announcement-processor-extractor/pom.xml +++ b/announcement-processor-extractor/pom.xml @@ -60,6 +60,14 @@ jacoco-maven-plugin 0.8.2 + + + + com.fasterxml.jackson.core + jackson-annotations + 2.9.0 + + diff --git a/announcement-processor-extractor/src/main/java/extractor/dto/AnnouncementDto.java b/announcement-processor-extractor/src/main/java/extractor/dto/AnnouncementDto.java new file mode 100644 index 0000000..0b7ac34 --- /dev/null +++ b/announcement-processor-extractor/src/main/java/extractor/dto/AnnouncementDto.java @@ -0,0 +1,73 @@ +package extractor.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.time.LocalDateTime; + +@Data +@NoArgsConstructor +public class AnnouncementDto { + + private String title; + private Integer price; + private LocalDateTime creationDate; + private String lessor; + private String propertyType; + private Float flatArea; + private Integer roomAmount; + private Integer bathAmount; + private String parkingAvailability; + private Boolean isSmokingAllowed; + private Boolean isPetFriendly; + private String lessorName; + private Integer additionalRentCost; + private Integer level; + private String furnishing; + private String description; + private String provider; + private String url; + private String phoneNumber; + + public AnnouncementDto( + @JsonProperty("title") String title, + @JsonProperty("price") Integer price, + @JsonProperty("creationDate") LocalDateTime creationDate, + @JsonProperty("lessor") String lessor, + @JsonProperty("propertyType") String propertyType, + @JsonProperty("flatArea") Float flatArea, + @JsonProperty("roomAmount") Integer roomAmount, + @JsonProperty("bathAmount") Integer bathAmount, + @JsonProperty("parkingAvailability") String parkingAvailability, + @JsonProperty("isSmokingAllowed") Boolean isSmokingAllowed, + @JsonProperty("isPetFriendly") Boolean isPetFriendly, + @JsonProperty("lessorName") String lessorName, + @JsonProperty("additionalRentCost") Integer additionalRentCost, + @JsonProperty("level") Integer level, + @JsonProperty("furnishing") String furnishing, + @JsonProperty("description") String description, + @JsonProperty("provider") String provider, + @JsonProperty("url") String url, + @JsonProperty("phoneNumber") String phoneNumber) { + this.title = title; + this.price = price; + this.creationDate = creationDate; + this.lessor = lessor; + this.propertyType = propertyType; + this.flatArea = flatArea; + this.roomAmount = roomAmount; + this.bathAmount = bathAmount; + this.parkingAvailability = parkingAvailability; + this.isSmokingAllowed = isSmokingAllowed; + this.isPetFriendly = isPetFriendly; + this.lessorName = lessorName; + this.additionalRentCost = additionalRentCost; + this.level = level; + this.furnishing = furnishing; + this.description = description; + this.provider = provider; + this.url = url; + this.phoneNumber = phoneNumber; + } +} diff --git a/announcement-processor-extractor/src/main/java/extractor/entity/Announcement.java b/announcement-processor-extractor/src/main/java/extractor/entity/Announcement.java index de7d2c1..d515b27 100644 --- a/announcement-processor-extractor/src/main/java/extractor/entity/Announcement.java +++ b/announcement-processor-extractor/src/main/java/extractor/entity/Announcement.java @@ -3,7 +3,7 @@ import lombok.AllArgsConstructor; import lombok.Data; -import java.time.LocalDate; +import java.time.LocalDateTime; /* * Contains extracted data which is sourced by the parser service. @@ -19,7 +19,7 @@ public class Announcement { private Location location; private PropertyData propertyData; private Lessor lessor; - private LocalDate creationDate; + private LocalDateTime creationDate; private String url; private String description; private String provider; // to enum From 0a103e2b1cea131329143cffa31595fd495c48c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Sroka?= Date: Sun, 2 Dec 2018 22:52:18 +0100 Subject: [PATCH 085/135] Add AnnouncementConsumer (and test) --- announcement-processor-extractor/pom.xml | 13 +++ .../consumer/AnnouncementConsumer.java | 10 +++ .../AnnouncementConsumerFromString.java | 64 ++++++++++++++ .../consumer/AnnouncementConsumerJms.java | 11 +++ .../java/extractor/dto/AnnouncementDto.java | 4 +- .../AnnouncementConsumerFromFileTest.java | 83 +++++++++++++++++++ .../src/test/java/parser/ExampleTest.java | 14 ---- 7 files changed, 183 insertions(+), 16 deletions(-) create mode 100644 announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumer.java create mode 100644 announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerFromString.java create mode 100644 announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerJms.java create mode 100644 announcement-processor-extractor/src/test/java/extractor/consumer/AnnouncementConsumerFromFileTest.java delete mode 100644 announcement-processor-extractor/src/test/java/parser/ExampleTest.java diff --git a/announcement-processor-extractor/pom.xml b/announcement-processor-extractor/pom.xml index c16151c..1611fb4 100644 --- a/announcement-processor-extractor/pom.xml +++ b/announcement-processor-extractor/pom.xml @@ -67,6 +67,19 @@ jackson-annotations 2.9.0 + + + com.fasterxml.jackson.core + jackson-databind + 2.9.7 + + + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + + diff --git a/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumer.java b/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumer.java new file mode 100644 index 0000000..58054a2 --- /dev/null +++ b/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumer.java @@ -0,0 +1,10 @@ +package extractor.consumer; + +import extractor.dto.AnnouncementDto; + +import java.io.IOException; + +public interface AnnouncementConsumer { + + AnnouncementDto consumeAnnouncement() throws IOException; +} diff --git a/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerFromString.java b/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerFromString.java new file mode 100644 index 0000000..d2b520f --- /dev/null +++ b/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerFromString.java @@ -0,0 +1,64 @@ +package extractor.consumer; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import extractor.dto.AnnouncementDto; + +import java.io.IOException; + +public class AnnouncementConsumerFromString implements AnnouncementConsumer { + + private ObjectMapper mapper; + private final String json = "{" + + "\"title\":" + + "\"Kawalerka 40m Głowackiego 4 Kraków PRZESTRONNE niskie koszty WIDOK na PANORAMĘ Krakowa POŁUDNIE\"," + + "\"price\": 1300," + + "\"creationDate\":\"2018-11-17T00:00:00\"," + + "\"lessor\":\"Właściciel\"," + + "\"propertyType\":\"Mieszkanie\"," + + "\"flatArea\":40.0," + + "\"roomAmount\":1," + + "\"bathAmount\":1," + + "\"parkingAvailability\":\"Ulica\"," + + "\"isSmokingAllowed\":true," + + "\"isPetFriendly\":false," + + "\"lessorName\":\"Bartek\"," + + "\"additionalRentCost\":null," + + "\"level\":null," + + "\"furnishing\":null," + + "\"description\":\"Proponuję mieszkanie na wynajem (kawalerka) o powierzchni 40m2 " + + "zlokalizowane przy ulicy Głowackiego 4 w Krakowie. Dzielnica Krowodrza." + + " Rejon Bronowice.\\n\\nKawalerka mieście się na 5 piętrze od strony południowej z" + + " widokiem na panoramę Krakowa.\\n3 minuty pieszo do przystanków tramwajowych." + + " Bliskie sąsiedztwo Uniwersytetu Pedagogicznego, AGH, Wydziału Fizyki i Architektury " + + "PK.\\n\\nBudynek oddany do użytkowania w 2008r. Podłoga z litego drewna, w aneksie kuchennym i" + + " w łazience płytki ceramiczne. Mieszkanie wyposażone tak jak na zdjęciach: (lodówka, płyta grzewcza," + + " piekarnik, pralka, stół, krzesła, łóżko 2os., łóżko 1os.)\\nMożliwość usunięcia mebli wg upodobań" + + " wynajmującego jak i ich dokupienia.\\n\\nNieruchomość posiada przestronny hol i windę. Na parterze " + + "zlokalizowane są sklepy m. in. Lewiatan - można w pantoflach robić zakupy ;)\\n\\nKoszty najmu " + + "(miesięczne): wynajem 1500zł, + czynsz administracyjny 250zł, + media wg zużycia " + + "(woda (~60zł/1os), prąd (~60zł/1os) ," + + " ogrzewanie, internet (60zł)).\\n\\nSZACOWANY CAŁKOWITY KOSZT wynajmu i eksploatacji miesięcznej:" + + " 1900-2000zł (2 osoby)\\n\\n2K PLN per month ;)\\nZalety to niskie koszty ogrzewania, w zasadzie " + + "pomijalne.\\n\\nINTERNET od UPC. Mieszkanie wyposażone w gniazdo RTV i SAT.\\n\\nUmowa min. na 9 miesięcy. " + + "W razie pytań proszę się nie krępować :)\\n\\nKontak mailowy lub telefoniczny (w rozsądnych godzinach):" + + "\\n\\n6 0 2 3 2 7 7 0 6\\n\\nnioobi (malpa) op (kropka) pl\\n\\n\\nMożliwość oglądnięcia mieszkania\\n\\n" + + "Palić papierosy można na dużym balkonie ;)\\n\\n\\nPośrednikom z góry dziękuję za zainteresowanie!\"," + + "\"provider\":\"GUMTREE\"," + + "\"url\":\"https://www.gumtree.pl/a-mieszkania-i-domy-do-wynajecia/krakow/" + + "kawalerka-40m-glowackiego-4-krakow-przestronne-niskie-koszty-widok-na-panorame-krakowa-poludnie/" + + "1003555014070910781350209\"," + + "\"phoneNumber\":null" + + "}"; + + public AnnouncementConsumerFromString() { + this.mapper = new ObjectMapper() + .registerModule(new JavaTimeModule()); + } + + @Override + public AnnouncementDto consumeAnnouncement() throws IOException { + AnnouncementDto announcementDto = mapper.readValue(this.json, AnnouncementDto.class); + return announcementDto; + } +} diff --git a/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerJms.java b/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerJms.java new file mode 100644 index 0000000..6b6688f --- /dev/null +++ b/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerJms.java @@ -0,0 +1,11 @@ +package extractor.consumer; + +import extractor.dto.AnnouncementDto; + +public class AnnouncementConsumerJms implements AnnouncementConsumer { + + @Override + public AnnouncementDto consumeAnnouncement() { + return null; + } +} diff --git a/announcement-processor-extractor/src/main/java/extractor/dto/AnnouncementDto.java b/announcement-processor-extractor/src/main/java/extractor/dto/AnnouncementDto.java index 0b7ac34..2cf5fbe 100644 --- a/announcement-processor-extractor/src/main/java/extractor/dto/AnnouncementDto.java +++ b/announcement-processor-extractor/src/main/java/extractor/dto/AnnouncementDto.java @@ -15,7 +15,7 @@ public class AnnouncementDto { private LocalDateTime creationDate; private String lessor; private String propertyType; - private Float flatArea; + private Double flatArea; private Integer roomAmount; private Integer bathAmount; private String parkingAvailability; @@ -36,7 +36,7 @@ public AnnouncementDto( @JsonProperty("creationDate") LocalDateTime creationDate, @JsonProperty("lessor") String lessor, @JsonProperty("propertyType") String propertyType, - @JsonProperty("flatArea") Float flatArea, + @JsonProperty("flatArea") Double flatArea, @JsonProperty("roomAmount") Integer roomAmount, @JsonProperty("bathAmount") Integer bathAmount, @JsonProperty("parkingAvailability") String parkingAvailability, diff --git a/announcement-processor-extractor/src/test/java/extractor/consumer/AnnouncementConsumerFromFileTest.java b/announcement-processor-extractor/src/test/java/extractor/consumer/AnnouncementConsumerFromFileTest.java new file mode 100644 index 0000000..7de9308 --- /dev/null +++ b/announcement-processor-extractor/src/test/java/extractor/consumer/AnnouncementConsumerFromFileTest.java @@ -0,0 +1,83 @@ +package extractor.consumer; + +import extractor.dto.AnnouncementDto; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; + +public class AnnouncementConsumerFromFileTest { + + AnnouncementConsumerFromString announcementConsumer = new AnnouncementConsumerFromString(); + + @Test + public void consumeAnnouncementTest() throws IOException { + String title = "Kawalerka 40m Głowackiego 4 Kraków PRZESTRONNE niskie koszty WIDOK na PANORAMĘ Krakowa POŁUDNIE"; + Integer price = 1300; + String strDate = "2018-11-17 00:00:00"; + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + LocalDateTime creationDate = LocalDateTime.parse(strDate, formatter); + String lessor = "Właściciel"; + String propertyType = "Mieszkanie"; + Double flatArea = 40.0; + Integer roomAmount = 1; + Integer bathAmount = 1; + String parkingAvailability = "Ulica"; + Boolean isSmokingAllowed = true; + Boolean isPetFriendly = false; + String lessorName = "Bartek"; + Integer additionalRentCost = null; + Integer level = null; + String furnishing = null; + String description = "Proponuję mieszkanie na wynajem (kawalerka) o powierzchni 40m2 " + + "zlokalizowane przy ulicy Głowackiego 4 w Krakowie. Dzielnica Krowodrza." + + " Rejon Bronowice.\n\nKawalerka mieście się na 5 piętrze od strony południowej z" + + " widokiem na panoramę Krakowa.\n3 minuty pieszo do przystanków tramwajowych." + + " Bliskie sąsiedztwo Uniwersytetu Pedagogicznego, AGH, Wydziału Fizyki i Architektury " + + "PK.\n\nBudynek oddany do użytkowania w 2008r. Podłoga z litego drewna, w aneksie kuchennym i" + + " w łazience płytki ceramiczne. Mieszkanie wyposażone tak jak na zdjęciach: (lodówka, płyta grzewcza," + + " piekarnik, pralka, stół, krzesła, łóżko 2os., łóżko 1os.)\nMożliwość usunięcia mebli wg upodobań" + + " wynajmującego jak i ich dokupienia.\n\nNieruchomość posiada przestronny hol i windę. Na parterze " + + "zlokalizowane są sklepy m. in. Lewiatan - można w pantoflach robić zakupy ;)\n\nKoszty najmu " + + "(miesięczne): wynajem 1500zł, + czynsz administracyjny 250zł, + media wg zużycia " + + "(woda (~60zł/1os), prąd (~60zł/1os) ," + + " ogrzewanie, internet (60zł)).\n\nSZACOWANY CAŁKOWITY KOSZT wynajmu i eksploatacji miesięcznej:" + + " 1900-2000zł (2 osoby)\n\n2K PLN per month ;)\nZalety to niskie koszty ogrzewania, w zasadzie " + + "pomijalne.\n\nINTERNET od UPC. Mieszkanie wyposażone w gniazdo RTV i SAT.\n\nUmowa min. na 9 miesięcy. " + + "W razie pytań proszę się nie krępować :)\n\nKontak mailowy lub telefoniczny (w rozsądnych godzinach):" + + "\n\n6 0 2 3 2 7 7 0 6\n\nnioobi (malpa) op (kropka) pl\n\n\nMożliwość oglądnięcia mieszkania\n\n" + + "Palić papierosy można na dużym balkonie ;)\n\n\nPośrednikom z góry dziękuję za zainteresowanie!"; + String provider = "GUMTREE"; + String url = "https://www.gumtree.pl/a-mieszkania-i-domy-do-wynajecia/krakow/" + + "kawalerka-40m-glowackiego-4-krakow-przestronne-niskie-koszty-widok-na-panorame-krakowa-poludnie/" + + "1003555014070910781350209"; + String phoneNumber = null; + + + AnnouncementDto template = new AnnouncementDto( + title, + price, + creationDate, + lessor, + propertyType, + flatArea, + roomAmount, + bathAmount, + parkingAvailability, + isSmokingAllowed, + isPetFriendly, + lessorName, + additionalRentCost, + level, + furnishing, + description, + provider, + url, + phoneNumber + ); + AnnouncementDto announcementDto = announcementConsumer.consumeAnnouncement(); + Assert.assertEquals(template, announcementDto); + } +} diff --git a/announcement-processor-extractor/src/test/java/parser/ExampleTest.java b/announcement-processor-extractor/src/test/java/parser/ExampleTest.java deleted file mode 100644 index 09de7c0..0000000 --- a/announcement-processor-extractor/src/test/java/parser/ExampleTest.java +++ /dev/null @@ -1,14 +0,0 @@ -package parser; - -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -@RunWith(JUnit4.class) -public class ExampleTest { - @Test - public void test(){ - Assert.assertTrue(true); - } -} From 755431d41d9ded3c7e5e4724c04792fed13a65bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Sroka?= Date: Sun, 2 Dec 2018 23:01:03 +0100 Subject: [PATCH 086/135] Add MapperService and shift handling mapping from AnnouncementConsumer --- .../consumer/AnnouncementConsumer.java | 3 +- .../AnnouncementConsumerFromString.java | 14 ++++----- .../java/extractor/service/MapperService.java | 29 +++++++++++++++++++ .../AnnouncementConsumerFromFileTest.java | 6 ++-- 4 files changed, 38 insertions(+), 14 deletions(-) create mode 100644 announcement-processor-extractor/src/main/java/extractor/service/MapperService.java diff --git a/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumer.java b/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumer.java index 58054a2..9c79a80 100644 --- a/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumer.java +++ b/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumer.java @@ -2,9 +2,8 @@ import extractor.dto.AnnouncementDto; -import java.io.IOException; public interface AnnouncementConsumer { - AnnouncementDto consumeAnnouncement() throws IOException; + AnnouncementDto consumeAnnouncement(); } diff --git a/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerFromString.java b/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerFromString.java index d2b520f..eb8864a 100644 --- a/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerFromString.java +++ b/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerFromString.java @@ -1,14 +1,12 @@ package extractor.consumer; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import extractor.dto.AnnouncementDto; +import extractor.service.MapperService; -import java.io.IOException; public class AnnouncementConsumerFromString implements AnnouncementConsumer { - private ObjectMapper mapper; + private MapperService mapperService; private final String json = "{" + "\"title\":" + "\"Kawalerka 40m Głowackiego 4 Kraków PRZESTRONNE niskie koszty WIDOK na PANORAMĘ Krakowa POŁUDNIE\"," + @@ -52,13 +50,11 @@ public class AnnouncementConsumerFromString implements AnnouncementConsumer { "}"; public AnnouncementConsumerFromString() { - this.mapper = new ObjectMapper() - .registerModule(new JavaTimeModule()); + this.mapperService = new MapperService(); } @Override - public AnnouncementDto consumeAnnouncement() throws IOException { - AnnouncementDto announcementDto = mapper.readValue(this.json, AnnouncementDto.class); - return announcementDto; + public AnnouncementDto consumeAnnouncement() { + return mapperService.getAnnouncementDtoFromJsonString(this.json); } } diff --git a/announcement-processor-extractor/src/main/java/extractor/service/MapperService.java b/announcement-processor-extractor/src/main/java/extractor/service/MapperService.java new file mode 100644 index 0000000..c694911 --- /dev/null +++ b/announcement-processor-extractor/src/main/java/extractor/service/MapperService.java @@ -0,0 +1,29 @@ +package extractor.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import extractor.dto.AnnouncementDto; +import org.springframework.stereotype.Service; + +import java.io.IOException; + +@Service +public class MapperService { + + private ObjectMapper mapper; + + public MapperService() { + this.mapper = new ObjectMapper() + .registerModule(new JavaTimeModule()); + } + + public AnnouncementDto getAnnouncementDtoFromJsonString(String json) { + try { + return mapper.readValue(json, AnnouncementDto.class); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } + +} diff --git a/announcement-processor-extractor/src/test/java/extractor/consumer/AnnouncementConsumerFromFileTest.java b/announcement-processor-extractor/src/test/java/extractor/consumer/AnnouncementConsumerFromFileTest.java index 7de9308..c27d060 100644 --- a/announcement-processor-extractor/src/test/java/extractor/consumer/AnnouncementConsumerFromFileTest.java +++ b/announcement-processor-extractor/src/test/java/extractor/consumer/AnnouncementConsumerFromFileTest.java @@ -4,16 +4,16 @@ import org.junit.Assert; import org.junit.Test; -import java.io.IOException; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; + public class AnnouncementConsumerFromFileTest { - AnnouncementConsumerFromString announcementConsumer = new AnnouncementConsumerFromString(); + private AnnouncementConsumerFromString announcementConsumer = new AnnouncementConsumerFromString(); @Test - public void consumeAnnouncementTest() throws IOException { + public void consumeAnnouncementTest() { String title = "Kawalerka 40m Głowackiego 4 Kraków PRZESTRONNE niskie koszty WIDOK na PANORAMĘ Krakowa POŁUDNIE"; Integer price = 1300; String strDate = "2018-11-17 00:00:00"; From 47e48210166d38e8860e62745d0c670d6d4011af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Sroka?= Date: Sun, 2 Dec 2018 23:24:23 +0100 Subject: [PATCH 087/135] Add lombok annotations to Price entity --- .../extractor/consumer/AnnouncementConsumerFromString.java | 2 +- .../src/main/java/extractor/entity/Price.java | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerFromString.java b/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerFromString.java index eb8864a..feb807c 100644 --- a/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerFromString.java +++ b/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerFromString.java @@ -50,7 +50,7 @@ public class AnnouncementConsumerFromString implements AnnouncementConsumer { "}"; public AnnouncementConsumerFromString() { - this.mapperService = new MapperService(); + this.mapperService = new MapperService(); // pass as argument } @Override diff --git a/announcement-processor-extractor/src/main/java/extractor/entity/Price.java b/announcement-processor-extractor/src/main/java/extractor/entity/Price.java index cd75724..7125256 100644 --- a/announcement-processor-extractor/src/main/java/extractor/entity/Price.java +++ b/announcement-processor-extractor/src/main/java/extractor/entity/Price.java @@ -1,11 +1,16 @@ package extractor.entity; +import lombok.AllArgsConstructor; +import lombok.Data; + import java.util.Map; /* * Consists all the prices extracted from the announcement. * It tries to evaluate an overall cost basing on the data contained in all the fields of the announcement. * */ +@Data +@AllArgsConstructor public class Price { // Integers may be changed to a custom Money type From 2e21c55fdd9880438d8cfcaf1ee53ef052c768fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Sroka?= Date: Mon, 3 Dec 2018 00:20:56 +0100 Subject: [PATCH 088/135] Change access modifiers #36 --- .../main/java/extractor/entity/Lessor.java | 8 ++++---- .../main/java/extractor/entity/Location.java | 2 -- .../src/main/java/extractor/entity/Price.java | 3 +-- .../java/extractor/entity/PropertyData.java | 19 +++++++++---------- .../java/extractor/service/MapperService.java | 1 - 5 files changed, 14 insertions(+), 19 deletions(-) diff --git a/announcement-processor-extractor/src/main/java/extractor/entity/Lessor.java b/announcement-processor-extractor/src/main/java/extractor/entity/Lessor.java index 75a8a4c..5595557 100644 --- a/announcement-processor-extractor/src/main/java/extractor/entity/Lessor.java +++ b/announcement-processor-extractor/src/main/java/extractor/entity/Lessor.java @@ -7,8 +7,8 @@ @AllArgsConstructor public class Lessor { - String name; - String lessorType; // to enum - String phoneNumber; - String email; + private String name; + private String lessorType; // to enum + private String phoneNumber; + private String email; } diff --git a/announcement-processor-extractor/src/main/java/extractor/entity/Location.java b/announcement-processor-extractor/src/main/java/extractor/entity/Location.java index 5d172ac..a74918d 100644 --- a/announcement-processor-extractor/src/main/java/extractor/entity/Location.java +++ b/announcement-processor-extractor/src/main/java/extractor/entity/Location.java @@ -17,6 +17,4 @@ public class Location { private String buildingNumber; private String flatNumber; private String district; - - } diff --git a/announcement-processor-extractor/src/main/java/extractor/entity/Price.java b/announcement-processor-extractor/src/main/java/extractor/entity/Price.java index 7125256..d890790 100644 --- a/announcement-processor-extractor/src/main/java/extractor/entity/Price.java +++ b/announcement-processor-extractor/src/main/java/extractor/entity/Price.java @@ -15,7 +15,7 @@ public class Price { // Integers may be changed to a custom Money type private Integer basePrice; - Map additionalPrices; + private Map additionalPrices; public Integer getSummedPrice() { return basePrice + additionalPrices.values() @@ -23,5 +23,4 @@ public Integer getSummedPrice() { .mapToInt(i -> i) .sum(); } - } diff --git a/announcement-processor-extractor/src/main/java/extractor/entity/PropertyData.java b/announcement-processor-extractor/src/main/java/extractor/entity/PropertyData.java index 50f7e0a..ab4c92e 100644 --- a/announcement-processor-extractor/src/main/java/extractor/entity/PropertyData.java +++ b/announcement-processor-extractor/src/main/java/extractor/entity/PropertyData.java @@ -7,14 +7,13 @@ @AllArgsConstructor public class PropertyData { - String propertyType; // to enum? - Float area; - Boolean isSmokingAllowed; - Boolean isPerFriendly; - Integer roomNumber; - Integer bathroomNumber; - String parkingAvailability; - Integer level; - String furnishing; - + private String propertyType; // to enum? + private Float area; + private Boolean isSmokingAllowed; + private Boolean isPerFriendly; + private Integer roomNumber; + private Integer bathroomNumber; + private String parkingAvailability; + private Integer level; + private String furnishing; } diff --git a/announcement-processor-extractor/src/main/java/extractor/service/MapperService.java b/announcement-processor-extractor/src/main/java/extractor/service/MapperService.java index c694911..a8ec52e 100644 --- a/announcement-processor-extractor/src/main/java/extractor/service/MapperService.java +++ b/announcement-processor-extractor/src/main/java/extractor/service/MapperService.java @@ -25,5 +25,4 @@ public AnnouncementDto getAnnouncementDtoFromJsonString(String json) { return null; } } - } From a9f0458bc6a9acb1fe315f1adbe5ebf925bb0900 Mon Sep 17 00:00:00 2001 From: michalsroka Date: Mon, 3 Dec 2018 21:55:25 +0100 Subject: [PATCH 089/135] Add basic data extraction from AnnouncementDto #36 --- .../java/extractor/entity/Announcement.java | 2 + .../main/java/extractor/entity/Lessor.java | 2 + .../main/java/extractor/entity/Location.java | 2 + .../src/main/java/extractor/entity/Price.java | 2 + .../java/extractor/entity/PropertyData.java | 4 +- .../AnnouncementExtractingService.java | 70 +++++++++++++++++++ .../AnnouncementExtractingServiceTest.java | 20 ++++++ 7 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 announcement-processor-extractor/src/main/java/extractor/service/AnnouncementExtractingService.java create mode 100644 announcement-processor-extractor/src/test/java/extractor/service/AnnouncementExtractingServiceTest.java diff --git a/announcement-processor-extractor/src/main/java/extractor/entity/Announcement.java b/announcement-processor-extractor/src/main/java/extractor/entity/Announcement.java index d515b27..10afac3 100644 --- a/announcement-processor-extractor/src/main/java/extractor/entity/Announcement.java +++ b/announcement-processor-extractor/src/main/java/extractor/entity/Announcement.java @@ -2,6 +2,7 @@ import lombok.AllArgsConstructor; import lombok.Data; +import lombok.NoArgsConstructor; import java.time.LocalDateTime; @@ -11,6 +12,7 @@ * * */ @AllArgsConstructor +@NoArgsConstructor @Data public class Announcement { diff --git a/announcement-processor-extractor/src/main/java/extractor/entity/Lessor.java b/announcement-processor-extractor/src/main/java/extractor/entity/Lessor.java index 5595557..4bbd08f 100644 --- a/announcement-processor-extractor/src/main/java/extractor/entity/Lessor.java +++ b/announcement-processor-extractor/src/main/java/extractor/entity/Lessor.java @@ -2,9 +2,11 @@ import lombok.AllArgsConstructor; import lombok.Data; +import lombok.NoArgsConstructor; @Data @AllArgsConstructor +@NoArgsConstructor public class Lessor { private String name; diff --git a/announcement-processor-extractor/src/main/java/extractor/entity/Location.java b/announcement-processor-extractor/src/main/java/extractor/entity/Location.java index a74918d..093b9a3 100644 --- a/announcement-processor-extractor/src/main/java/extractor/entity/Location.java +++ b/announcement-processor-extractor/src/main/java/extractor/entity/Location.java @@ -2,12 +2,14 @@ import lombok.AllArgsConstructor; import lombok.Data; +import lombok.NoArgsConstructor; /* * Consists information about the offered propoerty location. * */ @Data @AllArgsConstructor +@NoArgsConstructor public class Location { private String country; diff --git a/announcement-processor-extractor/src/main/java/extractor/entity/Price.java b/announcement-processor-extractor/src/main/java/extractor/entity/Price.java index d890790..f8451c6 100644 --- a/announcement-processor-extractor/src/main/java/extractor/entity/Price.java +++ b/announcement-processor-extractor/src/main/java/extractor/entity/Price.java @@ -2,6 +2,7 @@ import lombok.AllArgsConstructor; import lombok.Data; +import lombok.NoArgsConstructor; import java.util.Map; @@ -11,6 +12,7 @@ * */ @Data @AllArgsConstructor +@NoArgsConstructor public class Price { // Integers may be changed to a custom Money type diff --git a/announcement-processor-extractor/src/main/java/extractor/entity/PropertyData.java b/announcement-processor-extractor/src/main/java/extractor/entity/PropertyData.java index ab4c92e..8f81ff4 100644 --- a/announcement-processor-extractor/src/main/java/extractor/entity/PropertyData.java +++ b/announcement-processor-extractor/src/main/java/extractor/entity/PropertyData.java @@ -2,13 +2,15 @@ import lombok.AllArgsConstructor; import lombok.Data; +import lombok.NoArgsConstructor; @Data @AllArgsConstructor +@NoArgsConstructor public class PropertyData { private String propertyType; // to enum? - private Float area; + private Double area; private Boolean isSmokingAllowed; private Boolean isPerFriendly; private Integer roomNumber; diff --git a/announcement-processor-extractor/src/main/java/extractor/service/AnnouncementExtractingService.java b/announcement-processor-extractor/src/main/java/extractor/service/AnnouncementExtractingService.java new file mode 100644 index 0000000..74caefc --- /dev/null +++ b/announcement-processor-extractor/src/main/java/extractor/service/AnnouncementExtractingService.java @@ -0,0 +1,70 @@ +package extractor.service; + +import extractor.dto.AnnouncementDto; +import extractor.entity.*; +import org.springframework.stereotype.Service; + +import java.util.HashMap; +import java.util.Map; + +@Service +public class AnnouncementExtractingService { + + public Announcement extractFromAnnouncementDto(AnnouncementDto announcementDto) { + + Announcement announcement = new Announcement(); + + announcement.setTitle(announcementDto.getTitle()); + announcement.setCreationDate(announcementDto.getCreationDate()); + announcement.setDescription(announcementDto.getDescription()); + announcement.setProvider(announcementDto.getProvider()); + announcement.setUrl(announcementDto.getUrl()); + + announcement.setLessor(extractLessor(announcementDto)); + announcement.setLocation(extractLocation(announcementDto)); + announcement.setPrice(extractPrice(announcementDto)); + announcement.setPropertyData(extractPropertyData(announcementDto)); + + return announcement; + } + + private Lessor extractLessor(AnnouncementDto announcementDto) { + Lessor lessor = new Lessor(); + lessor.setName(announcementDto.getLessorName()); + lessor.setLessorType(announcementDto.getLessor()); + lessor.setPhoneNumber(announcementDto.getPhoneNumber()); + // TODO more advanced extraction e.g. description data extraction + return lessor; + } + + private Location extractLocation(AnnouncementDto announcementDto) { + Location location = new Location(); + // TODO more advanced extraction e.g. description data extraction + return location; + } + + private Price extractPrice(AnnouncementDto announcementDto) { + Price price = new Price(); + price.setBasePrice(announcementDto.getPrice()); + // TODO more advanced extraction e.g. description data extraction + Map pricesMap = new HashMap<>(); + pricesMap.put("SomePrice", announcementDto.getAdditionalRentCost()); + price.setAdditionalPrices(pricesMap); + return price; + } + + private PropertyData extractPropertyData(AnnouncementDto announcementDto) { + PropertyData propertyData = new PropertyData(); + propertyData.setArea(announcementDto.getFlatArea()); + propertyData.setBathroomNumber(announcementDto.getBathAmount()); + propertyData.setFurnishing(announcementDto.getFurnishing()); + propertyData.setIsPerFriendly(announcementDto.getIsPetFriendly()); + propertyData.setIsSmokingAllowed(announcementDto.getIsSmokingAllowed()); + propertyData.setLevel(announcementDto.getLevel()); + propertyData.setParkingAvailability(announcementDto.getParkingAvailability()); + propertyData.setPropertyType(announcementDto.getPropertyType()); + propertyData.setRoomNumber(announcementDto.getRoomAmount()); + // TODO more advanced extraction e.g. description data extraction + return propertyData; + } +} diff --git a/announcement-processor-extractor/src/test/java/extractor/service/AnnouncementExtractingServiceTest.java b/announcement-processor-extractor/src/test/java/extractor/service/AnnouncementExtractingServiceTest.java new file mode 100644 index 0000000..9266235 --- /dev/null +++ b/announcement-processor-extractor/src/test/java/extractor/service/AnnouncementExtractingServiceTest.java @@ -0,0 +1,20 @@ +package extractor.service; + +import extractor.consumer.AnnouncementConsumer; +import extractor.consumer.AnnouncementConsumerFromString; +import extractor.dto.AnnouncementDto; +import extractor.entity.Announcement; +import org.junit.Test; + +public class AnnouncementExtractingServiceTest { + + private final AnnouncementConsumer announcementConsumer = new AnnouncementConsumerFromString(); + private final AnnouncementExtractingService extractingService = new AnnouncementExtractingService(); + + @Test + public void extractFromAnnouncementDtoTest() { + AnnouncementDto announcementDto = announcementConsumer.consumeAnnouncement(); + Announcement announcement = extractingService.extractFromAnnouncementDto(announcementDto); + System.out.println(announcement); // TODO proper test + } +} From 6958e799d934827255355a65a082750ce6c5b431 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Sroka?= Date: Wed, 12 Dec 2018 23:28:23 +0100 Subject: [PATCH 090/135] Add scheduler class --- .../src/main/java/extractor/Comp.java | 16 ---------------- .../consumer/ReceivementScheduler.java | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 16 deletions(-) delete mode 100644 announcement-processor-extractor/src/main/java/extractor/Comp.java create mode 100644 announcement-processor-extractor/src/main/java/extractor/consumer/ReceivementScheduler.java diff --git a/announcement-processor-extractor/src/main/java/extractor/Comp.java b/announcement-processor-extractor/src/main/java/extractor/Comp.java deleted file mode 100644 index 2e5ac5c..0000000 --- a/announcement-processor-extractor/src/main/java/extractor/Comp.java +++ /dev/null @@ -1,16 +0,0 @@ -package extractor; - -import lombok.extern.java.Log; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Controller; - -import java.time.LocalDateTime; - -@Controller -@Log -public class Comp { - @Scheduled(fixedRate = 5000) - public void reportCurrentTime() { - log.info("The time is now: " + LocalDateTime.now().toString()); - } -} \ No newline at end of file diff --git a/announcement-processor-extractor/src/main/java/extractor/consumer/ReceivementScheduler.java b/announcement-processor-extractor/src/main/java/extractor/consumer/ReceivementScheduler.java new file mode 100644 index 0000000..aa023ba --- /dev/null +++ b/announcement-processor-extractor/src/main/java/extractor/consumer/ReceivementScheduler.java @@ -0,0 +1,18 @@ +package extractor.consumer; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +@EnableScheduling +public class ReceivementScheduler { + + @Scheduled(cron= "0 */2 * * * *") + private void scheduleTask() { + log.info("Executing ReceivementScheduler scheduledTask"); + + } +} From 53d0b52fcf99efe699c14e3d78db3ba9ae0283d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Wed, 19 Dec 2018 17:48:51 +0100 Subject: [PATCH 091/135] Allow to parse announcements on page via multiple threads. #10 --- .../walker/states/ProcessPageState.java | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java b/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java index 5a145d6..e49a0a7 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java +++ b/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java @@ -11,12 +11,16 @@ import java.util.HashMap; import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; @Slf4j @Component @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) public class ProcessPageState extends WalkerState { private HashMap pageWithAnnouncement; + private final int THREAD_POOL_SIZE = 10; public ProcessPageState(ProviderHelper providerHelper, AnnouncementParser announcementParser, HashMap pageWithAnnouncement) { super(providerHelper, announcementParser); @@ -27,13 +31,23 @@ public ProcessPageState(ProviderHelper providerHelper, AnnouncementParser announ public WalkerState run() { Document pageDocument = (Document) pageWithAnnouncement.get("pageDocument"); int divNumber = (int) pageWithAnnouncement.get("divNumber"); - List urlsToParse = providerHelper.getUrlsToParse(pageDocument, divNumber); - for (String urlToParse : urlsToParse) { - Announcement announcement = announcementParser.parsePage(urlToParse); - // TODO: Save record to DB_PARSER with valid counter - } + log.info("Number of pages to parse: " + urlsToParse.size()); + + try { + ExecutorService executorService = Executors.newFixedThreadPool(THREAD_POOL_SIZE); + for (String urlToParse : urlsToParse) { + executorService.submit(() -> { + Announcement announcement = announcementParser.parsePage(urlToParse); + }); + // TODO: Save record to DB_PARSER with valid counter + } + executorService.shutdown(); + executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); + } catch (InterruptedException e) { + log.error("Interrupted exception"); + } log.info("Parsing urls from page " + providerHelper.getActualPageURLNumber() + " done. Go to VisibleAfterReloadState"); return new VisibleAfterReloadState(providerHelper); From 3c27fec39732e0013cc6d38da81474fc80d049f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Wed, 19 Dec 2018 22:08:18 +0100 Subject: [PATCH 092/135] Code clean-up. #10 --- .../src/main/java/parser/walker/helpers/GumtreeHelper.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java b/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java index 5dd4244..5c56f84 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java +++ b/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java @@ -80,7 +80,8 @@ public List getUrlsToParse(Document document, int divNumber) { log.info("Geeting urls to parse"); return getElementsWithDataFromPage(document) .stream() - .limit(divNumber).map(this::getPageUrlFromElement) + .limit(divNumber) + .map(this::getPageUrlFromElement) .collect(Collectors.toList()); } @@ -90,7 +91,7 @@ LocalDate getEarliestDateOnAnnouncementPage(Document scannedPage) { Optional first = elementsWithData.stream() .map(element -> element.selectFirst(".creation-date").text()) .map(date -> convertStringToLocalDate(date, actualDateTime)) - .min(Comparator.reverseOrder()); + .max(Comparator.naturalOrder()); return first.orElse(null); } From 6cceacd5e04449492071271b2eb379420939b29f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Thu, 20 Dec 2018 00:45:13 +0100 Subject: [PATCH 093/135] Save parsed sites in parser registry. #10 --- .../src/main/java/parser/Announcement.java | 2 ++ .../src/main/java/parser/Comp.java | 16 ---------------- .../java/parser/registry/ParsingInfoService.java | 12 ++++++++++++ .../parser/walker/helpers/ProviderHelper.java | 4 ++++ .../parser/walker/states/ProcessPageState.java | 12 +++++++++--- 5 files changed, 27 insertions(+), 19 deletions(-) delete mode 100644 announcement-processor-parser/src/main/java/parser/Comp.java diff --git a/announcement-processor-parser/src/main/java/parser/Announcement.java b/announcement-processor-parser/src/main/java/parser/Announcement.java index 6fd3b2d..caf523a 100644 --- a/announcement-processor-parser/src/main/java/parser/Announcement.java +++ b/announcement-processor-parser/src/main/java/parser/Announcement.java @@ -1,5 +1,6 @@ package parser; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; @@ -8,6 +9,7 @@ @Getter @Setter +@EqualsAndHashCode public class Announcement { private String title; private BigDecimal price; diff --git a/announcement-processor-parser/src/main/java/parser/Comp.java b/announcement-processor-parser/src/main/java/parser/Comp.java deleted file mode 100644 index e5f3db9..0000000 --- a/announcement-processor-parser/src/main/java/parser/Comp.java +++ /dev/null @@ -1,16 +0,0 @@ -package parser; - -import lombok.extern.java.Log; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Controller; - -import java.time.LocalDateTime; - -@Controller -@Log -public class Comp { - @Scheduled(fixedRate = 5000) - public void reportCurrentTime() { - log.info("The time is now: " + LocalDateTime.now().toString()); - } -} diff --git a/announcement-processor-parser/src/main/java/parser/registry/ParsingInfoService.java b/announcement-processor-parser/src/main/java/parser/registry/ParsingInfoService.java index 505bbe3..5d4b6ac 100644 --- a/announcement-processor-parser/src/main/java/parser/registry/ParsingInfoService.java +++ b/announcement-processor-parser/src/main/java/parser/registry/ParsingInfoService.java @@ -1,6 +1,7 @@ package parser.registry; import org.springframework.stereotype.Service; +import parser.Announcement; import java.util.Optional; @@ -32,4 +33,15 @@ public Optional fetchLastRecordForProvider(String provider, Parsing } return announcement; } + + public void saveAnnouncementToRegistry(String provider, Announcement announcement, int counterNumber) { + ParsingInfo parsingInfo = new ParsingInfo(); + parsingInfo.setCounterPerDate(counterNumber); + parsingInfo.setDate(announcement.getCreationDate().toLocalDate()); + parsingInfo.setProvider(provider); + parsingInfo.setUrl(announcement.getUrl()); + parsingInfo.setPageHash("" + announcement.hashCode()); + + parsingInfoRepository.save(parsingInfo); + } } diff --git a/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java b/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java index c1a4cbd..2c1088f 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java +++ b/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java @@ -55,4 +55,8 @@ public int getActualPageURLNumber() { public ParsingInfo getLastParsedAnnouncement() { return lastParsedAnnouncement; } + + public ParsingInfoService getParsingInfoService() { + return parsingInfoService; + } } diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java b/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java index e49a0a7..e59c312 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java +++ b/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java @@ -32,16 +32,22 @@ public WalkerState run() { Document pageDocument = (Document) pageWithAnnouncement.get("pageDocument"); int divNumber = (int) pageWithAnnouncement.get("divNumber"); List urlsToParse = providerHelper.getUrlsToParse(pageDocument, divNumber); + Integer counterPerDate = providerHelper.getLastParsedAnnouncement().getCounterPerDate(); log.info("Number of pages to parse: " + urlsToParse.size()); try { ExecutorService executorService = Executors.newFixedThreadPool(THREAD_POOL_SIZE); - for (String urlToParse : urlsToParse) { + for (int i = 0; i < urlsToParse.size(); ++i) { + String url = urlsToParse.get(i); + int finalI = i; executorService.submit(() -> { - Announcement announcement = announcementParser.parsePage(urlToParse); + int counterPerPageForParsed = counterPerDate + urlsToParse.size() - finalI; + Announcement announcement = announcementParser.parsePage(url); + if (announcement != null) { + providerHelper.getParsingInfoService().saveAnnouncementToRegistry("GUMTREE", announcement, counterPerPageForParsed); + } }); - // TODO: Save record to DB_PARSER with valid counter } executorService.shutdown(); executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); From ef4f68c45de8ebee3b005593044d850f098df565 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sat, 22 Dec 2018 19:38:00 +0100 Subject: [PATCH 094/135] Code refactoring. #10 --- announcement-processor-parser/Dockerfile | 2 +- .../parser/scrapper/GumtreeAnnouncementParser.java | 2 +- .../src/main/java/parser/walker/GumtreePageWalker.java | 10 ++++------ .../java/parser/walker/states/ProcessPageState.java | 1 + .../main/java/parser/walker/states/WalkerState.java | 5 ----- docker-compose.yml | 2 +- 6 files changed, 8 insertions(+), 14 deletions(-) diff --git a/announcement-processor-parser/Dockerfile b/announcement-processor-parser/Dockerfile index 084b583..05c54e2 100644 --- a/announcement-processor-parser/Dockerfile +++ b/announcement-processor-parser/Dockerfile @@ -11,4 +11,4 @@ COPY src /srv/announcement-processor/parser/src RUN ["mvn", "clean", "install", "-DskipTests"] -CMD java -jar /srv/announcement-processor/parser/target/parser-service.jar \ No newline at end of file +CMD java -jar -Dspring.profiles.active=dev /srv/announcement-processor/parser/target/parser-service.jar \ No newline at end of file diff --git a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java index d5ad9a0..fcdd587 100644 --- a/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java +++ b/announcement-processor-parser/src/main/java/parser/scrapper/GumtreeAnnouncementParser.java @@ -47,7 +47,7 @@ public Announcement parsePage(String url) { } catch (IOException e) { throw new GumtreePageParseException("Cannot parser announcement from url: " + url); } finally { - log.info("Announcement from page: " + url + " has been parsed"); + log.info("Announcement from url: " + url.split("/krakow")[1] + " has been parsed"); } } diff --git a/announcement-processor-parser/src/main/java/parser/walker/GumtreePageWalker.java b/announcement-processor-parser/src/main/java/parser/walker/GumtreePageWalker.java index d2fa7d4..39e13ee 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/GumtreePageWalker.java +++ b/announcement-processor-parser/src/main/java/parser/walker/GumtreePageWalker.java @@ -3,22 +3,20 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; -import parser.scrapper.AnnouncementParser; import parser.scrapper.GumtreeAnnouncementParser; +import parser.walker.helpers.GumtreeHelper; +import parser.walker.states.FetchRegistryState; import parser.walker.states.StopState; import parser.walker.states.WalkerState; @Slf4j @Service public class GumtreePageWalker extends PageWalker { - private AnnouncementParser announcementParser; private WalkerState walkerState; - public GumtreePageWalker(@Qualifier("fetchRegistryState") WalkerState walkerState, + public GumtreePageWalker(@Qualifier("gumtreeHelper") GumtreeHelper gumtreeHelper, GumtreeAnnouncementParser gumtreeAnnouncementParser) { - this.walkerState = walkerState; - this.announcementParser = gumtreeAnnouncementParser; - walkerState.setAnnouncementParser(gumtreeAnnouncementParser); + this.walkerState = new FetchRegistryState(gumtreeHelper, gumtreeAnnouncementParser); } @Override diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java b/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java index e59c312..1c2949e 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java +++ b/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java @@ -46,6 +46,7 @@ public WalkerState run() { Announcement announcement = announcementParser.parsePage(url); if (announcement != null) { providerHelper.getParsingInfoService().saveAnnouncementToRegistry("GUMTREE", announcement, counterPerPageForParsed); + // TODO: Send announcement via JMS } }); } diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/WalkerState.java b/announcement-processor-parser/src/main/java/parser/walker/states/WalkerState.java index 55d0587..78ffd02 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/states/WalkerState.java +++ b/announcement-processor-parser/src/main/java/parser/walker/states/WalkerState.java @@ -2,7 +2,6 @@ import org.springframework.stereotype.Component; import parser.scrapper.AnnouncementParser; -import parser.scrapper.GumtreeAnnouncementParser; import parser.walker.helpers.ProviderHelper; @Component @@ -20,8 +19,4 @@ public WalkerState(ProviderHelper providerHelper, AnnouncementParser announcemen } public abstract WalkerState run(); - - public void setAnnouncementParser(GumtreeAnnouncementParser gumtreeAnnouncementParser){ - this.announcementParser = gumtreeAnnouncementParser; - } } diff --git a/docker-compose.yml b/docker-compose.yml index baf784d..10aeb27 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,7 +25,7 @@ services: - 8081:8081 volumes: - ./announcement-processor-parser/src:/srv/announcement-processor/parser/src - command: java -jar ./target/parser-service.jar + command: java -jar -Dspring.profiles.active=dev ./target/parser-service.jar api: build: ./announcement-processor-api From 1a227e31d66ca2346258a5b8236c8668c897f41f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sat, 22 Dec 2018 22:58:55 +0100 Subject: [PATCH 095/135] Fixed fetching registry state when no record in database. #10 --- .../main/java/parser/walker/states/FetchRegistryState.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/FetchRegistryState.java b/announcement-processor-parser/src/main/java/parser/walker/states/FetchRegistryState.java index 89e9cf0..c1e9f3c 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/states/FetchRegistryState.java +++ b/announcement-processor-parser/src/main/java/parser/walker/states/FetchRegistryState.java @@ -22,15 +22,16 @@ public FetchRegistryState(ProviderHelper providerHelper, AnnouncementParser anno @Override public WalkerState run() { Optional parsingInfoToFind = providerHelper.fetchLatestRecord(); + WalkerState state; if (parsingInfoToFind.isPresent()) { log.info("Url fetched: " + parsingInfoToFind.get().getUrl()); + state = new FindUrlState(providerHelper, announcementParser, parsingInfoToFind.get()); } else { log.info("Url cannot be fetched."); + state = new StopState(providerHelper); } - return parsingInfoToFind - .map(parsingInfo -> new FindUrlState(providerHelper, announcementParser, parsingInfo)) - .orElseGet(() -> new FindUrlState(providerHelper, announcementParser, null)); + return state; } } \ No newline at end of file From 1cb4d9083cfcac0b84c7537bf32bd9e3a1942084 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sat, 22 Dec 2018 22:59:34 +0100 Subject: [PATCH 096/135] VisibleAfterReloadState is done. #10 --- .../parser/walker/helpers/GumtreeHelper.java | 21 +++++++++++++++++++ .../parser/walker/helpers/ProviderHelper.java | 4 ++++ .../walker/states/ProcessPageState.java | 1 + .../states/VisibleAfterReloadState.java | 14 ++++++++++++- 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java b/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java index 5c56f84..270d903 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java +++ b/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java @@ -85,6 +85,19 @@ public List getUrlsToParse(Document document, int divNumber) { .collect(Collectors.toList()); } + @Override + public List getAllUrlsOnPage() { + try { + Document document = getPageAsDocumentFromUrl(actualPageURL); + Elements elements = getElementsWithDataFromPage(document); + + return elements.stream().map(this::getPageUrlFromElement).collect(Collectors.toList()); + } catch (IOException e) { + log.error("Error in getAllUrlsOnPage"); + return null; + } + } + LocalDate getEarliestDateOnAnnouncementPage(Document scannedPage) { Elements elementsWithData = getElementsWithDataFromPage(scannedPage); LocalDateTime actualDateTime = LocalDateTime.now(); @@ -161,4 +174,12 @@ String getNextPageUrl(String previousUrl, int nextPageNumber) { String[] splitted = previousUrl.split("(krakow/)(page-[0-9]+/)*"); return splitted[0] + "krakow/page-" + nextPageNumber + "/" + splitted[1].replaceAll("p[0-9]+", "p" + nextPageNumber); } + + @Override + public void updateLastParsedAnnouncement() { + Optional lastParsed = parsingInfoService + .fetchLastRecordForProvider("GUMTREE", null); + + lastParsed.ifPresent(parsingInfo -> lastParsedAnnouncement = parsingInfo); + } } diff --git a/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java b/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java index 2c1088f..922bc8f 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java +++ b/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java @@ -34,6 +34,10 @@ Document getPageAsDocumentFromUrl(String url) throws IOException { public abstract List getUrlsToParse(Document document, int divNumber); + public abstract List getAllUrlsOnPage(); + + public abstract void updateLastParsedAnnouncement(); + abstract HashMap findAnnouncementUrlOnPage(Document page, String url); abstract Elements getElementsWithDataFromPage(Document page); diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java b/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java index 1c2949e..c255469 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java +++ b/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java @@ -55,6 +55,7 @@ public WalkerState run() { } catch (InterruptedException e) { log.error("Interrupted exception"); } + providerHelper.updateLastParsedAnnouncement(); log.info("Parsing urls from page " + providerHelper.getActualPageURLNumber() + " done. Go to VisibleAfterReloadState"); return new VisibleAfterReloadState(providerHelper); diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/VisibleAfterReloadState.java b/announcement-processor-parser/src/main/java/parser/walker/states/VisibleAfterReloadState.java index 31d137c..8948c12 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/states/VisibleAfterReloadState.java +++ b/announcement-processor-parser/src/main/java/parser/walker/states/VisibleAfterReloadState.java @@ -5,6 +5,8 @@ import org.springframework.stereotype.Component; import parser.walker.helpers.ProviderHelper; +import java.util.List; + @Component @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) public class VisibleAfterReloadState extends WalkerState { @@ -15,6 +17,16 @@ public VisibleAfterReloadState(ProviderHelper providerHelper) { @Override public WalkerState run() { - return null; + List allUrls = providerHelper.getAllUrlsOnPage(); + boolean isStillOnSamePage = allUrls.contains(providerHelper.getLastParsedAnnouncement().getUrl()); + WalkerState state; + + if (isStillOnSamePage) { + state = new TopElementState(providerHelper); + } else { + state = new FetchRegistryState(providerHelper, announcementParser); + } + + return state; } } From 563e8ed390b57a2a356bbc6e090e5e1044a86f25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 23 Dec 2018 14:52:44 +0100 Subject: [PATCH 097/135] Tests update. #10 --- .../parser/scrapper/GumtreeParserRequiredFieldsTest.java | 2 +- .../test/java/parser/walker/helpers/GumtreeHelperTest.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserRequiredFieldsTest.java b/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserRequiredFieldsTest.java index d2d0308..087df6c 100644 --- a/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserRequiredFieldsTest.java +++ b/announcement-processor-parser/src/test/java/parser/scrapper/GumtreeParserRequiredFieldsTest.java @@ -39,7 +39,7 @@ public void throwGumtreePageParseExceptionInParsePage() throws IOException { Mockito.doThrow(GumtreePageParseException.class).when(spy).getPageContent(Mockito.anyString()); // then - spy.parsePage("test"); + spy.parsePage("https://www.gumtree.pl/a-mieszkania-i-domy-do-wynajecia/krakow/ul-kapelanka-d%C4%99bniki-trzy-pokoje-61-m2/1003845275080911021282609"); } @Test diff --git a/announcement-processor-parser/src/test/java/parser/walker/helpers/GumtreeHelperTest.java b/announcement-processor-parser/src/test/java/parser/walker/helpers/GumtreeHelperTest.java index f790085..8d7d679 100644 --- a/announcement-processor-parser/src/test/java/parser/walker/helpers/GumtreeHelperTest.java +++ b/announcement-processor-parser/src/test/java/parser/walker/helpers/GumtreeHelperTest.java @@ -16,7 +16,6 @@ import java.time.LocalDate; import java.time.LocalDateTime; -import java.time.Month; import java.util.HashMap; import java.util.List; import java.util.Optional; @@ -94,13 +93,14 @@ public void getEarliestDateOnAnnouncementPage() { // given String resourcePath = "/gumtree/walker/announcements-list.html"; Document documentToTest = ReaderUtil.getDocumentToTest(resourcePath); + LocalDate localDate = LocalDate.now(); // when LocalDate returnedDate = gumtreeHelper.getEarliestDateOnAnnouncementPage(documentToTest); // then - Assert.assertEquals(Month.DECEMBER, returnedDate.getMonth()); - Assert.assertEquals(10, returnedDate.getDayOfMonth()); + Assert.assertEquals(localDate.getMonth(), returnedDate.getMonth()); + Assert.assertEquals(localDate.getDayOfMonth(), returnedDate.getDayOfMonth()); } @Test From 7aa6cff8e3021abd562247f617020cf8bec9cf45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Mon, 24 Dec 2018 15:36:30 +0100 Subject: [PATCH 098/135] Code refactoring. #10 --- .../parser/walker/helpers/GumtreeHelper.java | 41 ++++++++----------- .../parser/walker/helpers/ProviderHelper.java | 20 ++++----- .../parser/walker/helpers/WalkerInfo.java | 14 +++++++ .../parser/walker/states/FindUrlState.java | 12 +++--- .../walker/states/ProcessPageState.java | 11 ++--- .../walker/helpers/GumtreeHelperTest.java | 15 +++---- 6 files changed, 57 insertions(+), 56 deletions(-) create mode 100644 announcement-processor-parser/src/main/java/parser/walker/helpers/WalkerInfo.java diff --git a/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java b/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java index 270d903..1ab9897 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java +++ b/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java @@ -12,7 +12,6 @@ import java.time.LocalDate; import java.time.LocalDateTime; import java.util.Comparator; -import java.util.HashMap; import java.util.List; import java.util.Optional; import java.util.regex.Matcher; @@ -38,13 +37,12 @@ public Optional fetchLatestRecord() { } @Override - public Optional> findPageWithAnnouncement(ParsingInfo parsingInfoToFind) { - HashMap returnValues = null; - + public Optional findPageWithAnnouncement(ParsingInfo parsingInfoToFind) { if (parsingInfoToFind != null) { try { String urlToScan = null; - HashMap pageToBeFound = null; + int requestedAnnouncementDivNumber = -1; + Document scannedPage = null; int scannedPageNumber = 0; int totalPage = -1; LocalDate date; @@ -52,27 +50,27 @@ public Optional> findPageWithAnnouncement(ParsingInfo pa do { scannedPageNumber++; urlToScan = getNextPageUrl(urlToScan, scannedPageNumber); - Document scannedPage = getPageAsDocumentFromUrl(urlToScan); + scannedPage = getPageAsDocumentFromUrl(urlToScan); date = getEarliestDateOnAnnouncementPage(scannedPage); if (date != null && !date.isBefore(parsingInfoToFind.getDate())) { - pageToBeFound = findAnnouncementUrlOnPage(scannedPage, parsingInfoToFind.getUrl()); + requestedAnnouncementDivNumber = findAnnouncementDivNumberOnPage(scannedPage, parsingInfoToFind.getUrl()); totalPage = getNumberOfTotalPages(scannedPage); } } - while (pageToBeFound == null && date != null && !date.isBefore(parsingInfoToFind.getDate()) && scannedPageNumber <= totalPage); - - if (pageToBeFound != null) { - returnValues = new HashMap<>(); - actualPageURL = urlToScan; - actualPageURLNumber = scannedPageNumber; - returnValues.put("pageDocument", pageToBeFound.get("pageDocument")); - returnValues.put("divNumber", pageToBeFound.get("divNumber")); + while (requestedAnnouncementDivNumber == -1 && date != null && !date.isBefore(parsingInfoToFind.getDate()) && scannedPageNumber <= totalPage); + + if (requestedAnnouncementDivNumber != -1) { + walkerInfo = new WalkerInfo(); + walkerInfo.setWalkPageUrl(urlToScan); + walkerInfo.setWalkPageUrlNumber(scannedPageNumber); + walkerInfo.setRequestedAnnouncementDivNumber(requestedAnnouncementDivNumber); + walkerInfo.setWalkPageDocument(scannedPage); } } catch (IOException e) { log.error("Error during finding page with announcement with url: " + parsingInfoToFind.getUrl()); } } - return Optional.ofNullable(returnValues); + return Optional.ofNullable(walkerInfo); } @Override @@ -88,7 +86,7 @@ public List getUrlsToParse(Document document, int divNumber) { @Override public List getAllUrlsOnPage() { try { - Document document = getPageAsDocumentFromUrl(actualPageURL); + Document document = getPageAsDocumentFromUrl(walkerInfo.getWalkPageUrl()); Elements elements = getElementsWithDataFromPage(document); return elements.stream().map(this::getPageUrlFromElement).collect(Collectors.toList()); @@ -140,18 +138,15 @@ String getPageUrlFromElement(Element element) { } @Override - HashMap findAnnouncementUrlOnPage(Document page, String url) { + int findAnnouncementDivNumberOnPage(Document page, String url) { Elements liElementsWithData = getElementsWithDataFromPage(page); for (int i = 0; i < liElementsWithData.size(); ++i) { String elementUrl = getPageUrlFromElement(liElementsWithData.get(i)); if (elementUrl.equals(url)) { - HashMap returnMap = new HashMap<>(); - returnMap.put("pageDocument", page); - returnMap.put("divNumber", i); - return returnMap; + return i; } } - return null; + return -1; } @Override diff --git a/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java b/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java index 922bc8f..64c3a00 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java +++ b/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java @@ -9,16 +9,14 @@ import parser.registry.ParsingInfoService; import java.io.IOException; -import java.util.HashMap; import java.util.List; import java.util.Optional; @Component public abstract class ProviderHelper { ParsingInfoService parsingInfoService; - String actualPageURL; - int actualPageURLNumber; ParsingInfo lastParsedAnnouncement; + WalkerInfo walkerInfo; public ProviderHelper(ParsingInfoService parsingInfoService) { this.parsingInfoService = parsingInfoService; @@ -30,7 +28,7 @@ Document getPageAsDocumentFromUrl(String url) throws IOException { public abstract Optional fetchLatestRecord(); - public abstract Optional> findPageWithAnnouncement(ParsingInfo parsingInfoToFind); + public abstract Optional findPageWithAnnouncement(ParsingInfo parsingInfoToFind); public abstract List getUrlsToParse(Document document, int divNumber); @@ -38,7 +36,7 @@ Document getPageAsDocumentFromUrl(String url) throws IOException { public abstract void updateLastParsedAnnouncement(); - abstract HashMap findAnnouncementUrlOnPage(Document page, String url); + abstract int findAnnouncementDivNumberOnPage(Document page, String url); abstract Elements getElementsWithDataFromPage(Document page); @@ -48,14 +46,6 @@ Document getPageAsDocumentFromUrl(String url) throws IOException { abstract int getNumberOfTotalPages(Document scannedPage); - public String getActualPageURL() { - return actualPageURL; - } - - public int getActualPageURLNumber() { - return actualPageURLNumber; - } - public ParsingInfo getLastParsedAnnouncement() { return lastParsedAnnouncement; } @@ -63,4 +53,8 @@ public ParsingInfo getLastParsedAnnouncement() { public ParsingInfoService getParsingInfoService() { return parsingInfoService; } + + public WalkerInfo getWalkerInfo() { + return walkerInfo; + } } diff --git a/announcement-processor-parser/src/main/java/parser/walker/helpers/WalkerInfo.java b/announcement-processor-parser/src/main/java/parser/walker/helpers/WalkerInfo.java new file mode 100644 index 0000000..dddd8ae --- /dev/null +++ b/announcement-processor-parser/src/main/java/parser/walker/helpers/WalkerInfo.java @@ -0,0 +1,14 @@ +package parser.walker.helpers; + +import lombok.Getter; +import lombok.Setter; +import org.jsoup.nodes.Document; + +@Getter +@Setter +public class WalkerInfo { + private int walkPageUrlNumber; + private String walkPageUrl; + private Document walkPageDocument; + private int requestedAnnouncementDivNumber; +} \ No newline at end of file diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/FindUrlState.java b/announcement-processor-parser/src/main/java/parser/walker/states/FindUrlState.java index 5334e4d..eed8333 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/states/FindUrlState.java +++ b/announcement-processor-parser/src/main/java/parser/walker/states/FindUrlState.java @@ -4,8 +4,8 @@ import parser.registry.ParsingInfo; import parser.scrapper.AnnouncementParser; import parser.walker.helpers.ProviderHelper; +import parser.walker.helpers.WalkerInfo; -import java.util.HashMap; import java.util.Optional; @Slf4j @@ -20,16 +20,16 @@ public class FindUrlState extends WalkerState { @Override public WalkerState run() { - Optional> pageWithAnnouncement = providerHelper.findPageWithAnnouncement(parsingInfoToFind); + Optional walkerInfo = providerHelper.findPageWithAnnouncement(parsingInfoToFind); - if (pageWithAnnouncement.isPresent()) { - log.info("Page has been found on page number: " + providerHelper.getActualPageURLNumber() + ". Go to ProcessPageState"); + if (walkerInfo.isPresent()) { + log.info("Page has been found on page number: " + providerHelper.getWalkerInfo().getWalkPageUrlNumber() + ". Go to ProcessPageState"); } else { log.info("Page not found. Go to FetchRegistryState"); } - return pageWithAnnouncement - .map(stringObjectHashMap -> new ProcessPageState(providerHelper, announcementParser, stringObjectHashMap)) + return walkerInfo + .map(walker -> new ProcessPageState(providerHelper, announcementParser)) .orElseGet(() -> new FetchRegistryState(providerHelper, announcementParser)); } } diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java b/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java index c255469..9cadebb 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java +++ b/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java @@ -9,7 +9,6 @@ import parser.scrapper.AnnouncementParser; import parser.walker.helpers.ProviderHelper; -import java.util.HashMap; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -19,18 +18,16 @@ @Component @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) public class ProcessPageState extends WalkerState { - private HashMap pageWithAnnouncement; private final int THREAD_POOL_SIZE = 10; - public ProcessPageState(ProviderHelper providerHelper, AnnouncementParser announcementParser, HashMap pageWithAnnouncement) { + public ProcessPageState(ProviderHelper providerHelper, AnnouncementParser announcementParser) { super(providerHelper, announcementParser); - this.pageWithAnnouncement = pageWithAnnouncement; } @Override public WalkerState run() { - Document pageDocument = (Document) pageWithAnnouncement.get("pageDocument"); - int divNumber = (int) pageWithAnnouncement.get("divNumber"); + Document pageDocument = providerHelper.getWalkerInfo().getWalkPageDocument(); + int divNumber = providerHelper.getWalkerInfo().getRequestedAnnouncementDivNumber(); List urlsToParse = providerHelper.getUrlsToParse(pageDocument, divNumber); Integer counterPerDate = providerHelper.getLastParsedAnnouncement().getCounterPerDate(); @@ -56,7 +53,7 @@ public WalkerState run() { log.error("Interrupted exception"); } providerHelper.updateLastParsedAnnouncement(); - log.info("Parsing urls from page " + providerHelper.getActualPageURLNumber() + " done. Go to VisibleAfterReloadState"); + log.info("Parsing urls from page " + providerHelper.getWalkerInfo().getWalkPageUrlNumber() + " done. Go to VisibleAfterReloadState"); return new VisibleAfterReloadState(providerHelper); } diff --git a/announcement-processor-parser/src/test/java/parser/walker/helpers/GumtreeHelperTest.java b/announcement-processor-parser/src/test/java/parser/walker/helpers/GumtreeHelperTest.java index 8d7d679..6df89ed 100644 --- a/announcement-processor-parser/src/test/java/parser/walker/helpers/GumtreeHelperTest.java +++ b/announcement-processor-parser/src/test/java/parser/walker/helpers/GumtreeHelperTest.java @@ -16,7 +16,6 @@ import java.time.LocalDate; import java.time.LocalDateTime; -import java.util.HashMap; import java.util.List; import java.util.Optional; @@ -39,7 +38,7 @@ public void findPageWithAnnouncementWhenParsingInfoIsNull() { // given // when - Optional> pageWithAnnouncement = gumtreeHelper.findPageWithAnnouncement(null); + Optional pageWithAnnouncement = gumtreeHelper.findPageWithAnnouncement(null); // then Assert.assertEquals(Optional.empty(), pageWithAnnouncement); @@ -53,11 +52,13 @@ public void findAnnouncementUrlOnPageIsNotNull() { Document documentToTest = ReaderUtil.getDocumentToTestWithBaseUrl(resourcePath, baseUrl); // when - HashMap announcementUrlOnPage = gumtreeHelper.findAnnouncementUrlOnPage(documentToTest, - "https://www.gumtree.pl/a-mieszkania-i-domy-do-wynajecia/krakow/przytulna-kawalerka-do-wynaj%C4%99cia/1003754341000911379548809"); + int announcementDivNumberOnPage = gumtreeHelper.findAnnouncementDivNumberOnPage( + documentToTest, + "https://www.gumtree.pl/a-mieszkania-i-domy-do-wynajecia/krakow/przytulna-kawalerka-do-wynaj%C4%99cia/1003754341000911379548809" + ); // then - Assert.assertNotNull(announcementUrlOnPage); + Assert.assertNotEquals(-1, announcementDivNumberOnPage); } @Test @@ -68,11 +69,11 @@ public void findAnnouncementUrlOnPageIsNull() { Document documentToTest = ReaderUtil.getDocumentToTestWithBaseUrl(resourcePath, baseUrl); // when - HashMap announcementUrlOnPage = gumtreeHelper.findAnnouncementUrlOnPage(documentToTest, + int announcementDivNumberOnPage = gumtreeHelper.findAnnouncementDivNumberOnPage(documentToTest, "https://www.gumtree.pl/a-mieszkania-i-domy-do-wynajecia/krakow/123213213dsadasdsa9"); // then - Assert.assertNull(announcementUrlOnPage); + Assert.assertEquals(-1, announcementDivNumberOnPage); } @Test From 1f85b7c459c54c7374ab0e475e0552fc9fc7462e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Mon, 24 Dec 2018 15:36:30 +0100 Subject: [PATCH 099/135] TopElementState is done. #10 --- .../parser/walker/helpers/GumtreeHelper.java | 1 + .../parser/walker/states/NewerPageState.java | 10 ++---- .../walker/states/ProcessPageState.java | 2 +- .../parser/walker/states/TopElementState.java | 22 ++++++++----- .../states/VisibleAfterReloadState.java | 33 ++++++++++++------- 5 files changed, 41 insertions(+), 27 deletions(-) diff --git a/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java b/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java index 1ab9897..b01df22 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java +++ b/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java @@ -87,6 +87,7 @@ public List getUrlsToParse(Document document, int divNumber) { public List getAllUrlsOnPage() { try { Document document = getPageAsDocumentFromUrl(walkerInfo.getWalkPageUrl()); + walkerInfo.setWalkPageDocument(document); Elements elements = getElementsWithDataFromPage(document); return elements.stream().map(this::getPageUrlFromElement).collect(Collectors.toList()); diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/NewerPageState.java b/announcement-processor-parser/src/main/java/parser/walker/states/NewerPageState.java index 6d4ffb8..2c279ea 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/states/NewerPageState.java +++ b/announcement-processor-parser/src/main/java/parser/walker/states/NewerPageState.java @@ -1,16 +1,12 @@ package parser.walker.states; -import org.springframework.beans.factory.config.ConfigurableBeanFactory; -import org.springframework.context.annotation.Scope; -import org.springframework.stereotype.Component; +import parser.scrapper.AnnouncementParser; import parser.walker.helpers.ProviderHelper; -@Component -@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) public class NewerPageState extends WalkerState { - public NewerPageState(ProviderHelper providerHelper) { - super(providerHelper); + public NewerPageState(ProviderHelper providerHelper, AnnouncementParser announcementParser) { + super(providerHelper, announcementParser); } @Override diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java b/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java index 9cadebb..250df7c 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java +++ b/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java @@ -55,6 +55,6 @@ public WalkerState run() { providerHelper.updateLastParsedAnnouncement(); log.info("Parsing urls from page " + providerHelper.getWalkerInfo().getWalkPageUrlNumber() + " done. Go to VisibleAfterReloadState"); - return new VisibleAfterReloadState(providerHelper); + return new VisibleAfterReloadState(providerHelper, announcementParser); } } diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/TopElementState.java b/announcement-processor-parser/src/main/java/parser/walker/states/TopElementState.java index b021bee..5c051b9 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/states/TopElementState.java +++ b/announcement-processor-parser/src/main/java/parser/walker/states/TopElementState.java @@ -1,20 +1,26 @@ package parser.walker.states; -import org.springframework.beans.factory.config.ConfigurableBeanFactory; -import org.springframework.context.annotation.Scope; -import org.springframework.stereotype.Component; +import lombok.extern.slf4j.Slf4j; +import parser.scrapper.AnnouncementParser; import parser.walker.helpers.ProviderHelper; -@Component -@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) +@Slf4j public class TopElementState extends WalkerState { - public TopElementState(ProviderHelper providerHelper) { - super(providerHelper); + public TopElementState(ProviderHelper providerHelper, AnnouncementParser announcementParser) { + super(providerHelper, announcementParser); } @Override public WalkerState run() { - return null; + int divNumber = providerHelper.getWalkerInfo().getRequestedAnnouncementDivNumber(); + + if (divNumber == 0) { + log.info("Element is at the top of the page. Go to NewerPageState"); + return new NewerPageState(providerHelper, announcementParser); + } else { + log.info("Element is not at the top of the page. Go to ProcessPageState"); + return new ProcessPageState(providerHelper, announcementParser); + } } } diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/VisibleAfterReloadState.java b/announcement-processor-parser/src/main/java/parser/walker/states/VisibleAfterReloadState.java index 8948c12..7e34418 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/states/VisibleAfterReloadState.java +++ b/announcement-processor-parser/src/main/java/parser/walker/states/VisibleAfterReloadState.java @@ -1,32 +1,43 @@ package parser.walker.states; -import org.springframework.beans.factory.config.ConfigurableBeanFactory; -import org.springframework.context.annotation.Scope; -import org.springframework.stereotype.Component; +import lombok.extern.slf4j.Slf4j; +import parser.scrapper.AnnouncementParser; import parser.walker.helpers.ProviderHelper; import java.util.List; -@Component -@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) +@Slf4j public class VisibleAfterReloadState extends WalkerState { - public VisibleAfterReloadState(ProviderHelper providerHelper) { - super(providerHelper); + public VisibleAfterReloadState(ProviderHelper providerHelper, AnnouncementParser announcementParser) { + super(providerHelper, announcementParser); } @Override public WalkerState run() { - List allUrls = providerHelper.getAllUrlsOnPage(); - boolean isStillOnSamePage = allUrls.contains(providerHelper.getLastParsedAnnouncement().getUrl()); + int divIndex = getDivIndexIfInList(); WalkerState state; - if (isStillOnSamePage) { - state = new TopElementState(providerHelper); + if (divIndex != -1) { + log.info("Url is visible on page. Go to TopElementState"); + providerHelper.getWalkerInfo().setRequestedAnnouncementDivNumber(divIndex); + state = new TopElementState(providerHelper, announcementParser); } else { + log.info("Url is not visible on page. Go to FetchRegistryState"); state = new FetchRegistryState(providerHelper, announcementParser); } return state; } + + private int getDivIndexIfInList() { + String desiredUrl = providerHelper.getLastParsedAnnouncement().getUrl(); + List allUrlsOnPage = providerHelper.getAllUrlsOnPage(); + for (int i = 0; i < allUrlsOnPage.size(); i++) { + if (allUrlsOnPage.get(i).equals(desiredUrl)) { + return i; + } + } + return -1; + } } From 75d0a6a12ebcc3c6044766c6fd172388d3abe505 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Fri, 4 Jan 2019 22:40:07 +0100 Subject: [PATCH 100/135] NewerPageState is done. #10 --- .../parser/walker/helpers/GumtreeHelper.java | 74 ++++++++++++++++--- .../parser/walker/helpers/ProviderHelper.java | 6 +- .../parser/walker/states/NewerPageState.java | 15 +++- .../states/VisibleAfterReloadState.java | 3 +- .../walker/helpers/GumtreeHelperTest.java | 34 +++++++++ 5 files changed, 117 insertions(+), 15 deletions(-) diff --git a/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java b/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java index b01df22..09db5c6 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java +++ b/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java @@ -50,6 +50,7 @@ public Optional findPageWithAnnouncement(ParsingInfo parsingInfoToFi do { scannedPageNumber++; urlToScan = getNextPageUrl(urlToScan, scannedPageNumber); + log.debug("\tFind page on page with number " + scannedPageNumber + ": " + urlToScan); scannedPage = getPageAsDocumentFromUrl(urlToScan); date = getEarliestDateOnAnnouncementPage(scannedPage); if (date != null && !date.isBefore(parsingInfoToFind.getDate())) { @@ -75,7 +76,7 @@ public Optional findPageWithAnnouncement(ParsingInfo parsingInfoToFi @Override public List getUrlsToParse(Document document, int divNumber) { - log.info("Geeting urls to parse"); + log.debug("Geeting urls to parse"); return getElementsWithDataFromPage(document) .stream() .limit(divNumber) @@ -84,17 +85,48 @@ public List getUrlsToParse(Document document, int divNumber) { } @Override - public List getAllUrlsOnPage() { - try { - Document document = getPageAsDocumentFromUrl(walkerInfo.getWalkPageUrl()); - walkerInfo.setWalkPageDocument(document); - Elements elements = getElementsWithDataFromPage(document); - - return elements.stream().map(this::getPageUrlFromElement).collect(Collectors.toList()); - } catch (IOException e) { - log.error("Error in getAllUrlsOnPage"); - return null; + public List getAllUrlsOnPage(boolean refreshPage) { + if (refreshPage) { + try { + walkerInfo.setWalkPageDocument(getPageAsDocumentFromUrl(walkerInfo.getWalkPageUrl())); + } catch (IOException e) { + log.error("Error in getAllUrlsOnPage"); + } } + + Elements elements = getElementsWithDataFromPage(walkerInfo.getWalkPageDocument()); + return elements.stream().map(this::getPageUrlFromElement).collect(Collectors.toList()); + } + + @Override + public void goToNewerPageWithDocumentUpdate() { + String previousPageUrl = getPreviousPageUrl(walkerInfo.getWalkPageUrl()); + if (previousPageUrl != null) { + try { + walkerInfo.setWalkPageUrl(previousPageUrl); + Document pageDocument = getPageAsDocumentFromUrl(previousPageUrl); + walkerInfo.setWalkPageUrlNumber(getPageNumberFromPageDocument(pageDocument)); + + String partOfUrl = lastParsedAnnouncement.getUrl().split("/krakow/")[1]; + + String fakeAnnouncement = "
  • \n" + + "\t
    \n" + + "\t\tTitle\n" + + "\t
    " + + "
    • "; + pageDocument.select(".view").get(divElementWithAnnouncementsNumber) + .selectFirst("ul").children().last().before(fakeAnnouncement); + walkerInfo.setWalkPageDocument(pageDocument); + walkerInfo.setRequestedAnnouncementDivNumber(getAllUrlsOnPage(false).size()); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + private int getPageNumberFromPageDocument(Document pageDocument) { + Element element = pageDocument.body().selectFirst(".pagination span.current"); + return Integer.valueOf(element.html()); } LocalDate getEarliestDateOnAnnouncementPage(Document scannedPage) { @@ -171,10 +203,30 @@ String getNextPageUrl(String previousUrl, int nextPageNumber) { return splitted[0] + "krakow/page-" + nextPageNumber + "/" + splitted[1].replaceAll("p[0-9]+", "p" + nextPageNumber); } + @Override + String getPreviousPageUrl(String actualUrl) { + if (actualUrl != null && actualUrl.contains("page")) { // page is > 1 + Pattern compile = Pattern.compile("(/page-)([0-9]+)(/)"); + Matcher matcher = compile.matcher(actualUrl); + + if (matcher.find()) { + int pageNumber = Integer.valueOf(matcher.group(2)); + if (pageNumber != 1) { + int previousPageNumber = pageNumber - 1; + return actualUrl + .replace("page-" + pageNumber, "page-" + previousPageNumber) + .replace("p" + pageNumber, "p" + previousPageNumber); + } + } + } + return null; + } + @Override public void updateLastParsedAnnouncement() { Optional lastParsed = parsingInfoService .fetchLastRecordForProvider("GUMTREE", null); + lastParsed.ifPresent(lp -> log.info("update last parsed: " + lp.getUrl())); lastParsed.ifPresent(parsingInfo -> lastParsedAnnouncement = parsingInfo); } diff --git a/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java b/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java index 64c3a00..e5bace0 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java +++ b/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java @@ -32,10 +32,12 @@ Document getPageAsDocumentFromUrl(String url) throws IOException { public abstract List getUrlsToParse(Document document, int divNumber); - public abstract List getAllUrlsOnPage(); + public abstract List getAllUrlsOnPage(boolean refreshPage); public abstract void updateLastParsedAnnouncement(); + public abstract void goToNewerPageWithDocumentUpdate(); + abstract int findAnnouncementDivNumberOnPage(Document page, String url); abstract Elements getElementsWithDataFromPage(Document page); @@ -44,6 +46,8 @@ Document getPageAsDocumentFromUrl(String url) throws IOException { abstract String getNextPageUrl(String previousUrl, int nextPageNumber); + abstract String getPreviousPageUrl(String actualUrl); + abstract int getNumberOfTotalPages(Document scannedPage); public ParsingInfo getLastParsedAnnouncement() { diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/NewerPageState.java b/announcement-processor-parser/src/main/java/parser/walker/states/NewerPageState.java index 2c279ea..c2a072a 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/states/NewerPageState.java +++ b/announcement-processor-parser/src/main/java/parser/walker/states/NewerPageState.java @@ -1,16 +1,27 @@ package parser.walker.states; +import lombok.extern.slf4j.Slf4j; import parser.scrapper.AnnouncementParser; import parser.walker.helpers.ProviderHelper; +@Slf4j public class NewerPageState extends WalkerState { - public NewerPageState(ProviderHelper providerHelper, AnnouncementParser announcementParser) { + NewerPageState(ProviderHelper providerHelper, AnnouncementParser announcementParser) { super(providerHelper, announcementParser); } @Override public WalkerState run() { - return null; + int walkPageUrlNumber = providerHelper.getWalkerInfo().getWalkPageUrlNumber(); + + if (walkPageUrlNumber == 1) { + log.info("It is newest page, go to StopState"); + return new StopState(providerHelper); + } + + providerHelper.goToNewerPageWithDocumentUpdate(); + log.info("It is not a newest page, go to ProcessPageState"); + return new ProcessPageState(providerHelper, announcementParser); } } diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/VisibleAfterReloadState.java b/announcement-processor-parser/src/main/java/parser/walker/states/VisibleAfterReloadState.java index 7e34418..af506e8 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/states/VisibleAfterReloadState.java +++ b/announcement-processor-parser/src/main/java/parser/walker/states/VisibleAfterReloadState.java @@ -15,6 +15,7 @@ public VisibleAfterReloadState(ProviderHelper providerHelper, AnnouncementParser @Override public WalkerState run() { + log.info("Refresh page: " + providerHelper.getWalkerInfo().getWalkPageUrl() + " => number: " + providerHelper.getWalkerInfo().getWalkPageUrlNumber()); int divIndex = getDivIndexIfInList(); WalkerState state; @@ -32,7 +33,7 @@ public WalkerState run() { private int getDivIndexIfInList() { String desiredUrl = providerHelper.getLastParsedAnnouncement().getUrl(); - List allUrlsOnPage = providerHelper.getAllUrlsOnPage(); + List allUrlsOnPage = providerHelper.getAllUrlsOnPage(true); for (int i = 0; i < allUrlsOnPage.size(); i++) { if (allUrlsOnPage.get(i).equals(desiredUrl)) { return i; diff --git a/announcement-processor-parser/src/test/java/parser/walker/helpers/GumtreeHelperTest.java b/announcement-processor-parser/src/test/java/parser/walker/helpers/GumtreeHelperTest.java index 6df89ed..d94800b 100644 --- a/announcement-processor-parser/src/test/java/parser/walker/helpers/GumtreeHelperTest.java +++ b/announcement-processor-parser/src/test/java/parser/walker/helpers/GumtreeHelperTest.java @@ -250,4 +250,38 @@ public void getNextPageUrlWhenPreviousUrlNotNull() { // then Assert.assertEquals(expectedUrl, returnedUrl); } + + @Test + public void getPreviousPageUrlWhenActualUrlIsNull() { + // when + String returnedUrl = gumtreeHelper.getPreviousPageUrl(null); + + // then + Assert.assertNull(returnedUrl); + } + + @Test + public void getPreviousPageUrlWhenActualUrlIsNotNullAndOverOne() { + // given + String previousUrl = "https://www.gumtree.pl/s-mieszkania-i-domy-do-wynajecia/krakow/page-3/v1c9008l3200208p3"; + String expectedUrl = "https://www.gumtree.pl/s-mieszkania-i-domy-do-wynajecia/krakow/page-2/v1c9008l3200208p2"; + + // when + String returnedUrl = gumtreeHelper.getPreviousPageUrl(previousUrl); + + // then + Assert.assertEquals(expectedUrl, returnedUrl); + } + + @Test + public void getPreviousPageUrlWhenActualUrlIsOne() { + // given + String previousUrl = "https://www.gumtree.pl/s-mieszkania-i-domy-do-wynajecia/krakow/page-1/v1c9008l3200208p1"; + + // when + String returnedUrl = gumtreeHelper.getPreviousPageUrl(previousUrl); + + // then + Assert.assertNull(returnedUrl); + } } From c3cc71abcdff60f20c2a7cfbb854870a78f10a6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sat, 5 Jan 2019 10:18:01 +0100 Subject: [PATCH 101/135] Added broker-dev configuration. #13 --- .../main/java/parser/AnnouncementSender.java | 40 +++++++++++++++++++ .../main/resources/application-dev.properties | 4 +- 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 announcement-processor-parser/src/main/java/parser/AnnouncementSender.java diff --git a/announcement-processor-parser/src/main/java/parser/AnnouncementSender.java b/announcement-processor-parser/src/main/java/parser/AnnouncementSender.java new file mode 100644 index 0000000..291d410 --- /dev/null +++ b/announcement-processor-parser/src/main/java/parser/AnnouncementSender.java @@ -0,0 +1,40 @@ +package parser; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.Getter; +import org.apache.activemq.ActiveMQConnectionFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import javax.jms.*; + +@Component +@Getter +public class AnnouncementSender { + @Value("${brokerURL}") + private String brokerUrl; + @Value("${queue}") + private String queueName; + + public void sendAnnouncement(Announcement announcement) { + try { + ConnectionFactory connectionFactory = new ActiveMQConnectionFactory(brokerUrl); + Connection connection = connectionFactory.createConnection(); + Session session = connection.createSession(false, Session.AUTO_ACKNOWLEDGE); + Queue queue = session.createQueue(queueName); + + MessageProducer messageProducer = session.createProducer(queue); + + ObjectMapper objectMapper = new ObjectMapper(); + + Message message = session.createTextMessage(objectMapper.writeValueAsString(announcement)); + messageProducer.send(message); + + connection.close(); + session.close(); + } catch (JMSException | JsonProcessingException e) { + e.printStackTrace(); + } + } +} diff --git a/announcement-processor-parser/src/main/resources/application-dev.properties b/announcement-processor-parser/src/main/resources/application-dev.properties index bc0481d..082b17f 100644 --- a/announcement-processor-parser/src/main/resources/application-dev.properties +++ b/announcement-processor-parser/src/main/resources/application-dev.properties @@ -10,4 +10,6 @@ spring.datasource.driver-class-name=org.h2.Driver spring.jpa.hibernate.ddl-auto=create-drop -spring.h2.console.settings.web-allow-others=true \ No newline at end of file +spring.h2.console.settings.web-allow-others=true +brokerURL=tcp://192.168.99.100:32768 +queue=announcements \ No newline at end of file From 6a3ac076f7eb99959e3cd0c44bb3a20ccf46863b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sat, 5 Jan 2019 14:00:35 +0100 Subject: [PATCH 102/135] Integration of sender with walker. #13 --- ...nnouncementProcessorParserApplication.java | 7 +++---- .../java/parser/walker/GumtreePageWalker.java | 19 ++++++++++++++----- .../main/java/parser/walker/PageWalker.java | 2 +- .../parser/walker/helpers/GumtreeHelper.java | 4 +--- .../parser/walker/helpers/ProviderHelper.java | 16 ++++++++-------- .../walker/states/ProcessPageState.java | 10 ++++++++-- .../parser/walker/states/WalkerState.java | 2 ++ 7 files changed, 37 insertions(+), 23 deletions(-) diff --git a/announcement-processor-parser/src/main/java/parser/AnnouncementProcessorParserApplication.java b/announcement-processor-parser/src/main/java/parser/AnnouncementProcessorParserApplication.java index f16e55f..56e336e 100644 --- a/announcement-processor-parser/src/main/java/parser/AnnouncementProcessorParserApplication.java +++ b/announcement-processor-parser/src/main/java/parser/AnnouncementProcessorParserApplication.java @@ -4,15 +4,14 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; -import parser.walker.PageWalker; +import org.springframework.scheduling.annotation.EnableScheduling; @EnableJpaRepositories @SpringBootApplication +@EnableScheduling public class AnnouncementProcessorParserApplication { public static void main(String[] args) { - ConfigurableApplicationContext run = SpringApplication.run(AnnouncementProcessorParserApplication.class, args); - PageWalker pageWalker = (PageWalker) run.getBean("gumtreePageWalker"); - pageWalker.walk("www.example.com"); + ConfigurableApplicationContext ctx = SpringApplication.run(AnnouncementProcessorParserApplication.class, args); } } diff --git a/announcement-processor-parser/src/main/java/parser/walker/GumtreePageWalker.java b/announcement-processor-parser/src/main/java/parser/walker/GumtreePageWalker.java index 39e13ee..f48dc20 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/GumtreePageWalker.java +++ b/announcement-processor-parser/src/main/java/parser/walker/GumtreePageWalker.java @@ -2,7 +2,9 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; +import parser.AnnouncementSender; import parser.scrapper.GumtreeAnnouncementParser; import parser.walker.helpers.GumtreeHelper; import parser.walker.states.FetchRegistryState; @@ -12,18 +14,25 @@ @Slf4j @Service public class GumtreePageWalker extends PageWalker { - private WalkerState walkerState; + private WalkerState startState; public GumtreePageWalker(@Qualifier("gumtreeHelper") GumtreeHelper gumtreeHelper, - GumtreeAnnouncementParser gumtreeAnnouncementParser) { - this.walkerState = new FetchRegistryState(gumtreeHelper, gumtreeAnnouncementParser); + GumtreeAnnouncementParser gumtreeAnnouncementParser, + AnnouncementSender announcementSender) { + gumtreeHelper.setAnnouncementSender(announcementSender); + this.startState = new FetchRegistryState(gumtreeHelper, gumtreeAnnouncementParser); } @Override - public void walk(String startPageUrl) { + @Scheduled(fixedDelay = 300000) + public void walk() { + log.info(""); + log.info("==============> Scheduled task started walker <=================================="); + WalkerState walkerState = startState; + walkerState.getProviderHelper().setLastParsedAnnouncement(null); while(walkerState != null && !(walkerState instanceof StopState)){ walkerState = walkerState.run(); } - log.info("GumtreePageWalker has parsed all pages"); + log.info("==============> GumtreePageWalker has parsed all pages <========================"); } } diff --git a/announcement-processor-parser/src/main/java/parser/walker/PageWalker.java b/announcement-processor-parser/src/main/java/parser/walker/PageWalker.java index 9c95742..69af9e6 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/PageWalker.java +++ b/announcement-processor-parser/src/main/java/parser/walker/PageWalker.java @@ -1,5 +1,5 @@ package parser.walker; public abstract class PageWalker { - public abstract void walk(String startPageUrl); + public abstract void walk(); } diff --git a/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java b/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java index 09db5c6..f58992d 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java +++ b/announcement-processor-parser/src/main/java/parser/walker/helpers/GumtreeHelper.java @@ -42,7 +42,7 @@ public Optional findPageWithAnnouncement(ParsingInfo parsingInfoToFi try { String urlToScan = null; int requestedAnnouncementDivNumber = -1; - Document scannedPage = null; + Document scannedPage; int scannedPageNumber = 0; int totalPage = -1; LocalDate date; @@ -76,7 +76,6 @@ public Optional findPageWithAnnouncement(ParsingInfo parsingInfoToFi @Override public List getUrlsToParse(Document document, int divNumber) { - log.debug("Geeting urls to parse"); return getElementsWithDataFromPage(document) .stream() .limit(divNumber) @@ -93,7 +92,6 @@ public List getAllUrlsOnPage(boolean refreshPage) { log.error("Error in getAllUrlsOnPage"); } } - Elements elements = getElementsWithDataFromPage(walkerInfo.getWalkPageDocument()); return elements.stream().map(this::getPageUrlFromElement).collect(Collectors.toList()); } diff --git a/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java b/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java index e5bace0..36ec956 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java +++ b/announcement-processor-parser/src/main/java/parser/walker/helpers/ProviderHelper.java @@ -1,10 +1,12 @@ package parser.walker.helpers; +import lombok.Getter; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import org.springframework.stereotype.Component; +import parser.AnnouncementSender; import parser.registry.ParsingInfo; import parser.registry.ParsingInfoService; @@ -12,11 +14,13 @@ import java.util.List; import java.util.Optional; +@Getter @Component public abstract class ProviderHelper { ParsingInfoService parsingInfoService; ParsingInfo lastParsedAnnouncement; WalkerInfo walkerInfo; + AnnouncementSender announcementSender; public ProviderHelper(ParsingInfoService parsingInfoService) { this.parsingInfoService = parsingInfoService; @@ -50,15 +54,11 @@ Document getPageAsDocumentFromUrl(String url) throws IOException { abstract int getNumberOfTotalPages(Document scannedPage); - public ParsingInfo getLastParsedAnnouncement() { - return lastParsedAnnouncement; + public void setAnnouncementSender(AnnouncementSender announcementSender) { + this.announcementSender = announcementSender; } - public ParsingInfoService getParsingInfoService() { - return parsingInfoService; - } - - public WalkerInfo getWalkerInfo() { - return walkerInfo; + public void setLastParsedAnnouncement(ParsingInfo lastParsedAnnouncement) { + this.lastParsedAnnouncement = lastParsedAnnouncement; } } diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java b/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java index 250df7c..c0dca39 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java +++ b/announcement-processor-parser/src/main/java/parser/walker/states/ProcessPageState.java @@ -41,9 +41,9 @@ public WalkerState run() { executorService.submit(() -> { int counterPerPageForParsed = counterPerDate + urlsToParse.size() - finalI; Announcement announcement = announcementParser.parsePage(url); - if (announcement != null) { + if (announcement != null && isDifferentThanLastParsed(announcement)) { providerHelper.getParsingInfoService().saveAnnouncementToRegistry("GUMTREE", announcement, counterPerPageForParsed); - // TODO: Send announcement via JMS + providerHelper.getAnnouncementSender().sendAnnouncement(announcement); } }); } @@ -57,4 +57,10 @@ public WalkerState run() { return new VisibleAfterReloadState(providerHelper, announcementParser); } + + private boolean isDifferentThanLastParsed(Announcement announcement) { + String lastParsedHash = providerHelper.getLastParsedAnnouncement().getPageHash(); + String lastParsedUrl = providerHelper.getLastParsedAnnouncement().getUrl(); + return !lastParsedUrl.equals(announcement.getUrl()) && !lastParsedHash.equals("" + announcement.hashCode()); + } } diff --git a/announcement-processor-parser/src/main/java/parser/walker/states/WalkerState.java b/announcement-processor-parser/src/main/java/parser/walker/states/WalkerState.java index 78ffd02..fed71db 100644 --- a/announcement-processor-parser/src/main/java/parser/walker/states/WalkerState.java +++ b/announcement-processor-parser/src/main/java/parser/walker/states/WalkerState.java @@ -1,9 +1,11 @@ package parser.walker.states; +import lombok.Getter; import org.springframework.stereotype.Component; import parser.scrapper.AnnouncementParser; import parser.walker.helpers.ProviderHelper; +@Getter @Component public abstract class WalkerState { ProviderHelper providerHelper; From a2d279688ea0a0e1cc6188503e1b3660c81cab9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sat, 5 Jan 2019 14:52:29 +0100 Subject: [PATCH 103/135] Fix for LocalDateTime format. #13 --- .../src/main/java/parser/AnnouncementSender.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/announcement-processor-parser/src/main/java/parser/AnnouncementSender.java b/announcement-processor-parser/src/main/java/parser/AnnouncementSender.java index 291d410..3501275 100644 --- a/announcement-processor-parser/src/main/java/parser/AnnouncementSender.java +++ b/announcement-processor-parser/src/main/java/parser/AnnouncementSender.java @@ -2,6 +2,8 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import lombok.Getter; import org.apache.activemq.ActiveMQConnectionFactory; import org.springframework.beans.factory.annotation.Value; @@ -27,6 +29,8 @@ public void sendAnnouncement(Announcement announcement) { MessageProducer messageProducer = session.createProducer(queue); ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.registerModule(new JavaTimeModule()); + objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); Message message = session.createTextMessage(objectMapper.writeValueAsString(announcement)); messageProducer.send(message); From 0f3344ce985ceead66a9325b3288b5d9e3ad4c1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 6 Jan 2019 16:05:07 +0100 Subject: [PATCH 104/135] Table definitions for database --- .../main/java/api/entity/AdditionalCosts.java | 26 +++++++++ .../main/java/api/entity/Announcement.java | 56 +++++++++++++++++++ .../src/main/java/api/entity/Currency.java | 13 +++++ .../src/main/java/api/entity/Lessor.java | 30 ++++++++++ .../src/main/java/api/entity/Location.java | 39 +++++++++++++ .../src/main/java/api/entity/PriceOffer.java | 30 ++++++++++ .../main/java/api/entity/PropertyData.java | 45 +++++++++++++++ .../repository/AdditionalCostsRepository.java | 7 +++ .../repository/AnnouncementRepository.java | 7 +++ .../java/api/repository/LessorRepository.java | 7 +++ .../api/repository/LocationRepository.java | 7 +++ .../api/repository/PriceOfferRepository.java | 7 +++ .../repository/PropertyDataRepository.java | 8 +++ 13 files changed, 282 insertions(+) create mode 100644 announcement-processor-api/src/main/java/api/entity/AdditionalCosts.java create mode 100644 announcement-processor-api/src/main/java/api/entity/Announcement.java create mode 100644 announcement-processor-api/src/main/java/api/entity/Currency.java create mode 100644 announcement-processor-api/src/main/java/api/entity/Lessor.java create mode 100644 announcement-processor-api/src/main/java/api/entity/Location.java create mode 100644 announcement-processor-api/src/main/java/api/entity/PriceOffer.java create mode 100644 announcement-processor-api/src/main/java/api/entity/PropertyData.java create mode 100644 announcement-processor-api/src/main/java/api/repository/AdditionalCostsRepository.java create mode 100644 announcement-processor-api/src/main/java/api/repository/AnnouncementRepository.java create mode 100644 announcement-processor-api/src/main/java/api/repository/LessorRepository.java create mode 100644 announcement-processor-api/src/main/java/api/repository/LocationRepository.java create mode 100644 announcement-processor-api/src/main/java/api/repository/PriceOfferRepository.java create mode 100644 announcement-processor-api/src/main/java/api/repository/PropertyDataRepository.java diff --git a/announcement-processor-api/src/main/java/api/entity/AdditionalCosts.java b/announcement-processor-api/src/main/java/api/entity/AdditionalCosts.java new file mode 100644 index 0000000..5a465f5 --- /dev/null +++ b/announcement-processor-api/src/main/java/api/entity/AdditionalCosts.java @@ -0,0 +1,26 @@ +package api.entity; + +import lombok.Data; + +import javax.persistence.*; +import java.math.BigDecimal; + +@Entity +@Data +@Table(name = "ADDITIONAL_COSTS") +public class AdditionalCosts { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "ID") + private Integer id; + + @ManyToOne + @JoinColumn(name = "ANNOUNCEMENT_ID") + private Announcement announcement; + + @Column(name = "COST_NAME") + private String costName; + + @Column(name = "COST_PRICE") + private BigDecimal costPrice; +} diff --git a/announcement-processor-api/src/main/java/api/entity/Announcement.java b/announcement-processor-api/src/main/java/api/entity/Announcement.java new file mode 100644 index 0000000..2d793b9 --- /dev/null +++ b/announcement-processor-api/src/main/java/api/entity/Announcement.java @@ -0,0 +1,56 @@ +package api.entity; + +import lombok.Data; + +import javax.persistence.*; +import java.math.BigDecimal; +import java.time.LocalDateTime; +import java.util.List; + +@Entity +@Data +@Table(name = "ANNOUNCEMENTS") +public class Announcement { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "ID") + private Integer id; + + @Column(name = "TITLE") + private String title; + + @Column(name = "BASE_PRICE") + private BigDecimal basePrice; + + @Column(name = "PROVIDER") + private String provider; + + @Column(name = "CREATION_DATE") + private LocalDateTime creationDate; + + @Column(name = "URL") + private String url; + + @Column(name = "DESCRIPTION") + private String description; + + @OneToOne + @JoinColumn(name = "PRICE_ID") + private PriceOffer priceOffer; + + @OneToOne + @JoinColumn(name = "LOCATION_ID") + private Location location; + + @OneToOne + @JoinColumn(name = "PROPERTY_DATA_ID") + private PropertyData propertyData; + + @OneToOne + @JoinColumn(name = "LESSOR_ID") + private Lessor lessor; + + @OneToMany(mappedBy = "announcement") + private List additionalCosts; +} diff --git a/announcement-processor-api/src/main/java/api/entity/Currency.java b/announcement-processor-api/src/main/java/api/entity/Currency.java new file mode 100644 index 0000000..4bc470b --- /dev/null +++ b/announcement-processor-api/src/main/java/api/entity/Currency.java @@ -0,0 +1,13 @@ +package api.entity; + +import javax.persistence.*; + +@Entity +@Table(name = "CURRENCIES") +public class Currency { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "ID") + private Integer id; +} diff --git a/announcement-processor-api/src/main/java/api/entity/Lessor.java b/announcement-processor-api/src/main/java/api/entity/Lessor.java new file mode 100644 index 0000000..81be3c9 --- /dev/null +++ b/announcement-processor-api/src/main/java/api/entity/Lessor.java @@ -0,0 +1,30 @@ +package api.entity; + +import lombok.Data; + +import javax.persistence.*; + +@Entity +@Data +@Table(name = "LESSORS") +public class Lessor { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "ID") + private Integer id; + + @Column(name = "NAME") + private String name; + + @Column(name = "LESSOR_TYPE") + private String lessorType; + + @Column(name = "PHONE_NUMBER") + private String phoneNumber; + + @Column(name = "EMAIL") + private String email; + + @OneToOne(mappedBy = "lessor") + private Announcement announcement; +} diff --git a/announcement-processor-api/src/main/java/api/entity/Location.java b/announcement-processor-api/src/main/java/api/entity/Location.java new file mode 100644 index 0000000..eefd20c --- /dev/null +++ b/announcement-processor-api/src/main/java/api/entity/Location.java @@ -0,0 +1,39 @@ +package api.entity; + +import lombok.Data; + +import javax.persistence.*; + +@Entity +@Data +@Table(name = "LOCATIONS") +public class Location { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "ID") + private Integer id; + + @Column(name = "COUNTRY") + private String country; + + @Column(name = "CITY") + private String city; + + @Column(name = "STREET") + private String street; + + @Column(name = "ZIP_CODE") + private String zipCode; + + @Column(name = "BUILDING_NUMBER") + private String buildingNumber; + + @Column(name = "FLAT_NUMBER") + private String flatNumber; + + @Column(name = "DISTRICT") + private String district; + + @OneToOne(mappedBy = "location") + private Announcement announcement; +} diff --git a/announcement-processor-api/src/main/java/api/entity/PriceOffer.java b/announcement-processor-api/src/main/java/api/entity/PriceOffer.java new file mode 100644 index 0000000..148a519 --- /dev/null +++ b/announcement-processor-api/src/main/java/api/entity/PriceOffer.java @@ -0,0 +1,30 @@ +package api.entity; + +import lombok.Data; + +import javax.persistence.*; + +@Entity +@Data +@Inheritance(strategy = InheritanceType.SINGLE_TABLE) +@DiscriminatorColumn(name = "DESCRIPTOR") +@Table(name = "PRICE_OFFER") +public class PriceOffer { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "ID") + private Integer id; + + @Column(name = "DESCRIPTION") + private String description; + + @OneToOne(mappedBy = "priceOffer") + private Announcement announcement; + + public PriceOffer() { + } + + public PriceOffer(String description) { + this.description = description; + } +} diff --git a/announcement-processor-api/src/main/java/api/entity/PropertyData.java b/announcement-processor-api/src/main/java/api/entity/PropertyData.java new file mode 100644 index 0000000..77a87aa --- /dev/null +++ b/announcement-processor-api/src/main/java/api/entity/PropertyData.java @@ -0,0 +1,45 @@ +package api.entity; + +import lombok.Data; + +import javax.persistence.*; + +@Entity +@Data +@Table(name = "PROPERTY_DATA") +public class PropertyData { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "ID") + private Integer id; + + @Column(name = "PROPERTY_TYPE") + private String propertyType; + + @Column(name = "AREA") + private Double area; + + @Column(name = "IS_FOR_SMOKERS") + private Boolean isSmokingAllowed; + + @Column(name = "IS_FOR_PETS") + private Boolean isPerFriendly; + + @Column(name = "ROOM_NUMBER") + private Integer roomNumber; + + @Column(name = "BATHROOM_NUMBER") + private Integer bathroomNumber; + + @Column(name = "PARKING_AVAILABILITY") + private String parkingAvailability; + + @Column(name = "LEVEL") + private Integer level; + + @Column(name = "FURNISHING") + private String furnishing; + + @OneToOne(mappedBy = "propertyData") + private Announcement announcement; +} diff --git a/announcement-processor-api/src/main/java/api/repository/AdditionalCostsRepository.java b/announcement-processor-api/src/main/java/api/repository/AdditionalCostsRepository.java new file mode 100644 index 0000000..f42a3af --- /dev/null +++ b/announcement-processor-api/src/main/java/api/repository/AdditionalCostsRepository.java @@ -0,0 +1,7 @@ +package api.repository; + +import api.entity.AdditionalCosts; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface AdditionalCostsRepository extends JpaRepository { +} diff --git a/announcement-processor-api/src/main/java/api/repository/AnnouncementRepository.java b/announcement-processor-api/src/main/java/api/repository/AnnouncementRepository.java new file mode 100644 index 0000000..77e201b --- /dev/null +++ b/announcement-processor-api/src/main/java/api/repository/AnnouncementRepository.java @@ -0,0 +1,7 @@ +package api.repository; + +import api.entity.Announcement; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface AnnouncementRepository extends JpaRepository { +} diff --git a/announcement-processor-api/src/main/java/api/repository/LessorRepository.java b/announcement-processor-api/src/main/java/api/repository/LessorRepository.java new file mode 100644 index 0000000..37a398b --- /dev/null +++ b/announcement-processor-api/src/main/java/api/repository/LessorRepository.java @@ -0,0 +1,7 @@ +package api.repository; + +import api.entity.Lessor; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface LessorRepository extends JpaRepository { +} diff --git a/announcement-processor-api/src/main/java/api/repository/LocationRepository.java b/announcement-processor-api/src/main/java/api/repository/LocationRepository.java new file mode 100644 index 0000000..cdaa876 --- /dev/null +++ b/announcement-processor-api/src/main/java/api/repository/LocationRepository.java @@ -0,0 +1,7 @@ +package api.repository; + +import api.entity.Location; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface LocationRepository extends JpaRepository { +} diff --git a/announcement-processor-api/src/main/java/api/repository/PriceOfferRepository.java b/announcement-processor-api/src/main/java/api/repository/PriceOfferRepository.java new file mode 100644 index 0000000..a343051 --- /dev/null +++ b/announcement-processor-api/src/main/java/api/repository/PriceOfferRepository.java @@ -0,0 +1,7 @@ +package api.repository; + +import api.entity.PriceOffer; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface PriceOfferRepository extends JpaRepository { +} diff --git a/announcement-processor-api/src/main/java/api/repository/PropertyDataRepository.java b/announcement-processor-api/src/main/java/api/repository/PropertyDataRepository.java new file mode 100644 index 0000000..e258e9f --- /dev/null +++ b/announcement-processor-api/src/main/java/api/repository/PropertyDataRepository.java @@ -0,0 +1,8 @@ +package api.repository; + +import api.entity.PropertyData; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface PropertyDataRepository extends JpaRepository { + +} From c03511aa7a8040763067cace9fc9e81dd52ea2d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 6 Jan 2019 16:05:24 +0100 Subject: [PATCH 105/135] Definitions of single table inheritance for price --- .../java/api/entity/price/ConsumerPrice.java | 25 +++++++++++++++++++ .../java/api/entity/price/ContactPrice.java | 18 +++++++++++++ .../java/api/entity/price/ExchangePrice.java | 17 +++++++++++++ 3 files changed, 60 insertions(+) create mode 100644 announcement-processor-api/src/main/java/api/entity/price/ConsumerPrice.java create mode 100644 announcement-processor-api/src/main/java/api/entity/price/ContactPrice.java create mode 100644 announcement-processor-api/src/main/java/api/entity/price/ExchangePrice.java diff --git a/announcement-processor-api/src/main/java/api/entity/price/ConsumerPrice.java b/announcement-processor-api/src/main/java/api/entity/price/ConsumerPrice.java new file mode 100644 index 0000000..e675fa3 --- /dev/null +++ b/announcement-processor-api/src/main/java/api/entity/price/ConsumerPrice.java @@ -0,0 +1,25 @@ +package api.entity.price; + +import api.entity.PriceOffer; + +import javax.persistence.Column; +import javax.persistence.DiscriminatorValue; +import javax.persistence.Entity; +import java.math.BigDecimal; + +@Entity +@DiscriminatorValue(value = "real_price") +public class ConsumerPrice extends PriceOffer { + + @Column(name = "REAL_PRICE") + private BigDecimal price; + + public ConsumerPrice() { + super(); + } + + public ConsumerPrice(String name, BigDecimal price) { + super("Cena"); + this.price = price; + } +} diff --git a/announcement-processor-api/src/main/java/api/entity/price/ContactPrice.java b/announcement-processor-api/src/main/java/api/entity/price/ContactPrice.java new file mode 100644 index 0000000..4e526e0 --- /dev/null +++ b/announcement-processor-api/src/main/java/api/entity/price/ContactPrice.java @@ -0,0 +1,18 @@ +package api.entity.price; + +import api.entity.PriceOffer; + +import javax.persistence.DiscriminatorValue; +import javax.persistence.Entity; + +@Entity +@DiscriminatorValue(value = "contact") +public class ContactPrice extends PriceOffer { + public ContactPrice() { + super(); + } + + public ContactPrice(String name) { + super("Proszę o kontakt"); + } +} diff --git a/announcement-processor-api/src/main/java/api/entity/price/ExchangePrice.java b/announcement-processor-api/src/main/java/api/entity/price/ExchangePrice.java new file mode 100644 index 0000000..23f08c8 --- /dev/null +++ b/announcement-processor-api/src/main/java/api/entity/price/ExchangePrice.java @@ -0,0 +1,17 @@ +package api.entity.price; + +import api.entity.PriceOffer; + +import javax.persistence.DiscriminatorValue; +import javax.persistence.Entity; + +@Entity +@DiscriminatorValue(value = "exchange") +public class ExchangePrice extends PriceOffer { + public ExchangePrice() { + } + + public ExchangePrice(String name) { + super("Wymiana/Zamiana"); + } +} From 451ada0d78c714ac81426d1d23ea0d87645986bd Mon Sep 17 00:00:00 2001 From: michalsroka Date: Sat, 5 Jan 2019 00:17:09 +0100 Subject: [PATCH 106/135] Add sketch for receiving messages --- .../consumer/AnnouncementConsumerJms.java | 54 ++++++++++++++++++- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerJms.java b/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerJms.java index 6b6688f..cb181b7 100644 --- a/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerJms.java +++ b/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerJms.java @@ -1,11 +1,61 @@ package extractor.consumer; import extractor.dto.AnnouncementDto; +import extractor.service.MapperService; +import org.apache.activemq.ActiveMQConnectionFactory; -public class AnnouncementConsumerJms implements AnnouncementConsumer { +import javax.jms.*; + +public class AnnouncementConsumerJms implements AnnouncementConsumer, ExceptionListener { + + private MapperService mapperService; + + public AnnouncementConsumerJms() { + this.mapperService = new MapperService(); + } @Override public AnnouncementDto consumeAnnouncement() { - return null; + + try { + ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory("vm://localhost"); + + Connection connection = connectionFactory.createConnection(); + connection.start(); + + connection.setExceptionListener(this); + + Session session = connection.createSession(false, Session.AUTO_ACKNOWLEDGE); + + Destination destination = session.createQueue("ANNOUNCEMENTS"); + + MessageConsumer consumer = session.createConsumer(destination); + + Message message = consumer.receive(100); + + AnnouncementDto announcementDto = null; + + if (message instanceof TextMessage) { + TextMessage textMessage = (TextMessage) message; + // TODO check if message can be a list of announcements + announcementDto = mapperService.getAnnouncementDtoFromJsonString(textMessage.getText()); + } + + consumer.close(); + session.close(); + connection.close(); + return announcementDto; + + } catch (Exception e) { + System.out.println("Caught: " + e); + e.printStackTrace(); + return null; + } + } + + @Override + public void onException(JMSException e) { + + System.out.println("JMS exception: " + e); } } From b99851848daf8e04a54cdca221036c9bc427526a Mon Sep 17 00:00:00 2001 From: michalsroka Date: Sat, 5 Jan 2019 00:24:07 +0100 Subject: [PATCH 107/135] Add operations to the scheduled process #36 --- .../extractor/consumer/ReceivementScheduler.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/announcement-processor-extractor/src/main/java/extractor/consumer/ReceivementScheduler.java b/announcement-processor-extractor/src/main/java/extractor/consumer/ReceivementScheduler.java index aa023ba..fef1f5a 100644 --- a/announcement-processor-extractor/src/main/java/extractor/consumer/ReceivementScheduler.java +++ b/announcement-processor-extractor/src/main/java/extractor/consumer/ReceivementScheduler.java @@ -1,5 +1,8 @@ package extractor.consumer; +import extractor.dto.AnnouncementDto; +import extractor.entity.Announcement; +import extractor.service.AnnouncementExtractingService; import lombok.extern.slf4j.Slf4j; import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.annotation.Scheduled; @@ -10,9 +13,20 @@ @EnableScheduling public class ReceivementScheduler { + private AnnouncementConsumerJms consumer; + private AnnouncementExtractingService extractor; + + public ReceivementScheduler() { + this.consumer = new AnnouncementConsumerJms(); + this.extractor = new AnnouncementExtractingService(); + } + @Scheduled(cron= "0 */2 * * * *") private void scheduleTask() { + log.info("Executing ReceivementScheduler scheduledTask"); - + AnnouncementDto announcementDto = consumer.consumeAnnouncement(); + Announcement announcement = extractor.extractFromAnnouncementDto(announcementDto); + // TODO save to the DB } } From def2ff18a7d4d5aa015eb4e28f8f26050a7c407c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sat, 5 Jan 2019 15:04:34 +0100 Subject: [PATCH 108/135] Added broker connection credentials for dev use. #14 --- announcement-processor-extractor/.gitignore | 2 ++ .../src/main/resources/application-dev.properties | 2 ++ .../src/main/resources/application.properties | 2 ++ 3 files changed, 6 insertions(+) create mode 100644 announcement-processor-extractor/src/main/resources/application-dev.properties diff --git a/announcement-processor-extractor/.gitignore b/announcement-processor-extractor/.gitignore index c4702dd..20e9cad 100644 --- a/announcement-processor-extractor/.gitignore +++ b/announcement-processor-extractor/.gitignore @@ -3,6 +3,8 @@ mvnw mvnw.cmd +src/main/resources/application-prod.properties + ### STS ### .apt_generated .classpath diff --git a/announcement-processor-extractor/src/main/resources/application-dev.properties b/announcement-processor-extractor/src/main/resources/application-dev.properties new file mode 100644 index 0000000..23efd10 --- /dev/null +++ b/announcement-processor-extractor/src/main/resources/application-dev.properties @@ -0,0 +1,2 @@ +brokerURL=tcp://192.168.99.100:32768 +queue=announcements \ No newline at end of file diff --git a/announcement-processor-extractor/src/main/resources/application.properties b/announcement-processor-extractor/src/main/resources/application.properties index e69de29..62ab1ff 100644 --- a/announcement-processor-extractor/src/main/resources/application.properties +++ b/announcement-processor-extractor/src/main/resources/application.properties @@ -0,0 +1,2 @@ +spring.profiles.active=prod +server.port=8082 \ No newline at end of file From 9b124fa2bbce48d1536f6ea06e54801359618672 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sat, 5 Jan 2019 15:27:04 +0100 Subject: [PATCH 109/135] JMS properties injected to consumer. #14 --- .../consumer/AnnouncementConsumerJms.java | 17 ++++++++++++----- .../consumer/ReceivementScheduler.java | 8 ++++---- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerJms.java b/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerJms.java index cb181b7..de4c7d3 100644 --- a/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerJms.java +++ b/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerJms.java @@ -3,22 +3,29 @@ import extractor.dto.AnnouncementDto; import extractor.service.MapperService; import org.apache.activemq.ActiveMQConnectionFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; import javax.jms.*; +@Service public class AnnouncementConsumerJms implements AnnouncementConsumer, ExceptionListener { - + @Value("${brokerURL}") + private String brokerUrl; + @Value("${queue}") + private String queueName; private MapperService mapperService; - public AnnouncementConsumerJms() { + AnnouncementConsumerJms() { this.mapperService = new MapperService(); } @Override public AnnouncementDto consumeAnnouncement() { - + System.out.println(brokerUrl); + System.out.println(queueName); try { - ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory("vm://localhost"); + ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory(brokerUrl); Connection connection = connectionFactory.createConnection(); connection.start(); @@ -27,7 +34,7 @@ public AnnouncementDto consumeAnnouncement() { Session session = connection.createSession(false, Session.AUTO_ACKNOWLEDGE); - Destination destination = session.createQueue("ANNOUNCEMENTS"); + Destination destination = session.createQueue(queueName); MessageConsumer consumer = session.createConsumer(destination); diff --git a/announcement-processor-extractor/src/main/java/extractor/consumer/ReceivementScheduler.java b/announcement-processor-extractor/src/main/java/extractor/consumer/ReceivementScheduler.java index fef1f5a..44fd4ac 100644 --- a/announcement-processor-extractor/src/main/java/extractor/consumer/ReceivementScheduler.java +++ b/announcement-processor-extractor/src/main/java/extractor/consumer/ReceivementScheduler.java @@ -16,12 +16,12 @@ public class ReceivementScheduler { private AnnouncementConsumerJms consumer; private AnnouncementExtractingService extractor; - public ReceivementScheduler() { - this.consumer = new AnnouncementConsumerJms(); - this.extractor = new AnnouncementExtractingService(); + public ReceivementScheduler(AnnouncementConsumerJms consumer, AnnouncementExtractingService extractor) { + this.consumer = consumer; + this.extractor = extractor; } - @Scheduled(cron= "0 */2 * * * *") + @Scheduled(cron = "*/10 * * * * *") private void scheduleTask() { log.info("Executing ReceivementScheduler scheduledTask"); From bf88ca8854473307bd7bfc7399007c32e3e34f83 Mon Sep 17 00:00:00 2001 From: michalsroka Date: Sat, 5 Jan 2019 23:20:02 +0100 Subject: [PATCH 110/135] Add more detailed logging --- .../consumer/AnnouncementConsumerJms.java | 16 +++++++++------- .../extractor/consumer/ReceivementScheduler.java | 2 ++ .../service/AnnouncementExtractingService.java | 5 +++++ .../src/main/resources/logback.xml | 3 +++ 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerJms.java b/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerJms.java index de4c7d3..2d0284a 100644 --- a/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerJms.java +++ b/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerJms.java @@ -2,6 +2,7 @@ import extractor.dto.AnnouncementDto; import extractor.service.MapperService; +import lombok.extern.slf4j.Slf4j; import org.apache.activemq.ActiveMQConnectionFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -9,7 +10,9 @@ import javax.jms.*; @Service +@Slf4j public class AnnouncementConsumerJms implements AnnouncementConsumer, ExceptionListener { + @Value("${brokerURL}") private String brokerUrl; @Value("${queue}") @@ -22,8 +25,8 @@ public class AnnouncementConsumerJms implements AnnouncementConsumer, ExceptionL @Override public AnnouncementDto consumeAnnouncement() { - System.out.println(brokerUrl); - System.out.println(queueName); + + log.debug("Running consumeAnnouncement, brokerUrl: {}, queueName: {}", brokerUrl, queueName); try { ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory(brokerUrl); @@ -39,22 +42,22 @@ public AnnouncementDto consumeAnnouncement() { MessageConsumer consumer = session.createConsumer(destination); Message message = consumer.receive(100); - + log.debug("Received message: {}", message); AnnouncementDto announcementDto = null; if (message instanceof TextMessage) { TextMessage textMessage = (TextMessage) message; - // TODO check if message can be a list of announcements + log.info("Received text message: {}", textMessage.getText()); announcementDto = mapperService.getAnnouncementDtoFromJsonString(textMessage.getText()); } consumer.close(); session.close(); connection.close(); + log.debug("Finished consumeAnnouncement"); return announcementDto; } catch (Exception e) { - System.out.println("Caught: " + e); e.printStackTrace(); return null; } @@ -62,7 +65,6 @@ public AnnouncementDto consumeAnnouncement() { @Override public void onException(JMSException e) { - - System.out.println("JMS exception: " + e); + log.error("JMS exception: {}" , e); } } diff --git a/announcement-processor-extractor/src/main/java/extractor/consumer/ReceivementScheduler.java b/announcement-processor-extractor/src/main/java/extractor/consumer/ReceivementScheduler.java index 44fd4ac..7212f41 100644 --- a/announcement-processor-extractor/src/main/java/extractor/consumer/ReceivementScheduler.java +++ b/announcement-processor-extractor/src/main/java/extractor/consumer/ReceivementScheduler.java @@ -26,7 +26,9 @@ private void scheduleTask() { log.info("Executing ReceivementScheduler scheduledTask"); AnnouncementDto announcementDto = consumer.consumeAnnouncement(); + log.debug("Received announcementDto: {}", announcementDto); Announcement announcement = extractor.extractFromAnnouncementDto(announcementDto); + log.debug("Converted announcementDto to announcement: {}", announcement); // TODO save to the DB } } diff --git a/announcement-processor-extractor/src/main/java/extractor/service/AnnouncementExtractingService.java b/announcement-processor-extractor/src/main/java/extractor/service/AnnouncementExtractingService.java index 74caefc..e82cd0b 100644 --- a/announcement-processor-extractor/src/main/java/extractor/service/AnnouncementExtractingService.java +++ b/announcement-processor-extractor/src/main/java/extractor/service/AnnouncementExtractingService.java @@ -48,6 +48,7 @@ private Price extractPrice(AnnouncementDto announcementDto) { price.setBasePrice(announcementDto.getPrice()); // TODO more advanced extraction e.g. description data extraction Map pricesMap = new HashMap<>(); + parseDescriptionPrice(announcementDto.getDescription(), pricesMap); pricesMap.put("SomePrice", announcementDto.getAdditionalRentCost()); price.setAdditionalPrices(pricesMap); return price; @@ -67,4 +68,8 @@ private PropertyData extractPropertyData(AnnouncementDto announcementDto) { // TODO more advanced extraction e.g. description data extraction return propertyData; } + + private void parseDescriptionPrice(String description, Map pricesMap) { + + } } diff --git a/announcement-processor-extractor/src/main/resources/logback.xml b/announcement-processor-extractor/src/main/resources/logback.xml index 5097d20..9075aba 100644 --- a/announcement-processor-extractor/src/main/resources/logback.xml +++ b/announcement-processor-extractor/src/main/resources/logback.xml @@ -15,4 +15,7 @@ + + +     \ No newline at end of file From 9e5a11e68d682d8efc36c7b2f91e4a04858be6b3 Mon Sep 17 00:00:00 2001 From: michalsroka Date: Sun, 6 Jan 2019 16:09:47 +0100 Subject: [PATCH 111/135] Add currency to Price --- .../src/main/java/extractor/entity/Announcement.java | 2 +- .../src/main/java/extractor/entity/Price.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/announcement-processor-extractor/src/main/java/extractor/entity/Announcement.java b/announcement-processor-extractor/src/main/java/extractor/entity/Announcement.java index 10afac3..645dc44 100644 --- a/announcement-processor-extractor/src/main/java/extractor/entity/Announcement.java +++ b/announcement-processor-extractor/src/main/java/extractor/entity/Announcement.java @@ -24,5 +24,5 @@ public class Announcement { private LocalDateTime creationDate; private String url; private String description; - private String provider; // to enum + private String provider; } diff --git a/announcement-processor-extractor/src/main/java/extractor/entity/Price.java b/announcement-processor-extractor/src/main/java/extractor/entity/Price.java index f8451c6..c81e42d 100644 --- a/announcement-processor-extractor/src/main/java/extractor/entity/Price.java +++ b/announcement-processor-extractor/src/main/java/extractor/entity/Price.java @@ -17,6 +17,7 @@ public class Price { // Integers may be changed to a custom Money type private Integer basePrice; + private String currency = "PLN"; private Map additionalPrices; public Integer getSummedPrice() { From 4ae888a507444378390ad3543a87d360bf98d2b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 6 Jan 2019 16:44:40 +0100 Subject: [PATCH 112/135] Added services and main controller --- .../controller/AnnouncementsController.java | 18 ++++++++++++++++++ .../java/api/service/ExtractorDataService.java | 7 +++++++ .../java/api/service/FrontEndDataService.java | 7 +++++++ 3 files changed, 32 insertions(+) create mode 100644 announcement-processor-api/src/main/java/api/controller/AnnouncementsController.java create mode 100644 announcement-processor-api/src/main/java/api/service/ExtractorDataService.java create mode 100644 announcement-processor-api/src/main/java/api/service/FrontEndDataService.java diff --git a/announcement-processor-api/src/main/java/api/controller/AnnouncementsController.java b/announcement-processor-api/src/main/java/api/controller/AnnouncementsController.java new file mode 100644 index 0000000..c2f9bff --- /dev/null +++ b/announcement-processor-api/src/main/java/api/controller/AnnouncementsController.java @@ -0,0 +1,18 @@ +package api.controller; + +import api.service.ExtractorDataService; +import api.service.FrontEndDataService; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/announcements") +public class AnnouncementsController { + private ExtractorDataService extractorDataService; + private FrontEndDataService frontEndDataService; + + public AnnouncementsController(ExtractorDataService extractorDataService, FrontEndDataService frontEndDataService) { + this.extractorDataService = extractorDataService; + this.frontEndDataService = frontEndDataService; + } +} diff --git a/announcement-processor-api/src/main/java/api/service/ExtractorDataService.java b/announcement-processor-api/src/main/java/api/service/ExtractorDataService.java new file mode 100644 index 0000000..30dd4d8 --- /dev/null +++ b/announcement-processor-api/src/main/java/api/service/ExtractorDataService.java @@ -0,0 +1,7 @@ +package api.service; + +import org.springframework.stereotype.Service; + +@Service +public class ExtractorDataService { +} diff --git a/announcement-processor-api/src/main/java/api/service/FrontEndDataService.java b/announcement-processor-api/src/main/java/api/service/FrontEndDataService.java new file mode 100644 index 0000000..9f11ff0 --- /dev/null +++ b/announcement-processor-api/src/main/java/api/service/FrontEndDataService.java @@ -0,0 +1,7 @@ +package api.service; + +import org.springframework.stereotype.Service; + +@Service +public class FrontEndDataService { +} From d927c667ecd98d2e26a34ceae107553ff788e373 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 6 Jan 2019 17:28:14 +0100 Subject: [PATCH 113/135] Deleted base_price field, added currency field --- .../src/main/java/api/entity/Announcement.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/announcement-processor-api/src/main/java/api/entity/Announcement.java b/announcement-processor-api/src/main/java/api/entity/Announcement.java index 2d793b9..e3ab234 100644 --- a/announcement-processor-api/src/main/java/api/entity/Announcement.java +++ b/announcement-processor-api/src/main/java/api/entity/Announcement.java @@ -3,7 +3,6 @@ import lombok.Data; import javax.persistence.*; -import java.math.BigDecimal; import java.time.LocalDateTime; import java.util.List; @@ -20,9 +19,6 @@ public class Announcement { @Column(name = "TITLE") private String title; - @Column(name = "BASE_PRICE") - private BigDecimal basePrice; - @Column(name = "PROVIDER") private String provider; @@ -35,6 +31,9 @@ public class Announcement { @Column(name = "DESCRIPTION") private String description; + @Column(name = "CURRENCY") + private String currency; + @OneToOne @JoinColumn(name = "PRICE_ID") private PriceOffer priceOffer; From f53ac9fcd0763797eab07c1dcc7769e309e597a1 Mon Sep 17 00:00:00 2001 From: michalsroka Date: Sun, 6 Jan 2019 17:05:48 +0100 Subject: [PATCH 114/135] Jackson version update --- announcement-processor-extractor/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/announcement-processor-extractor/pom.xml b/announcement-processor-extractor/pom.xml index 1611fb4..ce8a37d 100644 --- a/announcement-processor-extractor/pom.xml +++ b/announcement-processor-extractor/pom.xml @@ -65,13 +65,13 @@ com.fasterxml.jackson.core jackson-annotations - 2.9.0 + 2.9.8 com.fasterxml.jackson.core jackson-databind - 2.9.7 + 2.9.8 From a01e0ea5a84200cf280de378fe8e0a5d4ad74ddf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 6 Jan 2019 18:04:09 +0100 Subject: [PATCH 115/135] Remote facade for full location address --- .../src/main/java/api/entity/Location.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/announcement-processor-api/src/main/java/api/entity/Location.java b/announcement-processor-api/src/main/java/api/entity/Location.java index eefd20c..a870f75 100644 --- a/announcement-processor-api/src/main/java/api/entity/Location.java +++ b/announcement-processor-api/src/main/java/api/entity/Location.java @@ -36,4 +36,10 @@ public class Location { @OneToOne(mappedBy = "location") private Announcement announcement; + + public String getFullLocation() { + return city + " " + zipCode + " " + district + "\n" + + street + " " + buildingNumber + " / " + flatNumber + "\n" + + country; + } } From 60796cbc3661ae52455185382622954f2819260e Mon Sep 17 00:00:00 2001 From: michalsroka Date: Sun, 6 Jan 2019 18:15:53 +0100 Subject: [PATCH 116/135] Change extractor's Price Integer to BigDecimal --- .../src/main/java/extractor/entity/Price.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/announcement-processor-extractor/src/main/java/extractor/entity/Price.java b/announcement-processor-extractor/src/main/java/extractor/entity/Price.java index c81e42d..5213528 100644 --- a/announcement-processor-extractor/src/main/java/extractor/entity/Price.java +++ b/announcement-processor-extractor/src/main/java/extractor/entity/Price.java @@ -4,6 +4,7 @@ import lombok.Data; import lombok.NoArgsConstructor; +import java.math.BigDecimal; import java.util.Map; /* @@ -16,14 +17,15 @@ public class Price { // Integers may be changed to a custom Money type - private Integer basePrice; + private BigDecimal basePrice; private String currency = "PLN"; - private Map additionalPrices; + private Map additionalPrices; - public Integer getSummedPrice() { - return basePrice + additionalPrices.values() - .stream() - .mapToInt(i -> i) - .sum(); + public BigDecimal getSummedPrice() { + return basePrice.add( + additionalPrices.values() + .stream() + .reduce(BigDecimal.ZERO, BigDecimal::add) + ); } } From 82d5f0f0fb45113a18cc78bff959f7fefdbc3241 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 6 Jan 2019 18:17:19 +0100 Subject: [PATCH 117/135] Added @Data annotation --- .../src/main/java/api/entity/Currency.java | 13 +++++++++++++ .../main/java/api/entity/price/ConsumerPrice.java | 2 ++ .../main/java/api/entity/price/ContactPrice.java | 2 ++ .../main/java/api/entity/price/ExchangePrice.java | 2 ++ 4 files changed, 19 insertions(+) diff --git a/announcement-processor-api/src/main/java/api/entity/Currency.java b/announcement-processor-api/src/main/java/api/entity/Currency.java index 4bc470b..9274cc0 100644 --- a/announcement-processor-api/src/main/java/api/entity/Currency.java +++ b/announcement-processor-api/src/main/java/api/entity/Currency.java @@ -1,8 +1,12 @@ package api.entity; +import lombok.Data; + import javax.persistence.*; +import java.math.BigDecimal; @Entity +@Data @Table(name = "CURRENCIES") public class Currency { @@ -10,4 +14,13 @@ public class Currency { @GeneratedValue(strategy = GenerationType.IDENTITY) @Column(name = "ID") private Integer id; + + @Column(name = "CURRENCY_FROM") + private String currencyFrom; + + @Column(name = "CURRENCY_TO") + private String currencyTo; + + @Column(name = "MULTIPLIER") + private BigDecimal multiplier; } diff --git a/announcement-processor-api/src/main/java/api/entity/price/ConsumerPrice.java b/announcement-processor-api/src/main/java/api/entity/price/ConsumerPrice.java index e675fa3..3ee7514 100644 --- a/announcement-processor-api/src/main/java/api/entity/price/ConsumerPrice.java +++ b/announcement-processor-api/src/main/java/api/entity/price/ConsumerPrice.java @@ -1,6 +1,7 @@ package api.entity.price; import api.entity.PriceOffer; +import lombok.Data; import javax.persistence.Column; import javax.persistence.DiscriminatorValue; @@ -8,6 +9,7 @@ import java.math.BigDecimal; @Entity +@Data @DiscriminatorValue(value = "real_price") public class ConsumerPrice extends PriceOffer { diff --git a/announcement-processor-api/src/main/java/api/entity/price/ContactPrice.java b/announcement-processor-api/src/main/java/api/entity/price/ContactPrice.java index 4e526e0..544e883 100644 --- a/announcement-processor-api/src/main/java/api/entity/price/ContactPrice.java +++ b/announcement-processor-api/src/main/java/api/entity/price/ContactPrice.java @@ -1,11 +1,13 @@ package api.entity.price; import api.entity.PriceOffer; +import lombok.Data; import javax.persistence.DiscriminatorValue; import javax.persistence.Entity; @Entity +@Data @DiscriminatorValue(value = "contact") public class ContactPrice extends PriceOffer { public ContactPrice() { diff --git a/announcement-processor-api/src/main/java/api/entity/price/ExchangePrice.java b/announcement-processor-api/src/main/java/api/entity/price/ExchangePrice.java index 23f08c8..957fadd 100644 --- a/announcement-processor-api/src/main/java/api/entity/price/ExchangePrice.java +++ b/announcement-processor-api/src/main/java/api/entity/price/ExchangePrice.java @@ -1,11 +1,13 @@ package api.entity.price; import api.entity.PriceOffer; +import lombok.Data; import javax.persistence.DiscriminatorValue; import javax.persistence.Entity; @Entity +@Data @DiscriminatorValue(value = "exchange") public class ExchangePrice extends PriceOffer { public ExchangePrice() { From 9daefad0c462e9269c428f2fab515c5e9adc3a75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 6 Jan 2019 18:29:16 +0100 Subject: [PATCH 118/135] Money pattern and currency service --- .../src/main/java/api/model/Money.java | 31 +++++++++++++++++++ .../api/repository/CurrencyRepository.java | 12 +++++++ .../java/api/service/CurrencyService.java | 22 +++++++++++++ 3 files changed, 65 insertions(+) create mode 100644 announcement-processor-api/src/main/java/api/model/Money.java create mode 100644 announcement-processor-api/src/main/java/api/repository/CurrencyRepository.java create mode 100644 announcement-processor-api/src/main/java/api/service/CurrencyService.java diff --git a/announcement-processor-api/src/main/java/api/model/Money.java b/announcement-processor-api/src/main/java/api/model/Money.java new file mode 100644 index 0000000..0115ff4 --- /dev/null +++ b/announcement-processor-api/src/main/java/api/model/Money.java @@ -0,0 +1,31 @@ +package api.model; + +import api.service.CurrencyService; +import com.fasterxml.jackson.annotation.JsonIgnore; + +import java.math.BigDecimal; + +public class Money { + @JsonIgnore + private CurrencyService currencyService; + private BigDecimal price; + private String actualCurrency; + + public Money(CurrencyService currencyService, BigDecimal price, String actualCurrency) { + this.currencyService = currencyService; + this.price = price; + this.actualCurrency = actualCurrency; + } + + public void recalculateToCurrency(String desiredCurrency) { + BigDecimal newPrice = currencyService.calculateToCurrency(actualCurrency, desiredCurrency, price); + if (newPrice != null) { + actualCurrency = desiredCurrency; + price = newPrice; + } + } + + public String getPriceWithCurrency() { + return price + " " + actualCurrency; + } +} diff --git a/announcement-processor-api/src/main/java/api/repository/CurrencyRepository.java b/announcement-processor-api/src/main/java/api/repository/CurrencyRepository.java new file mode 100644 index 0000000..7ca1771 --- /dev/null +++ b/announcement-processor-api/src/main/java/api/repository/CurrencyRepository.java @@ -0,0 +1,12 @@ +package api.repository; + +import api.entity.Currency; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +import java.util.Optional; + +@Repository +public interface CurrencyRepository extends JpaRepository { + Optional findCurrencyByCurrencyFromAndCurrencyTo(String from, String to); +} diff --git a/announcement-processor-api/src/main/java/api/service/CurrencyService.java b/announcement-processor-api/src/main/java/api/service/CurrencyService.java new file mode 100644 index 0000000..f9fb8a0 --- /dev/null +++ b/announcement-processor-api/src/main/java/api/service/CurrencyService.java @@ -0,0 +1,22 @@ +package api.service; + +import api.entity.Currency; +import api.repository.CurrencyRepository; +import org.springframework.stereotype.Service; + +import java.math.BigDecimal; +import java.util.Optional; + +@Service +public class CurrencyService { + private CurrencyRepository currencyRepository; + + public CurrencyService(CurrencyRepository currencyRepository) { + this.currencyRepository = currencyRepository; + } + + public BigDecimal calculateToCurrency(String actualCurrency, String desiredCurrency, BigDecimal price) { + Optional currency = currencyRepository.findCurrencyByCurrencyFromAndCurrencyTo(actualCurrency, desiredCurrency); + return currency.map(currency1 -> price.multiply(currency1.getMultiplier())).orElse(null); + } +} From 00e51ba34ee19de614775d68b9c42ef609572389 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 6 Jan 2019 18:33:39 +0100 Subject: [PATCH 119/135] Controller update mappings --- .../controller/AnnouncementsController.java | 39 ++++++++++++++++++- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/announcement-processor-api/src/main/java/api/controller/AnnouncementsController.java b/announcement-processor-api/src/main/java/api/controller/AnnouncementsController.java index c2f9bff..65ba071 100644 --- a/announcement-processor-api/src/main/java/api/controller/AnnouncementsController.java +++ b/announcement-processor-api/src/main/java/api/controller/AnnouncementsController.java @@ -1,9 +1,14 @@ package api.controller; +import api.model.DetailedAnnouncementInfo; +import api.model.GeneralAnnouncementInfo; import api.service.ExtractorDataService; import api.service.FrontEndDataService; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +import java.util.List; @RestController @RequestMapping("/announcements") @@ -15,4 +20,34 @@ public AnnouncementsController(ExtractorDataService extractorDataService, FrontE this.extractorDataService = extractorDataService; this.frontEndDataService = frontEndDataService; } + + @GetMapping("/get/all") + public ResponseEntity> getAllAnnouncements(@RequestParam String currency) { + List allAnnouncements = frontEndDataService.getAllAnnouncements(currency); + if (allAnnouncements != null) { + return new ResponseEntity<>(allAnnouncements, HttpStatus.OK); + } else { + return new ResponseEntity<>(HttpStatus.NOT_FOUND); + } + } + + @GetMapping("get/{id}/detailed-info") + public ResponseEntity getDetailedInfoAnnouncement(@PathVariable Integer id, @RequestParam String currency) { + DetailedAnnouncementInfo detailedInfoAnnouncement = frontEndDataService.getDetailedInfoAnnouncement(id, currency); + if (detailedInfoAnnouncement != null) { + return new ResponseEntity<>(detailedInfoAnnouncement, HttpStatus.OK); + } else { + return new ResponseEntity<>(HttpStatus.NOT_FOUND); + } + } + + @GetMapping("get/{id}/general-info") + public ResponseEntity getGeneralInfoAnnouncement(@PathVariable Integer id, @RequestParam String currency) { + GeneralAnnouncementInfo generalInfoAnnouncement = frontEndDataService.getGeneralInfoAnnouncement(id, currency); + if (generalInfoAnnouncement != null) { + return new ResponseEntity<>(generalInfoAnnouncement, HttpStatus.OK); + } else { + return new ResponseEntity<>(HttpStatus.NOT_FOUND); + } + } } From 4ebde62f6cfec12d7468dcfed45bf6b0d7ae846a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 6 Jan 2019 18:38:49 +0100 Subject: [PATCH 120/135] Implementation of get request for GeneralAnnouncementInfo single and all --- .../api/model/GeneralAnnouncementInfo.java | 14 ++++ .../java/api/service/FrontEndDataService.java | 73 +++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 announcement-processor-api/src/main/java/api/model/GeneralAnnouncementInfo.java diff --git a/announcement-processor-api/src/main/java/api/model/GeneralAnnouncementInfo.java b/announcement-processor-api/src/main/java/api/model/GeneralAnnouncementInfo.java new file mode 100644 index 0000000..5dc12db --- /dev/null +++ b/announcement-processor-api/src/main/java/api/model/GeneralAnnouncementInfo.java @@ -0,0 +1,14 @@ +package api.model; + +import lombok.Data; + +@Data +public class GeneralAnnouncementInfo { + private String title; + private String baseCost; + private String provider; + private String creationDate; + private String url; + private String lessorType; + private String location; +} diff --git a/announcement-processor-api/src/main/java/api/service/FrontEndDataService.java b/announcement-processor-api/src/main/java/api/service/FrontEndDataService.java index 9f11ff0..826d211 100644 --- a/announcement-processor-api/src/main/java/api/service/FrontEndDataService.java +++ b/announcement-processor-api/src/main/java/api/service/FrontEndDataService.java @@ -1,7 +1,80 @@ package api.service; +import api.entity.Announcement; +import api.entity.PriceOffer; +import api.entity.price.ConsumerPrice; +import api.entity.price.ContactPrice; +import api.entity.price.ExchangePrice; +import api.model.DetailedAnnouncementInfo; +import api.model.GeneralAnnouncementInfo; +import api.model.Money; +import api.repository.AnnouncementRepository; import org.springframework.stereotype.Service; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + @Service public class FrontEndDataService { + private AnnouncementRepository announcementRepository; + private CurrencyService currencyService; + + public FrontEndDataService(AnnouncementRepository announcementRepository, CurrencyService currencyService) { + this.announcementRepository = announcementRepository; + this.currencyService = currencyService; + } + + public List getAllAnnouncements(String desiredCurrency) { + List all = announcementRepository.findAll(); + if (all != null && !all.isEmpty()) { + return all.stream() + .map(announcement -> convertAnnouncementToGeneralFormat(announcement, desiredCurrency)) + .collect(Collectors.toList()); + } + return null; + } + + public GeneralAnnouncementInfo getGeneralInfoAnnouncement(Integer id, String desiredCurrency) { + Optional ann = announcementRepository.findById(id); + return ann.map(announcement -> convertAnnouncementToGeneralFormat(announcement, desiredCurrency)) + .orElse(null); + } + + public DetailedAnnouncementInfo getDetailedInfoAnnouncement(Integer id, String desiredCurrency) { + return null; + } + + private GeneralAnnouncementInfo convertAnnouncementToGeneralFormat(Announcement announcement, String desiredCurrency) { + + String cost = getBaseCostText(announcement, desiredCurrency, currencyService); + + GeneralAnnouncementInfo generalAnnouncementInfo = new GeneralAnnouncementInfo(); + generalAnnouncementInfo.setBaseCost(cost); + generalAnnouncementInfo.setCreationDate(announcement.getCreationDate().toString()); + generalAnnouncementInfo.setLessorType(announcement.getLessor().getLessorType()); + generalAnnouncementInfo.setLocation(announcement.getLocation().getFullLocation()); + generalAnnouncementInfo.setProvider(announcement.getProvider()); + generalAnnouncementInfo.setTitle(announcement.getTitle()); + generalAnnouncementInfo.setUrl(announcement.getUrl()); + return generalAnnouncementInfo; + } + + private String getBaseCostText(Announcement announcement, String desiredCurrency, CurrencyService currencyService) { + PriceOffer priceOffer = announcement.getPriceOffer(); + + if (priceOffer instanceof ConsumerPrice) { + ConsumerPrice consumerPrice = (ConsumerPrice) priceOffer; + Money cost = new Money(currencyService, consumerPrice.getPrice(), announcement.getCurrency()); + cost.recalculateToCurrency(desiredCurrency); + return cost.getPriceWithCurrency(); + } else if (priceOffer instanceof ContactPrice) { + System.out.println("contact price"); + return priceOffer.getDescription(); + } else if (priceOffer instanceof ExchangePrice) { + System.out.println("exchange"); + return priceOffer.getDescription(); + } + return null; + } } From bf0f6ddc17d2dc78c2b1d58680b5a47f32a62bf4 Mon Sep 17 00:00:00 2001 From: michalsroka Date: Sun, 6 Jan 2019 18:39:00 +0100 Subject: [PATCH 121/135] Add extractor sending HTTP announcement --- .../consumer/ReceivementScheduler.java | 10 ++- .../java/extractor/dto/AnnouncementDto.java | 9 +-- .../AnnouncementExtractingService.java | 5 +- .../service/AnnouncementSenderService.java | 70 +++++++++++++++++++ .../java/extractor/service/MapperService.java | 6 ++ .../main/resources/application-dev.properties | 4 +- .../AnnouncementConsumerFromFileTest.java | 5 +- 7 files changed, 98 insertions(+), 11 deletions(-) create mode 100644 announcement-processor-extractor/src/main/java/extractor/service/AnnouncementSenderService.java diff --git a/announcement-processor-extractor/src/main/java/extractor/consumer/ReceivementScheduler.java b/announcement-processor-extractor/src/main/java/extractor/consumer/ReceivementScheduler.java index 7212f41..c656b99 100644 --- a/announcement-processor-extractor/src/main/java/extractor/consumer/ReceivementScheduler.java +++ b/announcement-processor-extractor/src/main/java/extractor/consumer/ReceivementScheduler.java @@ -3,6 +3,7 @@ import extractor.dto.AnnouncementDto; import extractor.entity.Announcement; import extractor.service.AnnouncementExtractingService; +import extractor.service.AnnouncementSenderService; import lombok.extern.slf4j.Slf4j; import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.annotation.Scheduled; @@ -15,10 +16,14 @@ public class ReceivementScheduler { private AnnouncementConsumerJms consumer; private AnnouncementExtractingService extractor; + private AnnouncementSenderService announcementSender; - public ReceivementScheduler(AnnouncementConsumerJms consumer, AnnouncementExtractingService extractor) { + public ReceivementScheduler(AnnouncementConsumerJms consumer, + AnnouncementExtractingService extractor, + AnnouncementSenderService announcementSender) { this.consumer = consumer; this.extractor = extractor; + this.announcementSender = announcementSender; } @Scheduled(cron = "*/10 * * * * *") @@ -29,6 +34,7 @@ private void scheduleTask() { log.debug("Received announcementDto: {}", announcementDto); Announcement announcement = extractor.extractFromAnnouncementDto(announcementDto); log.debug("Converted announcementDto to announcement: {}", announcement); - // TODO save to the DB + announcementSender.send(announcement); + log.info("Sent announcement to API"); } } diff --git a/announcement-processor-extractor/src/main/java/extractor/dto/AnnouncementDto.java b/announcement-processor-extractor/src/main/java/extractor/dto/AnnouncementDto.java index 2cf5fbe..4ebd64f 100644 --- a/announcement-processor-extractor/src/main/java/extractor/dto/AnnouncementDto.java +++ b/announcement-processor-extractor/src/main/java/extractor/dto/AnnouncementDto.java @@ -4,6 +4,7 @@ import lombok.Data; import lombok.NoArgsConstructor; +import java.math.BigDecimal; import java.time.LocalDateTime; @Data @@ -11,7 +12,7 @@ public class AnnouncementDto { private String title; - private Integer price; + private BigDecimal price; private LocalDateTime creationDate; private String lessor; private String propertyType; @@ -22,7 +23,7 @@ public class AnnouncementDto { private Boolean isSmokingAllowed; private Boolean isPetFriendly; private String lessorName; - private Integer additionalRentCost; + private BigDecimal additionalRentCost; private Integer level; private String furnishing; private String description; @@ -32,7 +33,7 @@ public class AnnouncementDto { public AnnouncementDto( @JsonProperty("title") String title, - @JsonProperty("price") Integer price, + @JsonProperty("price") BigDecimal price, @JsonProperty("creationDate") LocalDateTime creationDate, @JsonProperty("lessor") String lessor, @JsonProperty("propertyType") String propertyType, @@ -43,7 +44,7 @@ public AnnouncementDto( @JsonProperty("isSmokingAllowed") Boolean isSmokingAllowed, @JsonProperty("isPetFriendly") Boolean isPetFriendly, @JsonProperty("lessorName") String lessorName, - @JsonProperty("additionalRentCost") Integer additionalRentCost, + @JsonProperty("additionalRentCost") BigDecimal additionalRentCost, @JsonProperty("level") Integer level, @JsonProperty("furnishing") String furnishing, @JsonProperty("description") String description, diff --git a/announcement-processor-extractor/src/main/java/extractor/service/AnnouncementExtractingService.java b/announcement-processor-extractor/src/main/java/extractor/service/AnnouncementExtractingService.java index e82cd0b..aea3372 100644 --- a/announcement-processor-extractor/src/main/java/extractor/service/AnnouncementExtractingService.java +++ b/announcement-processor-extractor/src/main/java/extractor/service/AnnouncementExtractingService.java @@ -4,6 +4,7 @@ import extractor.entity.*; import org.springframework.stereotype.Service; +import java.math.BigDecimal; import java.util.HashMap; import java.util.Map; @@ -47,7 +48,7 @@ private Price extractPrice(AnnouncementDto announcementDto) { Price price = new Price(); price.setBasePrice(announcementDto.getPrice()); // TODO more advanced extraction e.g. description data extraction - Map pricesMap = new HashMap<>(); + Map pricesMap = new HashMap<>(); parseDescriptionPrice(announcementDto.getDescription(), pricesMap); pricesMap.put("SomePrice", announcementDto.getAdditionalRentCost()); price.setAdditionalPrices(pricesMap); @@ -69,7 +70,7 @@ private PropertyData extractPropertyData(AnnouncementDto announcementDto) { return propertyData; } - private void parseDescriptionPrice(String description, Map pricesMap) { + private void parseDescriptionPrice(String description, Map pricesMap) { } } diff --git a/announcement-processor-extractor/src/main/java/extractor/service/AnnouncementSenderService.java b/announcement-processor-extractor/src/main/java/extractor/service/AnnouncementSenderService.java new file mode 100644 index 0000000..a58780c --- /dev/null +++ b/announcement-processor-extractor/src/main/java/extractor/service/AnnouncementSenderService.java @@ -0,0 +1,70 @@ +package extractor.service; + +import extractor.entity.Announcement; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; + +import org.springframework.beans.factory.annotation.Value; + +@Slf4j +@Service +public class AnnouncementSenderService { + + @Value("${net.api.url}") + private String apiUrl; + private MapperService mapper; + + public AnnouncementSenderService(MapperService mapper) { + this.mapper = mapper; + } + + public void send(Announcement announcement) { + + try { + + log.info("Sending data to url: {}", apiUrl); + URL url = new URL(this.apiUrl); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setDoOutput(true); + conn.setRequestMethod("POST"); + conn.setRequestProperty("Content-Type", "application/json"); + + log.debug("Mapping announcement to JSON..."); + String announcementJson = mapper.getJsonFromAnnouncement(announcement); + log.debug("Mapped announcement JSON: {}", announcementJson); + + OutputStream os = conn.getOutputStream(); + os.write(announcementJson.getBytes()); + os.flush(); + + if (conn.getResponseCode() != HttpURLConnection.HTTP_CREATED) { + throw new RuntimeException("Failed : HTTP error code : " + + conn.getResponseCode()); + } + + BufferedReader br = new BufferedReader(new InputStreamReader((conn.getInputStream()))); + String output; + log.info("Output from server: \n"); + + while ((output = br.readLine()) != null) { + System.out.println(output); + } + + conn.disconnect(); + + } catch ( + MalformedURLException e) { + log.error("Exception: {}", e); + } catch (IOException e) { + log.error("Exception: {}", e); + } + } +} \ No newline at end of file diff --git a/announcement-processor-extractor/src/main/java/extractor/service/MapperService.java b/announcement-processor-extractor/src/main/java/extractor/service/MapperService.java index a8ec52e..395b4ab 100644 --- a/announcement-processor-extractor/src/main/java/extractor/service/MapperService.java +++ b/announcement-processor-extractor/src/main/java/extractor/service/MapperService.java @@ -1,8 +1,10 @@ package extractor.service; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import extractor.dto.AnnouncementDto; +import extractor.entity.Announcement; import org.springframework.stereotype.Service; import java.io.IOException; @@ -25,4 +27,8 @@ public AnnouncementDto getAnnouncementDtoFromJsonString(String json) { return null; } } + + public String getJsonFromAnnouncement(Announcement announcement) throws JsonProcessingException { + return mapper.writeValueAsString(announcement); + } } diff --git a/announcement-processor-extractor/src/main/resources/application-dev.properties b/announcement-processor-extractor/src/main/resources/application-dev.properties index 23efd10..ec9ca01 100644 --- a/announcement-processor-extractor/src/main/resources/application-dev.properties +++ b/announcement-processor-extractor/src/main/resources/application-dev.properties @@ -1,2 +1,4 @@ brokerURL=tcp://192.168.99.100:32768 -queue=announcements \ No newline at end of file +queue=announcements + +net.url="http://localhost:8080/api/announcement/" \ No newline at end of file diff --git a/announcement-processor-extractor/src/test/java/extractor/consumer/AnnouncementConsumerFromFileTest.java b/announcement-processor-extractor/src/test/java/extractor/consumer/AnnouncementConsumerFromFileTest.java index c27d060..51b3046 100644 --- a/announcement-processor-extractor/src/test/java/extractor/consumer/AnnouncementConsumerFromFileTest.java +++ b/announcement-processor-extractor/src/test/java/extractor/consumer/AnnouncementConsumerFromFileTest.java @@ -4,6 +4,7 @@ import org.junit.Assert; import org.junit.Test; +import java.math.BigDecimal; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; @@ -15,7 +16,7 @@ public class AnnouncementConsumerFromFileTest { @Test public void consumeAnnouncementTest() { String title = "Kawalerka 40m Głowackiego 4 Kraków PRZESTRONNE niskie koszty WIDOK na PANORAMĘ Krakowa POŁUDNIE"; - Integer price = 1300; + BigDecimal price = new BigDecimal(1300); String strDate = "2018-11-17 00:00:00"; DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); LocalDateTime creationDate = LocalDateTime.parse(strDate, formatter); @@ -28,7 +29,7 @@ public void consumeAnnouncementTest() { Boolean isSmokingAllowed = true; Boolean isPetFriendly = false; String lessorName = "Bartek"; - Integer additionalRentCost = null; + BigDecimal additionalRentCost = null; Integer level = null; String furnishing = null; String description = "Proponuję mieszkanie na wynajem (kawalerka) o powierzchni 40m2 " + From c2613545f17f018587d26baa9d910d63ee30fa47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 6 Jan 2019 18:55:50 +0100 Subject: [PATCH 122/135] Implemented endpoint for detailed info for announcement --- .../src/main/java/api/entity/Location.java | 6 --- .../api/model/DetailedAnnouncementInfo.java | 23 ++++++++++ .../java/api/service/FrontEndDataService.java | 46 +++++++++++++++++-- 3 files changed, 65 insertions(+), 10 deletions(-) create mode 100644 announcement-processor-api/src/main/java/api/model/DetailedAnnouncementInfo.java diff --git a/announcement-processor-api/src/main/java/api/entity/Location.java b/announcement-processor-api/src/main/java/api/entity/Location.java index a870f75..eefd20c 100644 --- a/announcement-processor-api/src/main/java/api/entity/Location.java +++ b/announcement-processor-api/src/main/java/api/entity/Location.java @@ -36,10 +36,4 @@ public class Location { @OneToOne(mappedBy = "location") private Announcement announcement; - - public String getFullLocation() { - return city + " " + zipCode + " " + district + "\n" - + street + " " + buildingNumber + " / " + flatNumber + "\n" - + country; - } } diff --git a/announcement-processor-api/src/main/java/api/model/DetailedAnnouncementInfo.java b/announcement-processor-api/src/main/java/api/model/DetailedAnnouncementInfo.java new file mode 100644 index 0000000..d27786e --- /dev/null +++ b/announcement-processor-api/src/main/java/api/model/DetailedAnnouncementInfo.java @@ -0,0 +1,23 @@ +package api.model; + +import api.entity.AdditionalCosts; +import api.entity.Lessor; +import api.entity.PropertyData; +import lombok.Data; + +import java.util.List; + +@Data +public class DetailedAnnouncementInfo { + private String title; + private String baseCost; + private String provider; + private String creationDate; + private String url; + private String location; + private String description; + + private PropertyData propertyData; + private Lessor lessor; + private List additionalCosts; +} diff --git a/announcement-processor-api/src/main/java/api/service/FrontEndDataService.java b/announcement-processor-api/src/main/java/api/service/FrontEndDataService.java index 826d211..a3a0f62 100644 --- a/announcement-processor-api/src/main/java/api/service/FrontEndDataService.java +++ b/announcement-processor-api/src/main/java/api/service/FrontEndDataService.java @@ -1,6 +1,7 @@ package api.service; import api.entity.Announcement; +import api.entity.Location; import api.entity.PriceOffer; import api.entity.price.ConsumerPrice; import api.entity.price.ContactPrice; @@ -11,6 +12,8 @@ import api.repository.AnnouncementRepository; import org.springframework.stereotype.Service; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; @@ -42,24 +45,59 @@ public GeneralAnnouncementInfo getGeneralInfoAnnouncement(Integer id, String des } public DetailedAnnouncementInfo getDetailedInfoAnnouncement(Integer id, String desiredCurrency) { - return null; + Optional ann = announcementRepository.findById(id); + return ann.map(announcement -> convertAnnouncementToDetailedFormat(announcement, desiredCurrency)) + .orElse(null); } private GeneralAnnouncementInfo convertAnnouncementToGeneralFormat(Announcement announcement, String desiredCurrency) { - String cost = getBaseCostText(announcement, desiredCurrency, currencyService); GeneralAnnouncementInfo generalAnnouncementInfo = new GeneralAnnouncementInfo(); generalAnnouncementInfo.setBaseCost(cost); - generalAnnouncementInfo.setCreationDate(announcement.getCreationDate().toString()); + generalAnnouncementInfo.setCreationDate(getFormattedLocalDateTime(announcement.getCreationDate())); generalAnnouncementInfo.setLessorType(announcement.getLessor().getLessorType()); - generalAnnouncementInfo.setLocation(announcement.getLocation().getFullLocation()); + generalAnnouncementInfo.setLocation(getFullLocation(announcement.getLocation())); generalAnnouncementInfo.setProvider(announcement.getProvider()); generalAnnouncementInfo.setTitle(announcement.getTitle()); generalAnnouncementInfo.setUrl(announcement.getUrl()); return generalAnnouncementInfo; } + private DetailedAnnouncementInfo convertAnnouncementToDetailedFormat(Announcement announcement, String desiredCurrency) { + String cost = getBaseCostText(announcement, desiredCurrency, currencyService); + + DetailedAnnouncementInfo detailedAnnouncementInfo = new DetailedAnnouncementInfo(); + + detailedAnnouncementInfo.setTitle(announcement.getTitle()); + detailedAnnouncementInfo.setBaseCost(cost); + detailedAnnouncementInfo.setProvider(announcement.getProvider()); + detailedAnnouncementInfo.setCreationDate(getFormattedLocalDateTime(announcement.getCreationDate())); + detailedAnnouncementInfo.setUrl(announcement.getUrl()); + detailedAnnouncementInfo.setLocation(getFullLocation(announcement.getLocation())); + detailedAnnouncementInfo.setDescription(announcement.getDescription()); + + detailedAnnouncementInfo.setPropertyData(announcement.getPropertyData()); + detailedAnnouncementInfo.setAdditionalCosts(announcement.getAdditionalCosts()); + detailedAnnouncementInfo.setLessor(announcement.getLessor()); + + return detailedAnnouncementInfo; + } + + private String getFullLocation(Location location) { + return location.getCity() + " " + location.getZipCode() + " " + location.getDistrict() + "\n" + + location.getStreet() + " " + location.getBuildingNumber() + " / " + location.getFlatNumber() + "\n" + + location.getCountry(); + } + + private String getFormattedLocalDateTime(LocalDateTime ldt) { + if (ldt != null) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + return ldt.format(formatter); + } + return null; + } + private String getBaseCostText(Announcement announcement, String desiredCurrency, CurrencyService currencyService) { PriceOffer priceOffer = announcement.getPriceOffer(); From b9ff29a85209f33f83e88b23bffd29e07f229079 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Sun, 6 Jan 2019 19:44:03 +0100 Subject: [PATCH 123/135] optional @RequestParam and NO_CONTENT status when no results --- .../java/api/controller/AnnouncementsController.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/announcement-processor-api/src/main/java/api/controller/AnnouncementsController.java b/announcement-processor-api/src/main/java/api/controller/AnnouncementsController.java index 65ba071..66b7eca 100644 --- a/announcement-processor-api/src/main/java/api/controller/AnnouncementsController.java +++ b/announcement-processor-api/src/main/java/api/controller/AnnouncementsController.java @@ -22,32 +22,32 @@ public AnnouncementsController(ExtractorDataService extractorDataService, FrontE } @GetMapping("/get/all") - public ResponseEntity> getAllAnnouncements(@RequestParam String currency) { + public ResponseEntity> getAllAnnouncements(@RequestParam(required = false) String currency) { List allAnnouncements = frontEndDataService.getAllAnnouncements(currency); if (allAnnouncements != null) { return new ResponseEntity<>(allAnnouncements, HttpStatus.OK); } else { - return new ResponseEntity<>(HttpStatus.NOT_FOUND); + return new ResponseEntity<>(HttpStatus.NO_CONTENT); } } @GetMapping("get/{id}/detailed-info") - public ResponseEntity getDetailedInfoAnnouncement(@PathVariable Integer id, @RequestParam String currency) { + public ResponseEntity getDetailedInfoAnnouncement(@PathVariable Integer id, @RequestParam(required = false) String currency) { DetailedAnnouncementInfo detailedInfoAnnouncement = frontEndDataService.getDetailedInfoAnnouncement(id, currency); if (detailedInfoAnnouncement != null) { return new ResponseEntity<>(detailedInfoAnnouncement, HttpStatus.OK); } else { - return new ResponseEntity<>(HttpStatus.NOT_FOUND); + return new ResponseEntity<>(HttpStatus.NO_CONTENT); } } @GetMapping("get/{id}/general-info") - public ResponseEntity getGeneralInfoAnnouncement(@PathVariable Integer id, @RequestParam String currency) { + public ResponseEntity getGeneralInfoAnnouncement(@PathVariable Integer id, @RequestParam(required = false) String currency) { GeneralAnnouncementInfo generalInfoAnnouncement = frontEndDataService.getGeneralInfoAnnouncement(id, currency); if (generalInfoAnnouncement != null) { return new ResponseEntity<>(generalInfoAnnouncement, HttpStatus.OK); } else { - return new ResponseEntity<>(HttpStatus.NOT_FOUND); + return new ResponseEntity<>(HttpStatus.NO_CONTENT); } } } From 337adc6f2a80a2dc2ce059564406664b54798d50 Mon Sep 17 00:00:00 2001 From: michalsroka Date: Sun, 6 Jan 2019 19:46:29 +0100 Subject: [PATCH 124/135] Add AnnouncementDto to API --- .../controller/AnnouncementsController.java | 12 ++++++ .../main/java/api/dto/AnnouncementDto.java | 43 +++++++++++++++++++ .../api/dto/announcementElements/Lessor.java | 14 ++++++ .../dto/announcementElements/Location.java | 17 ++++++++ .../api/dto/announcementElements/Price.java | 16 +++++++ .../announcementElements/PropertyData.java | 19 ++++++++ .../src/main/java/extractor/entity/Price.java | 14 +++--- .../service/AnnouncementSenderService.java | 12 +++--- .../main/resources/application-dev.properties | 2 +- 9 files changed, 135 insertions(+), 14 deletions(-) create mode 100644 announcement-processor-api/src/main/java/api/dto/AnnouncementDto.java create mode 100644 announcement-processor-api/src/main/java/api/dto/announcementElements/Lessor.java create mode 100644 announcement-processor-api/src/main/java/api/dto/announcementElements/Location.java create mode 100644 announcement-processor-api/src/main/java/api/dto/announcementElements/Price.java create mode 100644 announcement-processor-api/src/main/java/api/dto/announcementElements/PropertyData.java diff --git a/announcement-processor-api/src/main/java/api/controller/AnnouncementsController.java b/announcement-processor-api/src/main/java/api/controller/AnnouncementsController.java index c2f9bff..2fb16b5 100644 --- a/announcement-processor-api/src/main/java/api/controller/AnnouncementsController.java +++ b/announcement-processor-api/src/main/java/api/controller/AnnouncementsController.java @@ -2,6 +2,8 @@ import api.service.ExtractorDataService; import api.service.FrontEndDataService; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -15,4 +17,14 @@ public AnnouncementsController(ExtractorDataService extractorDataService, FrontE this.extractorDataService = extractorDataService; this.frontEndDataService = frontEndDataService; } + + @PostMapping("/send") + public void receiveAnnouncement() { + + } + + @GetMapping("/lala") + public void qubec() { + System.out.println("lala"); + } } diff --git a/announcement-processor-api/src/main/java/api/dto/AnnouncementDto.java b/announcement-processor-api/src/main/java/api/dto/AnnouncementDto.java new file mode 100644 index 0000000..b149429 --- /dev/null +++ b/announcement-processor-api/src/main/java/api/dto/AnnouncementDto.java @@ -0,0 +1,43 @@ +package api.dto; + +import api.dto.announcementElements.Lessor; +import api.dto.announcementElements.Location; +import api.dto.announcementElements.Price; +import api.dto.announcementElements.PropertyData; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.time.LocalDateTime; + +public class AnnouncementDto { + + public AnnouncementDto( + @JsonProperty("title") String title, + @JsonProperty("price") Price price, + @JsonProperty("location") Location location, + @JsonProperty("propertyData") PropertyData propertyData, + @JsonProperty("lessor") Lessor lessor, + @JsonProperty("creationDate") LocalDateTime creationDate, + @JsonProperty("url") String url, + @JsonProperty("description") String description, + @JsonProperty("provider") String provider) { + this.title = title; + this.price = price; + this.location = location; + this.propertyData = propertyData; + this.lessor = lessor; + this.creationDate = creationDate; + this.url = url; + this.description = description; + this.provider = provider; + } + + private String title; + private Price price; + private Location location; + private PropertyData propertyData; + private Lessor lessor; + private LocalDateTime creationDate; + private String url; + private String description; + private String provider; +} diff --git a/announcement-processor-api/src/main/java/api/dto/announcementElements/Lessor.java b/announcement-processor-api/src/main/java/api/dto/announcementElements/Lessor.java new file mode 100644 index 0000000..656e6e6 --- /dev/null +++ b/announcement-processor-api/src/main/java/api/dto/announcementElements/Lessor.java @@ -0,0 +1,14 @@ +package api.dto.announcementElements; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class Lessor { + + private String name; + private String lessorType; // to enum + private String phoneNumber; + private String email; +} diff --git a/announcement-processor-api/src/main/java/api/dto/announcementElements/Location.java b/announcement-processor-api/src/main/java/api/dto/announcementElements/Location.java new file mode 100644 index 0000000..1970960 --- /dev/null +++ b/announcement-processor-api/src/main/java/api/dto/announcementElements/Location.java @@ -0,0 +1,17 @@ +package api.dto.announcementElements; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class Location { + + private String country; + private String city; + private String street; + private String zipCode; + private String buildingNumber; + private String flatNumber; + private String district; +} diff --git a/announcement-processor-api/src/main/java/api/dto/announcementElements/Price.java b/announcement-processor-api/src/main/java/api/dto/announcementElements/Price.java new file mode 100644 index 0000000..2ca5077 --- /dev/null +++ b/announcement-processor-api/src/main/java/api/dto/announcementElements/Price.java @@ -0,0 +1,16 @@ +package api.dto.announcementElements; + +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.math.BigDecimal; +import java.util.Map; + +@Data +@AllArgsConstructor +public class Price { + + private BigDecimal basePrice; + private String currency = "PLN"; + private Map additionalPrices; +} diff --git a/announcement-processor-api/src/main/java/api/dto/announcementElements/PropertyData.java b/announcement-processor-api/src/main/java/api/dto/announcementElements/PropertyData.java new file mode 100644 index 0000000..72f3a85 --- /dev/null +++ b/announcement-processor-api/src/main/java/api/dto/announcementElements/PropertyData.java @@ -0,0 +1,19 @@ +package api.dto.announcementElements; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class PropertyData { + + private String propertyType; // to enum? + private Double area; + private Boolean isSmokingAllowed; + private Boolean isPerFriendly; + private Integer roomNumber; + private Integer bathroomNumber; + private String parkingAvailability; + private Integer level; + private String furnishing; +} diff --git a/announcement-processor-extractor/src/main/java/extractor/entity/Price.java b/announcement-processor-extractor/src/main/java/extractor/entity/Price.java index 5213528..8f2ea16 100644 --- a/announcement-processor-extractor/src/main/java/extractor/entity/Price.java +++ b/announcement-processor-extractor/src/main/java/extractor/entity/Price.java @@ -21,11 +21,11 @@ public class Price { private String currency = "PLN"; private Map additionalPrices; - public BigDecimal getSummedPrice() { - return basePrice.add( - additionalPrices.values() - .stream() - .reduce(BigDecimal.ZERO, BigDecimal::add) - ); - } +// public BigDecimal getSummedPrice() { +// return basePrice.add( +// additionalPrices.values() +// .stream() +// .reduce(BigDecimal.ZERO, BigDecimal::add) +// ); +// } } diff --git a/announcement-processor-extractor/src/main/java/extractor/service/AnnouncementSenderService.java b/announcement-processor-extractor/src/main/java/extractor/service/AnnouncementSenderService.java index a58780c..3b489fa 100644 --- a/announcement-processor-extractor/src/main/java/extractor/service/AnnouncementSenderService.java +++ b/announcement-processor-extractor/src/main/java/extractor/service/AnnouncementSenderService.java @@ -30,6 +30,10 @@ public void send(Announcement announcement) { try { + log.debug("Mapping announcement to JSON..."); + String announcementJson = mapper.getJsonFromAnnouncement(announcement); + log.info("Mapped announcement JSON: {}", announcementJson); + log.info("Sending data to url: {}", apiUrl); URL url = new URL(this.apiUrl); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); @@ -37,10 +41,6 @@ public void send(Announcement announcement) { conn.setRequestMethod("POST"); conn.setRequestProperty("Content-Type", "application/json"); - log.debug("Mapping announcement to JSON..."); - String announcementJson = mapper.getJsonFromAnnouncement(announcement); - log.debug("Mapped announcement JSON: {}", announcementJson); - OutputStream os = conn.getOutputStream(); os.write(announcementJson.getBytes()); os.flush(); @@ -62,9 +62,9 @@ public void send(Announcement announcement) { } catch ( MalformedURLException e) { - log.error("Exception: {}", e); + log.error("Exception:", e); } catch (IOException e) { - log.error("Exception: {}", e); + log.error("Exception:", e); } } } \ No newline at end of file diff --git a/announcement-processor-extractor/src/main/resources/application-dev.properties b/announcement-processor-extractor/src/main/resources/application-dev.properties index ec9ca01..debb5eb 100644 --- a/announcement-processor-extractor/src/main/resources/application-dev.properties +++ b/announcement-processor-extractor/src/main/resources/application-dev.properties @@ -1,4 +1,4 @@ brokerURL=tcp://192.168.99.100:32768 queue=announcements -net.url="http://localhost:8080/api/announcement/" \ No newline at end of file +net.api.url="http://localhost:8080/announcements/send" \ No newline at end of file From 721f653cfda84e6c0bcf05770a722f5a43e98a65 Mon Sep 17 00:00:00 2001 From: michalsroka Date: Sun, 6 Jan 2019 23:34:25 +0100 Subject: [PATCH 125/135] Change objects to DTO objects in api --- .../controller/AnnouncementsController.java | 20 +++++++-------- .../main/java/api/dto/AnnouncementDto.java | 25 +++++++++---------- .../Lessor.java => LessorDto.java} | 4 +-- .../Location.java => LocationDto.java} | 5 ++-- .../Price.java => PriceDto.java} | 6 ++--- ...PropertyData.java => PropertyDataDto.java} | 6 ++--- .../api/service/ExtractorDataService.java | 1 + .../service/AnnouncementSenderService.java | 2 +- 8 files changed, 35 insertions(+), 34 deletions(-) rename announcement-processor-api/src/main/java/api/dto/{announcementElements/Lessor.java => LessorDto.java} (77%) rename announcement-processor-api/src/main/java/api/dto/{announcementElements/Location.java => LocationDto.java} (82%) rename announcement-processor-api/src/main/java/api/dto/{announcementElements/Price.java => PriceDto.java} (70%) rename announcement-processor-api/src/main/java/api/dto/{announcementElements/PropertyData.java => PropertyDataDto.java} (75%) diff --git a/announcement-processor-api/src/main/java/api/controller/AnnouncementsController.java b/announcement-processor-api/src/main/java/api/controller/AnnouncementsController.java index 2fb16b5..f5d3e65 100644 --- a/announcement-processor-api/src/main/java/api/controller/AnnouncementsController.java +++ b/announcement-processor-api/src/main/java/api/controller/AnnouncementsController.java @@ -1,12 +1,14 @@ package api.controller; +import api.dto.AnnouncementDto; import api.service.ExtractorDataService; import api.service.FrontEndDataService; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; +@Slf4j @RestController @RequestMapping("/announcements") public class AnnouncementsController { @@ -19,12 +21,10 @@ public AnnouncementsController(ExtractorDataService extractorDataService, FrontE } @PostMapping("/send") - public void receiveAnnouncement() { + public ResponseEntity receiveAnnouncement(@RequestBody AnnouncementDto announcementDto) { + log.debug("Received receiveAnnouncement POST request with body: {}", announcementDto); + log.info("Received receiveAnnouncement POST request with announcement URL: {}", announcementDto.getUrl()); - } - - @GetMapping("/lala") - public void qubec() { - System.out.println("lala"); + return new ResponseEntity<>(announcementDto, HttpStatus.OK); } } diff --git a/announcement-processor-api/src/main/java/api/dto/AnnouncementDto.java b/announcement-processor-api/src/main/java/api/dto/AnnouncementDto.java index b149429..73864ac 100644 --- a/announcement-processor-api/src/main/java/api/dto/AnnouncementDto.java +++ b/announcement-processor-api/src/main/java/api/dto/AnnouncementDto.java @@ -1,21 +1,20 @@ package api.dto; -import api.dto.announcementElements.Lessor; -import api.dto.announcementElements.Location; -import api.dto.announcementElements.Price; -import api.dto.announcementElements.PropertyData; import com.fasterxml.jackson.annotation.JsonProperty; - +import lombok.Data; +import lombok.ToString; import java.time.LocalDateTime; +@ToString +@Data public class AnnouncementDto { public AnnouncementDto( @JsonProperty("title") String title, - @JsonProperty("price") Price price, - @JsonProperty("location") Location location, - @JsonProperty("propertyData") PropertyData propertyData, - @JsonProperty("lessor") Lessor lessor, + @JsonProperty("price") PriceDto price, + @JsonProperty("location") LocationDto location, + @JsonProperty("propertyData") PropertyDataDto propertyData, + @JsonProperty("lessor") LessorDto lessor, @JsonProperty("creationDate") LocalDateTime creationDate, @JsonProperty("url") String url, @JsonProperty("description") String description, @@ -32,10 +31,10 @@ public AnnouncementDto( } private String title; - private Price price; - private Location location; - private PropertyData propertyData; - private Lessor lessor; + private PriceDto price; + private LocationDto location; + private PropertyDataDto propertyData; + private LessorDto lessor; private LocalDateTime creationDate; private String url; private String description; diff --git a/announcement-processor-api/src/main/java/api/dto/announcementElements/Lessor.java b/announcement-processor-api/src/main/java/api/dto/LessorDto.java similarity index 77% rename from announcement-processor-api/src/main/java/api/dto/announcementElements/Lessor.java rename to announcement-processor-api/src/main/java/api/dto/LessorDto.java index 656e6e6..d8b2ee9 100644 --- a/announcement-processor-api/src/main/java/api/dto/announcementElements/Lessor.java +++ b/announcement-processor-api/src/main/java/api/dto/LessorDto.java @@ -1,11 +1,11 @@ -package api.dto.announcementElements; +package api.dto; import lombok.AllArgsConstructor; import lombok.Data; @Data @AllArgsConstructor -public class Lessor { +public class LessorDto { private String name; private String lessorType; // to enum diff --git a/announcement-processor-api/src/main/java/api/dto/announcementElements/Location.java b/announcement-processor-api/src/main/java/api/dto/LocationDto.java similarity index 82% rename from announcement-processor-api/src/main/java/api/dto/announcementElements/Location.java rename to announcement-processor-api/src/main/java/api/dto/LocationDto.java index 1970960..2ccbc6f 100644 --- a/announcement-processor-api/src/main/java/api/dto/announcementElements/Location.java +++ b/announcement-processor-api/src/main/java/api/dto/LocationDto.java @@ -1,11 +1,12 @@ -package api.dto.announcementElements; +package api.dto; import lombok.AllArgsConstructor; import lombok.Data; @Data @AllArgsConstructor -public class Location { +public class LocationDto { + private String country; private String city; diff --git a/announcement-processor-api/src/main/java/api/dto/announcementElements/Price.java b/announcement-processor-api/src/main/java/api/dto/PriceDto.java similarity index 70% rename from announcement-processor-api/src/main/java/api/dto/announcementElements/Price.java rename to announcement-processor-api/src/main/java/api/dto/PriceDto.java index 2ca5077..385d28e 100644 --- a/announcement-processor-api/src/main/java/api/dto/announcementElements/Price.java +++ b/announcement-processor-api/src/main/java/api/dto/PriceDto.java @@ -1,4 +1,4 @@ -package api.dto.announcementElements; +package api.dto; import lombok.AllArgsConstructor; import lombok.Data; @@ -8,9 +8,9 @@ @Data @AllArgsConstructor -public class Price { +public class PriceDto { private BigDecimal basePrice; - private String currency = "PLN"; + private String currency; private Map additionalPrices; } diff --git a/announcement-processor-api/src/main/java/api/dto/announcementElements/PropertyData.java b/announcement-processor-api/src/main/java/api/dto/PropertyDataDto.java similarity index 75% rename from announcement-processor-api/src/main/java/api/dto/announcementElements/PropertyData.java rename to announcement-processor-api/src/main/java/api/dto/PropertyDataDto.java index 72f3a85..e98c00f 100644 --- a/announcement-processor-api/src/main/java/api/dto/announcementElements/PropertyData.java +++ b/announcement-processor-api/src/main/java/api/dto/PropertyDataDto.java @@ -1,13 +1,13 @@ -package api.dto.announcementElements; +package api.dto; import lombok.AllArgsConstructor; import lombok.Data; @Data @AllArgsConstructor -public class PropertyData { +public class PropertyDataDto { - private String propertyType; // to enum? + private String propertyType; private Double area; private Boolean isSmokingAllowed; private Boolean isPerFriendly; diff --git a/announcement-processor-api/src/main/java/api/service/ExtractorDataService.java b/announcement-processor-api/src/main/java/api/service/ExtractorDataService.java index 30dd4d8..86607d2 100644 --- a/announcement-processor-api/src/main/java/api/service/ExtractorDataService.java +++ b/announcement-processor-api/src/main/java/api/service/ExtractorDataService.java @@ -4,4 +4,5 @@ @Service public class ExtractorDataService { + } diff --git a/announcement-processor-extractor/src/main/java/extractor/service/AnnouncementSenderService.java b/announcement-processor-extractor/src/main/java/extractor/service/AnnouncementSenderService.java index 3b489fa..72e50ad 100644 --- a/announcement-processor-extractor/src/main/java/extractor/service/AnnouncementSenderService.java +++ b/announcement-processor-extractor/src/main/java/extractor/service/AnnouncementSenderService.java @@ -45,7 +45,7 @@ public void send(Announcement announcement) { os.write(announcementJson.getBytes()); os.flush(); - if (conn.getResponseCode() != HttpURLConnection.HTTP_CREATED) { + if (conn.getResponseCode() != HttpURLConnection.HTTP_OK) { throw new RuntimeException("Failed : HTTP error code : " + conn.getResponseCode()); } From 509d5f20b48974bc6b86cd8190d79239aac616d6 Mon Sep 17 00:00:00 2001 From: michalsroka Date: Mon, 7 Jan 2019 02:10:01 +0100 Subject: [PATCH 126/135] Add mapping AnnouncementDto to Announcement --- announcement-processor-api/pom.xml | 7 ++ .../api/configuration/ModelMapperConfig.java | 17 ++++ .../controller/AnnouncementsController.java | 3 +- .../AnnouncementDtoEntityConverter.java | 78 +++++++++++++++++++ .../main/java/api/entity/PropertyData.java | 2 +- .../repository/AnnouncementRepository.java | 2 + .../api/service/ExtractorDataService.java | 29 +++++++ .../consumer/ReceivementScheduler.java | 2 +- 8 files changed, 137 insertions(+), 3 deletions(-) create mode 100644 announcement-processor-api/src/main/java/api/configuration/ModelMapperConfig.java create mode 100644 announcement-processor-api/src/main/java/api/converter/AnnouncementDtoEntityConverter.java diff --git a/announcement-processor-api/pom.xml b/announcement-processor-api/pom.xml index bff3df2..e199f55 100644 --- a/announcement-processor-api/pom.xml +++ b/announcement-processor-api/pom.xml @@ -77,6 +77,13 @@ org.springframework.boot spring-boot-starter-data-jpa + + + + org.modelmapper + modelmapper + 2.3.2 + diff --git a/announcement-processor-api/src/main/java/api/configuration/ModelMapperConfig.java b/announcement-processor-api/src/main/java/api/configuration/ModelMapperConfig.java new file mode 100644 index 0000000..4abaf4b --- /dev/null +++ b/announcement-processor-api/src/main/java/api/configuration/ModelMapperConfig.java @@ -0,0 +1,17 @@ +package api.configuration; + +import api.converter.AnnouncementDtoEntityConverter; +import org.modelmapper.ModelMapper; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class ModelMapperConfig { + + @Bean + public ModelMapper modelMapper() { + ModelMapper mapper = new ModelMapper(); + mapper.addConverter(new AnnouncementDtoEntityConverter()); + return mapper; + } +} diff --git a/announcement-processor-api/src/main/java/api/controller/AnnouncementsController.java b/announcement-processor-api/src/main/java/api/controller/AnnouncementsController.java index f5d3e65..18fd6e8 100644 --- a/announcement-processor-api/src/main/java/api/controller/AnnouncementsController.java +++ b/announcement-processor-api/src/main/java/api/controller/AnnouncementsController.java @@ -24,7 +24,8 @@ public AnnouncementsController(ExtractorDataService extractorDataService, FrontE public ResponseEntity receiveAnnouncement(@RequestBody AnnouncementDto announcementDto) { log.debug("Received receiveAnnouncement POST request with body: {}", announcementDto); log.info("Received receiveAnnouncement POST request with announcement URL: {}", announcementDto.getUrl()); - + extractorDataService.convertAndSave(announcementDto); + log.info("Extracted and saved to DB; announcement URL: {}", announcementDto.getUrl()); return new ResponseEntity<>(announcementDto, HttpStatus.OK); } } diff --git a/announcement-processor-api/src/main/java/api/converter/AnnouncementDtoEntityConverter.java b/announcement-processor-api/src/main/java/api/converter/AnnouncementDtoEntityConverter.java new file mode 100644 index 0000000..2367bc1 --- /dev/null +++ b/announcement-processor-api/src/main/java/api/converter/AnnouncementDtoEntityConverter.java @@ -0,0 +1,78 @@ +package api.converter; + +import api.dto.AnnouncementDto; +import api.entity.*; +import org.modelmapper.Converter; +import org.modelmapper.spi.MappingContext; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class AnnouncementDtoEntityConverter implements Converter { + + @Override + public Announcement convert(MappingContext mappingContext) { + + AnnouncementDto announcementDto = mappingContext.getSource(); + + Announcement announcement = new Announcement(); + announcement.setTitle(announcementDto.getTitle()); + announcement.setProvider(announcementDto.getProvider()); + announcement.setCreationDate(announcementDto.getCreationDate()); + announcement.setUrl(announcementDto.getUrl()); + announcement.setDescription(announcementDto.getDescription()); + announcement.setCurrency(announcementDto.getPrice().getCurrency()); + + PriceOffer priceOffer = new PriceOffer(); + // TODO check what the description is about + priceOffer.setAnnouncement(announcement); + priceOffer.setDescription(announcement.getDescription()); + announcement.setPriceOffer(priceOffer); + + Location location = new Location(); + location.setAnnouncement(announcement); + location.setCountry(announcementDto.getLocation().getCountry()); + location.setCity(announcementDto.getLocation().getCity()); + location.setStreet(announcementDto.getLocation().getStreet()); + location.setZipCode(announcementDto.getLocation().getZipCode()); + location.setBuildingNumber(announcementDto.getLocation().getBuildingNumber()); + location.setFlatNumber(announcementDto.getLocation().getFlatNumber()); + location.setDistrict(announcementDto.getLocation().getDistrict()); + announcement.setLocation(location); + + PropertyData propertyData = new PropertyData(); + propertyData.setAnnouncement(announcement); + propertyData.setPropertyType(announcementDto.getPropertyData().getPropertyType()); + propertyData.setArea(announcementDto.getPropertyData().getArea()); + propertyData.setIsSmokingAllowed(announcementDto.getPropertyData().getIsSmokingAllowed()); + propertyData.setIsPetFriendly(announcementDto.getPropertyData().getIsPerFriendly()); + propertyData.setRoomNumber(announcementDto.getPropertyData().getRoomNumber()); + propertyData.setBathroomNumber(announcementDto.getPropertyData().getBathroomNumber()); + propertyData.setParkingAvailability(announcementDto.getPropertyData().getParkingAvailability()); + propertyData.setLevel(announcementDto.getPropertyData().getLevel()); + propertyData.setFurnishing(announcementDto.getPropertyData().getFurnishing()); + announcement.setPropertyData(propertyData); + + Lessor lessor = new Lessor(); + lessor.setAnnouncement(announcement); + lessor.setName(announcementDto.getLessor().getName()); + lessor.setLessorType(announcementDto.getLessor().getLessorType()); + lessor.setPhoneNumber(announcementDto.getLessor().getPhoneNumber()); + lessor.setEmail(announcementDto.getLessor().getEmail()); + announcement.setLessor(lessor); + + List additionalCostsList = new ArrayList<>(); + for (Map.Entry entry : announcementDto.getPrice().getAdditionalPrices().entrySet()) { + AdditionalCosts additionalCosts = new AdditionalCosts(); + additionalCosts.setAnnouncement(announcement); + additionalCosts.setCostName(entry.getKey()); + additionalCosts.setCostPrice(entry.getValue()); + additionalCostsList.add(additionalCosts); + } + announcement.setAdditionalCosts(additionalCostsList); + + return announcement; + } +} diff --git a/announcement-processor-api/src/main/java/api/entity/PropertyData.java b/announcement-processor-api/src/main/java/api/entity/PropertyData.java index 77a87aa..6e9d4d6 100644 --- a/announcement-processor-api/src/main/java/api/entity/PropertyData.java +++ b/announcement-processor-api/src/main/java/api/entity/PropertyData.java @@ -23,7 +23,7 @@ public class PropertyData { private Boolean isSmokingAllowed; @Column(name = "IS_FOR_PETS") - private Boolean isPerFriendly; + private Boolean isPetFriendly; @Column(name = "ROOM_NUMBER") private Integer roomNumber; diff --git a/announcement-processor-api/src/main/java/api/repository/AnnouncementRepository.java b/announcement-processor-api/src/main/java/api/repository/AnnouncementRepository.java index 77e201b..b85bbb2 100644 --- a/announcement-processor-api/src/main/java/api/repository/AnnouncementRepository.java +++ b/announcement-processor-api/src/main/java/api/repository/AnnouncementRepository.java @@ -2,6 +2,8 @@ import api.entity.Announcement; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; +@Repository public interface AnnouncementRepository extends JpaRepository { } diff --git a/announcement-processor-api/src/main/java/api/service/ExtractorDataService.java b/announcement-processor-api/src/main/java/api/service/ExtractorDataService.java index 86607d2..28e26e9 100644 --- a/announcement-processor-api/src/main/java/api/service/ExtractorDataService.java +++ b/announcement-processor-api/src/main/java/api/service/ExtractorDataService.java @@ -1,8 +1,37 @@ package api.service; +import api.dto.AnnouncementDto; +import api.entity.Announcement; +import api.repository.AnnouncementRepository; +import lombok.extern.slf4j.Slf4j; +import org.modelmapper.ModelMapper; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +@Slf4j @Service public class ExtractorDataService { + private final ModelMapper mapper; + private final AnnouncementRepository announcementRepository; + + @Autowired + public ExtractorDataService(ModelMapper mapper, AnnouncementRepository announcementRepository) { + this.mapper = mapper; + this.announcementRepository = announcementRepository; + } + + + public Announcement convertAndSave(AnnouncementDto announcementDto) { + Announcement announcement = announcementDtoToEntity(announcementDto); + announcementRepository.save(announcement); + return announcement; + } + + private Announcement announcementDtoToEntity(AnnouncementDto announcementDto) { + log.info("Mapping announcementDto to announcement..."); + Announcement announcement = mapper.map(announcementDto, Announcement.class); + log.info("Mapped announcementDto to announcement. AnnouncementTitle: {}", announcement.getTitle()); + return announcement; + } } diff --git a/announcement-processor-extractor/src/main/java/extractor/consumer/ReceivementScheduler.java b/announcement-processor-extractor/src/main/java/extractor/consumer/ReceivementScheduler.java index c656b99..9f2d27f 100644 --- a/announcement-processor-extractor/src/main/java/extractor/consumer/ReceivementScheduler.java +++ b/announcement-processor-extractor/src/main/java/extractor/consumer/ReceivementScheduler.java @@ -28,7 +28,7 @@ public ReceivementScheduler(AnnouncementConsumerJms consumer, @Scheduled(cron = "*/10 * * * * *") private void scheduleTask() { - + // TODO fix NullPointer when queue empty log.info("Executing ReceivementScheduler scheduledTask"); AnnouncementDto announcementDto = consumer.consumeAnnouncement(); log.debug("Received announcementDto: {}", announcementDto); From 5b35cb4c3254435a0a89499a947e2e2036417a00 Mon Sep 17 00:00:00 2001 From: michalsroka Date: Mon, 7 Jan 2019 19:39:38 +0100 Subject: [PATCH 127/135] Change DB settings --- .../main/java/api/entity/AdditionalCosts.java | 2 ++ .../main/java/api/entity/Announcement.java | 19 +++++++++++++------ .../src/main/java/api/entity/Lessor.java | 2 ++ .../src/main/java/api/entity/Location.java | 2 ++ .../src/main/java/api/entity/PriceOffer.java | 2 ++ .../main/java/api/entity/PropertyData.java | 2 ++ .../main/resources/application-dev.properties | 2 +- .../main/resources/application-dev.properties | 2 +- 8 files changed, 25 insertions(+), 8 deletions(-) diff --git a/announcement-processor-api/src/main/java/api/entity/AdditionalCosts.java b/announcement-processor-api/src/main/java/api/entity/AdditionalCosts.java index 5a465f5..d49f506 100644 --- a/announcement-processor-api/src/main/java/api/entity/AdditionalCosts.java +++ b/announcement-processor-api/src/main/java/api/entity/AdditionalCosts.java @@ -1,5 +1,6 @@ package api.entity; +import com.fasterxml.jackson.annotation.JsonManagedReference; import lombok.Data; import javax.persistence.*; @@ -14,6 +15,7 @@ public class AdditionalCosts { @Column(name = "ID") private Integer id; + @JsonManagedReference @ManyToOne @JoinColumn(name = "ANNOUNCEMENT_ID") private Announcement announcement; diff --git a/announcement-processor-api/src/main/java/api/entity/Announcement.java b/announcement-processor-api/src/main/java/api/entity/Announcement.java index e3ab234..e0d2e5b 100644 --- a/announcement-processor-api/src/main/java/api/entity/Announcement.java +++ b/announcement-processor-api/src/main/java/api/entity/Announcement.java @@ -1,5 +1,7 @@ package api.entity; +import com.fasterxml.jackson.annotation.JsonBackReference; +import com.fasterxml.jackson.annotation.JsonManagedReference; import lombok.Data; import javax.persistence.*; @@ -28,28 +30,33 @@ public class Announcement { @Column(name = "URL") private String url; - @Column(name = "DESCRIPTION") + @Column(name = "DESCRIPTION", length = 15000) private String description; @Column(name = "CURRENCY") private String currency; - @OneToOne + @JsonManagedReference + @OneToOne(cascade = CascadeType.ALL) @JoinColumn(name = "PRICE_ID") private PriceOffer priceOffer; - @OneToOne + @JsonManagedReference + @OneToOne(cascade = CascadeType.ALL) @JoinColumn(name = "LOCATION_ID") private Location location; - @OneToOne + @JsonManagedReference + @OneToOne(cascade = CascadeType.ALL) @JoinColumn(name = "PROPERTY_DATA_ID") private PropertyData propertyData; - @OneToOne + @JsonManagedReference + @OneToOne(cascade = CascadeType.ALL) @JoinColumn(name = "LESSOR_ID") private Lessor lessor; - @OneToMany(mappedBy = "announcement") + @JsonBackReference + @OneToMany(mappedBy = "announcement", cascade = CascadeType.ALL) private List additionalCosts; } diff --git a/announcement-processor-api/src/main/java/api/entity/Lessor.java b/announcement-processor-api/src/main/java/api/entity/Lessor.java index 81be3c9..4fab049 100644 --- a/announcement-processor-api/src/main/java/api/entity/Lessor.java +++ b/announcement-processor-api/src/main/java/api/entity/Lessor.java @@ -1,5 +1,6 @@ package api.entity; +import com.fasterxml.jackson.annotation.JsonBackReference; import lombok.Data; import javax.persistence.*; @@ -25,6 +26,7 @@ public class Lessor { @Column(name = "EMAIL") private String email; + @JsonBackReference @OneToOne(mappedBy = "lessor") private Announcement announcement; } diff --git a/announcement-processor-api/src/main/java/api/entity/Location.java b/announcement-processor-api/src/main/java/api/entity/Location.java index eefd20c..1912280 100644 --- a/announcement-processor-api/src/main/java/api/entity/Location.java +++ b/announcement-processor-api/src/main/java/api/entity/Location.java @@ -1,5 +1,6 @@ package api.entity; +import com.fasterxml.jackson.annotation.JsonBackReference; import lombok.Data; import javax.persistence.*; @@ -34,6 +35,7 @@ public class Location { @Column(name = "DISTRICT") private String district; + @JsonBackReference @OneToOne(mappedBy = "location") private Announcement announcement; } diff --git a/announcement-processor-api/src/main/java/api/entity/PriceOffer.java b/announcement-processor-api/src/main/java/api/entity/PriceOffer.java index 148a519..8c1e3f7 100644 --- a/announcement-processor-api/src/main/java/api/entity/PriceOffer.java +++ b/announcement-processor-api/src/main/java/api/entity/PriceOffer.java @@ -1,5 +1,6 @@ package api.entity; +import com.fasterxml.jackson.annotation.JsonBackReference; import lombok.Data; import javax.persistence.*; @@ -18,6 +19,7 @@ public class PriceOffer { @Column(name = "DESCRIPTION") private String description; + @JsonBackReference @OneToOne(mappedBy = "priceOffer") private Announcement announcement; diff --git a/announcement-processor-api/src/main/java/api/entity/PropertyData.java b/announcement-processor-api/src/main/java/api/entity/PropertyData.java index 6e9d4d6..c215ba2 100644 --- a/announcement-processor-api/src/main/java/api/entity/PropertyData.java +++ b/announcement-processor-api/src/main/java/api/entity/PropertyData.java @@ -1,5 +1,6 @@ package api.entity; +import com.fasterxml.jackson.annotation.JsonBackReference; import lombok.Data; import javax.persistence.*; @@ -40,6 +41,7 @@ public class PropertyData { @Column(name = "FURNISHING") private String furnishing; + @JsonBackReference @OneToOne(mappedBy = "propertyData") private Announcement announcement; } diff --git a/announcement-processor-extractor/src/main/resources/application-dev.properties b/announcement-processor-extractor/src/main/resources/application-dev.properties index debb5eb..521fa11 100644 --- a/announcement-processor-extractor/src/main/resources/application-dev.properties +++ b/announcement-processor-extractor/src/main/resources/application-dev.properties @@ -1,4 +1,4 @@ -brokerURL=tcp://192.168.99.100:32768 +brokerURL=tcp://192.168.99.100:32780 queue=announcements net.api.url="http://localhost:8080/announcements/send" \ No newline at end of file diff --git a/announcement-processor-parser/src/main/resources/application-dev.properties b/announcement-processor-parser/src/main/resources/application-dev.properties index 082b17f..2dc752d 100644 --- a/announcement-processor-parser/src/main/resources/application-dev.properties +++ b/announcement-processor-parser/src/main/resources/application-dev.properties @@ -11,5 +11,5 @@ spring.datasource.driver-class-name=org.h2.Driver spring.jpa.hibernate.ddl-auto=create-drop spring.h2.console.settings.web-allow-others=true -brokerURL=tcp://192.168.99.100:32768 +brokerURL=tcp://192.168.99.100:32780 queue=announcements \ No newline at end of file From 544e096d6a5c329f691b5bbef53ea51d80a6bfbd Mon Sep 17 00:00:00 2001 From: michalsroka Date: Mon, 7 Jan 2019 19:40:58 +0100 Subject: [PATCH 128/135] Change announcement dto-entity PriceOffer conversion --- .../api/converter/AnnouncementDtoEntityConverter.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/announcement-processor-api/src/main/java/api/converter/AnnouncementDtoEntityConverter.java b/announcement-processor-api/src/main/java/api/converter/AnnouncementDtoEntityConverter.java index 2367bc1..81739f4 100644 --- a/announcement-processor-api/src/main/java/api/converter/AnnouncementDtoEntityConverter.java +++ b/announcement-processor-api/src/main/java/api/converter/AnnouncementDtoEntityConverter.java @@ -2,6 +2,7 @@ import api.dto.AnnouncementDto; import api.entity.*; +import api.entity.price.ConsumerPrice; import org.modelmapper.Converter; import org.modelmapper.spi.MappingContext; @@ -25,11 +26,9 @@ public Announcement convert(MappingContext mappin announcement.setDescription(announcementDto.getDescription()); announcement.setCurrency(announcementDto.getPrice().getCurrency()); - PriceOffer priceOffer = new PriceOffer(); - // TODO check what the description is about - priceOffer.setAnnouncement(announcement); - priceOffer.setDescription(announcement.getDescription()); - announcement.setPriceOffer(priceOffer); + // TODO: check which case of price it is (from package price) and inject accordingly + ConsumerPrice consumerPrice = new ConsumerPrice("name", announcementDto.getPrice().getBasePrice()); + announcement.setPriceOffer(consumerPrice); Location location = new Location(); location.setAnnouncement(announcement); From 928094852713fef3e28beb8cbd181a91db333d48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Mon, 7 Jan 2019 22:21:06 +0100 Subject: [PATCH 129/135] Added loggers to controller --- .../java/api/controller/AnnouncementsController.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/announcement-processor-api/src/main/java/api/controller/AnnouncementsController.java b/announcement-processor-api/src/main/java/api/controller/AnnouncementsController.java index 020bc95..2f2a6b3 100644 --- a/announcement-processor-api/src/main/java/api/controller/AnnouncementsController.java +++ b/announcement-processor-api/src/main/java/api/controller/AnnouncementsController.java @@ -9,9 +9,6 @@ import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.*; import java.util.List; @@ -38,30 +35,39 @@ public ResponseEntity receiveAnnouncement(@RequestBody Announce @GetMapping("/get/all") public ResponseEntity> getAllAnnouncements(@RequestParam(required = false) String currency) { + log.info("ENDPOINT /announcements/get/all was called"); List allAnnouncements = frontEndDataService.getAllAnnouncements(currency); if (allAnnouncements != null) { + log.info("ENDPOINT /announcements/get/all - STATUS OK"); return new ResponseEntity<>(allAnnouncements, HttpStatus.OK); } else { + log.info("ENDPOINT /announcements/get/all - STATUS NO CONTENT"); return new ResponseEntity<>(HttpStatus.NO_CONTENT); } } @GetMapping("get/{id}/detailed-info") public ResponseEntity getDetailedInfoAnnouncement(@PathVariable Integer id, @RequestParam(required = false) String currency) { + log.info("ENDPOINT /announcements/get/" + id + "/detailed-info was called"); DetailedAnnouncementInfo detailedInfoAnnouncement = frontEndDataService.getDetailedInfoAnnouncement(id, currency); if (detailedInfoAnnouncement != null) { + log.info("ENDPOINT /announcements/get/" + id + "/detailed-info - STATUS OK"); return new ResponseEntity<>(detailedInfoAnnouncement, HttpStatus.OK); } else { + log.info("ENDPOINT /announcements/get/" + id + "/detailed-info - STATUS NO CONTENT"); return new ResponseEntity<>(HttpStatus.NO_CONTENT); } } @GetMapping("get/{id}/general-info") public ResponseEntity getGeneralInfoAnnouncement(@PathVariable Integer id, @RequestParam(required = false) String currency) { + log.info("ENDPOINT /announcements/get/" + id + "/general-info was called"); GeneralAnnouncementInfo generalInfoAnnouncement = frontEndDataService.getGeneralInfoAnnouncement(id, currency); if (generalInfoAnnouncement != null) { + log.info("ENDPOINT /announcements/get/" + id + "/general-info - STATUS OK"); return new ResponseEntity<>(generalInfoAnnouncement, HttpStatus.OK); } else { + log.info("ENDPOINT /announcements/get/" + id + "/general-info - STATUS NO CONTENT"); return new ResponseEntity<>(HttpStatus.NO_CONTENT); } } From 22477fbe1d9d1de70600ef37f4dd7c7c135f92a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Mon, 7 Jan 2019 22:21:34 +0100 Subject: [PATCH 130/135] Fix for dev properties --- .../src/main/resources/application-dev.properties | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/announcement-processor-extractor/src/main/resources/application-dev.properties b/announcement-processor-extractor/src/main/resources/application-dev.properties index 521fa11..fbbd6e5 100644 --- a/announcement-processor-extractor/src/main/resources/application-dev.properties +++ b/announcement-processor-extractor/src/main/resources/application-dev.properties @@ -1,4 +1,3 @@ brokerURL=tcp://192.168.99.100:32780 queue=announcements - -net.api.url="http://localhost:8080/announcements/send" \ No newline at end of file +net.api.url=http://localhost:8080/announcements/send \ No newline at end of file From 06530f74ad176e924c8ec3d554ab1e55c7e216dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Mon, 7 Jan 2019 23:20:58 +0100 Subject: [PATCH 131/135] Added rounding for Money pattern --- announcement-processor-api/src/main/java/api/model/Money.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/announcement-processor-api/src/main/java/api/model/Money.java b/announcement-processor-api/src/main/java/api/model/Money.java index 0115ff4..7b5871d 100644 --- a/announcement-processor-api/src/main/java/api/model/Money.java +++ b/announcement-processor-api/src/main/java/api/model/Money.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import java.math.BigDecimal; +import java.math.RoundingMode; public class Money { @JsonIgnore @@ -26,6 +27,6 @@ public void recalculateToCurrency(String desiredCurrency) { } public String getPriceWithCurrency() { - return price + " " + actualCurrency; + return price.setScale(2, RoundingMode.CEILING) + " " + actualCurrency; } } From d8d8acfbb4c14b45bb944e586d40760cbdc501a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Mon, 7 Jan 2019 23:28:59 +0100 Subject: [PATCH 132/135] Fixed location display --- .../api/model/DetailedAnnouncementInfo.java | 3 ++- .../java/api/service/FrontEndDataService.java | 19 +++++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/announcement-processor-api/src/main/java/api/model/DetailedAnnouncementInfo.java b/announcement-processor-api/src/main/java/api/model/DetailedAnnouncementInfo.java index d27786e..7006c73 100644 --- a/announcement-processor-api/src/main/java/api/model/DetailedAnnouncementInfo.java +++ b/announcement-processor-api/src/main/java/api/model/DetailedAnnouncementInfo.java @@ -2,6 +2,7 @@ import api.entity.AdditionalCosts; import api.entity.Lessor; +import api.entity.Location; import api.entity.PropertyData; import lombok.Data; @@ -14,10 +15,10 @@ public class DetailedAnnouncementInfo { private String provider; private String creationDate; private String url; - private String location; private String description; private PropertyData propertyData; private Lessor lessor; private List additionalCosts; + private Location location; } diff --git a/announcement-processor-api/src/main/java/api/service/FrontEndDataService.java b/announcement-processor-api/src/main/java/api/service/FrontEndDataService.java index a3a0f62..ef3c456 100644 --- a/announcement-processor-api/src/main/java/api/service/FrontEndDataService.java +++ b/announcement-processor-api/src/main/java/api/service/FrontEndDataService.java @@ -15,8 +15,10 @@ import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; +import java.util.stream.Stream; @Service public class FrontEndDataService { @@ -74,7 +76,7 @@ private DetailedAnnouncementInfo convertAnnouncementToDetailedFormat(Announcemen detailedAnnouncementInfo.setProvider(announcement.getProvider()); detailedAnnouncementInfo.setCreationDate(getFormattedLocalDateTime(announcement.getCreationDate())); detailedAnnouncementInfo.setUrl(announcement.getUrl()); - detailedAnnouncementInfo.setLocation(getFullLocation(announcement.getLocation())); + detailedAnnouncementInfo.setLocation(announcement.getLocation()); detailedAnnouncementInfo.setDescription(announcement.getDescription()); detailedAnnouncementInfo.setPropertyData(announcement.getPropertyData()); @@ -85,9 +87,18 @@ private DetailedAnnouncementInfo convertAnnouncementToDetailedFormat(Announcemen } private String getFullLocation(Location location) { - return location.getCity() + " " + location.getZipCode() + " " + location.getDistrict() + "\n" - + location.getStreet() + " " + location.getBuildingNumber() + " / " + location.getFlatNumber() + "\n" - + location.getCountry(); + String city = Stream.of(location.getCity(), location.getZipCode(), location.getDistrict()) + .filter(Objects::nonNull) + .collect(Collectors.joining(",")); + String street = Stream.of(location.getStreet(), location.getBuildingNumber(), location.getFlatNumber()) + .filter(Objects::nonNull) + .collect(Collectors.joining(",")); + String country = location.getCountry(); + String out = Stream.of(city, street, country).filter(Objects::nonNull).collect(Collectors.joining("\n")); + if (out.equals("\n")) { + return "Brak"; + } + return out; } private String getFormattedLocalDateTime(LocalDateTime ldt) { From 2b3c00d7ae86334d8042e819c98853c31c1b9330 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20S=C5=82owiak?= Date: Mon, 7 Jan 2019 23:30:10 +0100 Subject: [PATCH 133/135] Fixed json back reference for additionalcosts --- .../src/main/java/api/entity/AdditionalCosts.java | 4 ++-- .../src/main/java/api/entity/Announcement.java | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/announcement-processor-api/src/main/java/api/entity/AdditionalCosts.java b/announcement-processor-api/src/main/java/api/entity/AdditionalCosts.java index d49f506..cdc02be 100644 --- a/announcement-processor-api/src/main/java/api/entity/AdditionalCosts.java +++ b/announcement-processor-api/src/main/java/api/entity/AdditionalCosts.java @@ -1,6 +1,6 @@ package api.entity; -import com.fasterxml.jackson.annotation.JsonManagedReference; +import com.fasterxml.jackson.annotation.JsonBackReference; import lombok.Data; import javax.persistence.*; @@ -15,7 +15,7 @@ public class AdditionalCosts { @Column(name = "ID") private Integer id; - @JsonManagedReference + @JsonBackReference @ManyToOne @JoinColumn(name = "ANNOUNCEMENT_ID") private Announcement announcement; diff --git a/announcement-processor-api/src/main/java/api/entity/Announcement.java b/announcement-processor-api/src/main/java/api/entity/Announcement.java index e0d2e5b..0ee4d97 100644 --- a/announcement-processor-api/src/main/java/api/entity/Announcement.java +++ b/announcement-processor-api/src/main/java/api/entity/Announcement.java @@ -1,6 +1,5 @@ package api.entity; -import com.fasterxml.jackson.annotation.JsonBackReference; import com.fasterxml.jackson.annotation.JsonManagedReference; import lombok.Data; @@ -56,7 +55,7 @@ public class Announcement { @JoinColumn(name = "LESSOR_ID") private Lessor lessor; - @JsonBackReference + @JsonManagedReference @OneToMany(mappedBy = "announcement", cascade = CascadeType.ALL) private List additionalCosts; } From 899791c0be90f8cbc82e28df508d19e500ac4ac6 Mon Sep 17 00:00:00 2001 From: michalsroka Date: Mon, 7 Jan 2019 23:45:13 +0100 Subject: [PATCH 134/135] Add additional cases of price case: exchange, contact --- .../converter/AnnouncementDtoEntityConverter.java | 15 ++++++++++++--- .../consumer/AnnouncementConsumerJms.java | 9 +++++---- .../AnnouncementExtractingServiceTest.java | 2 +- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/announcement-processor-api/src/main/java/api/converter/AnnouncementDtoEntityConverter.java b/announcement-processor-api/src/main/java/api/converter/AnnouncementDtoEntityConverter.java index 81739f4..ec81984 100644 --- a/announcement-processor-api/src/main/java/api/converter/AnnouncementDtoEntityConverter.java +++ b/announcement-processor-api/src/main/java/api/converter/AnnouncementDtoEntityConverter.java @@ -3,6 +3,8 @@ import api.dto.AnnouncementDto; import api.entity.*; import api.entity.price.ConsumerPrice; +import api.entity.price.ContactPrice; +import api.entity.price.ExchangePrice; import org.modelmapper.Converter; import org.modelmapper.spi.MappingContext; @@ -26,9 +28,16 @@ public Announcement convert(MappingContext mappin announcement.setDescription(announcementDto.getDescription()); announcement.setCurrency(announcementDto.getPrice().getCurrency()); - // TODO: check which case of price it is (from package price) and inject accordingly - ConsumerPrice consumerPrice = new ConsumerPrice("name", announcementDto.getPrice().getBasePrice()); - announcement.setPriceOffer(consumerPrice); + if (announcementDto.getPrice().getBasePrice() == null) { + ContactPrice contactPrice = new ContactPrice("contact"); + announcement.setPriceOffer(contactPrice); + } else if (announcementDto.getPrice().getBasePrice().equals(BigDecimal.valueOf(-1))) { + ExchangePrice exchangePrice = new ExchangePrice("exchange"); + announcement.setPriceOffer(exchangePrice); + } else { + ConsumerPrice consumerPrice = new ConsumerPrice("name", announcementDto.getPrice().getBasePrice()); + announcement.setPriceOffer(consumerPrice); + } Location location = new Location(); location.setAnnouncement(announcement); diff --git a/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerJms.java b/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerJms.java index 2d0284a..5f83032 100644 --- a/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerJms.java +++ b/announcement-processor-extractor/src/main/java/extractor/consumer/AnnouncementConsumerJms.java @@ -26,7 +26,7 @@ public class AnnouncementConsumerJms implements AnnouncementConsumer, ExceptionL @Override public AnnouncementDto consumeAnnouncement() { - log.debug("Running consumeAnnouncement, brokerUrl: {}, queueName: {}", brokerUrl, queueName); + log.info("Running consumeAnnouncement, brokerUrl: {}, queueName: {}", brokerUrl, queueName); try { ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory(brokerUrl); @@ -42,19 +42,20 @@ public AnnouncementDto consumeAnnouncement() { MessageConsumer consumer = session.createConsumer(destination); Message message = consumer.receive(100); - log.debug("Received message: {}", message); + log.debug("Received message"); AnnouncementDto announcementDto = null; if (message instanceof TextMessage) { TextMessage textMessage = (TextMessage) message; - log.info("Received text message: {}", textMessage.getText()); + log.info("Received text message"); + log.debug("Text message payload: {}", textMessage.getText()); announcementDto = mapperService.getAnnouncementDtoFromJsonString(textMessage.getText()); } consumer.close(); session.close(); connection.close(); - log.debug("Finished consumeAnnouncement"); + log.info("Finished consumeAnnouncement"); return announcementDto; } catch (Exception e) { diff --git a/announcement-processor-extractor/src/test/java/extractor/service/AnnouncementExtractingServiceTest.java b/announcement-processor-extractor/src/test/java/extractor/service/AnnouncementExtractingServiceTest.java index 9266235..948a529 100644 --- a/announcement-processor-extractor/src/test/java/extractor/service/AnnouncementExtractingServiceTest.java +++ b/announcement-processor-extractor/src/test/java/extractor/service/AnnouncementExtractingServiceTest.java @@ -15,6 +15,6 @@ public class AnnouncementExtractingServiceTest { public void extractFromAnnouncementDtoTest() { AnnouncementDto announcementDto = announcementConsumer.consumeAnnouncement(); Announcement announcement = extractingService.extractFromAnnouncementDto(announcementDto); - System.out.println(announcement); // TODO proper test + System.out.println(announcement); } } From 1995bec28c3e1b221bd6c256d6011fb8d01c66b2 Mon Sep 17 00:00:00 2001 From: michalsroka Date: Tue, 8 Jan 2019 09:02:17 +0100 Subject: [PATCH 135/135] Add parsing price from description --- .../AnnouncementExtractingService.java | 55 ++++++++++++-- .../AnnouncementExtractingServiceTest.java | 76 +++++++++++++++++++ 2 files changed, 126 insertions(+), 5 deletions(-) diff --git a/announcement-processor-extractor/src/main/java/extractor/service/AnnouncementExtractingService.java b/announcement-processor-extractor/src/main/java/extractor/service/AnnouncementExtractingService.java index aea3372..db25c72 100644 --- a/announcement-processor-extractor/src/main/java/extractor/service/AnnouncementExtractingService.java +++ b/announcement-processor-extractor/src/main/java/extractor/service/AnnouncementExtractingService.java @@ -5,8 +5,11 @@ import org.springframework.stereotype.Service; import java.math.BigDecimal; +import java.util.ArrayList; import java.util.HashMap; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; @Service public class AnnouncementExtractingService { @@ -34,23 +37,22 @@ private Lessor extractLessor(AnnouncementDto announcementDto) { lessor.setName(announcementDto.getLessorName()); lessor.setLessorType(announcementDto.getLessor()); lessor.setPhoneNumber(announcementDto.getPhoneNumber()); - // TODO more advanced extraction e.g. description data extraction return lessor; } private Location extractLocation(AnnouncementDto announcementDto) { Location location = new Location(); - // TODO more advanced extraction e.g. description data extraction + return location; } private Price extractPrice(AnnouncementDto announcementDto) { Price price = new Price(); price.setBasePrice(announcementDto.getPrice()); - // TODO more advanced extraction e.g. description data extraction Map pricesMap = new HashMap<>(); + parseDescriptionPrice(announcementDto.getDescription(), pricesMap); - pricesMap.put("SomePrice", announcementDto.getAdditionalRentCost()); + price.setAdditionalPrices(pricesMap); return price; } @@ -66,11 +68,54 @@ private PropertyData extractPropertyData(AnnouncementDto announcementDto) { propertyData.setParkingAvailability(announcementDto.getParkingAvailability()); propertyData.setPropertyType(announcementDto.getPropertyType()); propertyData.setRoomNumber(announcementDto.getRoomAmount()); - // TODO more advanced extraction e.g. description data extraction return propertyData; } private void parseDescriptionPrice(String description, Map pricesMap) { + ArrayList utilities = new ArrayList<>(); + utilities.add("gaz"); + utilities.add("prąd"); + utilities.add("czynsz"); + utilities.add("ogrzewanie"); + utilities.add("CO"); + utilities.add("śmieci"); + utilities.add("media"); + utilities.add("internet"); + utilities.add("woda"); + + for (String word : utilities) { + Pattern p = Pattern.compile("\\b" + word); + Matcher m = p.matcher(description); + while (m.find()) { + BigDecimal foundCost = checkNeighbor(description, m.end()); + if (!pricesMap.containsKey(word) || pricesMap.get(word) == null) { + pricesMap.put(word, foundCost); + } + } + } + + } + + private BigDecimal checkNeighbor(String description, int end) { + + int i = end + 1; + int counter = 25; + boolean notFound = true; + String digitBuilder = ""; + while (notFound && i <= description.length() - 1 && counter >= 0) { + char ch = description.charAt(i); + if (Character.isDigit(ch)) { + digitBuilder = digitBuilder + ch; + } else { + counter--; + if (digitBuilder.length() > 0) { + return new BigDecimal(digitBuilder); + } + } + + i++; + } + return null; } } diff --git a/announcement-processor-extractor/src/test/java/extractor/service/AnnouncementExtractingServiceTest.java b/announcement-processor-extractor/src/test/java/extractor/service/AnnouncementExtractingServiceTest.java index 948a529..1b7e5ad 100644 --- a/announcement-processor-extractor/src/test/java/extractor/service/AnnouncementExtractingServiceTest.java +++ b/announcement-processor-extractor/src/test/java/extractor/service/AnnouncementExtractingServiceTest.java @@ -6,6 +6,13 @@ import extractor.entity.Announcement; import org.junit.Test; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + public class AnnouncementExtractingServiceTest { private final AnnouncementConsumer announcementConsumer = new AnnouncementConsumerFromString(); @@ -17,4 +24,73 @@ public void extractFromAnnouncementDtoTest() { Announcement announcement = extractingService.extractFromAnnouncementDto(announcementDto); System.out.println(announcement); } + + @Test + public void parsingTest() { + + Map pricesMap = new HashMap<>(); + String description = "Proponuję mieszkanie na wynajem (kawalerka) o powierzchni 40m2 " + + "zlokalizowane przy ulicy Głowackiego 4 w Krakowie. Dzielnica Krowodrza." + + " Rejon Bronowice.\n\nKawalerka mieście się na 5 piętrze od strony południowej z" + + " widokiem na panoramę Krakowa.\n3 minuty pieszo do przystanków tramwajowych." + + " Bliskie sąsiedztwo Uniwersytetu Pedagogicznego, AGH, Wydziału Fizyki i Architektury " + + "PK.\n\nBudynek oddany do użytkowania w 2008r. Podłoga z litego drewna, w aneksie kuchennym i" + + " w łazience płytki ceramiczne. Mieszkanie wyposażone tak jak na zdjęciach: (lodówka, płyta grzewcza," + + " piekarnik, pralka, stół, krzesła, łóżko 2os., łóżko 1os.)\nMożliwość usunięcia mebli wg upodobań" + + " wynajmującego jak i ich dokupienia.\n\nNieruchomość posiada przestronny hol i windę. Na parterze " + + "zlokalizowane są sklepy m. in. Lewiatan - można w pantoflach robić zakupy ;)\n\nKoszty najmu " + + "(miesięczne): wynajem 1500zł, + czynsz administracyjny 250zł, + media wg zużycia " + + "(woda (~60zł/1os), prąd (~60zł/1os) ," + + " ogrzewanie, internet (60zł)).\n\nSZACOWANY CAŁKOWITY KOSZT wynajmu i eksploatacji miesięcznej:" + + " 1900-2000zł (2 osoby)\n\n2K PLN per month ;)\nZalety to niskie koszty ogrzewania, w zasadzie " + + "pomijalne.\n\nINTERNET od UPC. Mieszkanie wyposażone w gniazdo RTV i SAT.\n\nUmowa min. na 9 miesięcy. " + + "W razie pytań proszę się nie krępować :)\n\nKontak mailowy lub telefoniczny (w rozsądnych godzinach):" + + "\n\n6 0 2 3 2 7 7 0 6\n\nnioobi (malpa) op (kropka) pl\n\n\nMożliwość oglądnięcia mieszkania\n\n" + + "Palić papierosy można na dużym balkonie ;)\n\n\nPośrednikom z góry dziękuję za zainteresowanie!"; + + ArrayList utilities = new ArrayList<>(); + utilities.add("gaz"); + utilities.add("prąd"); + utilities.add("czynsz"); + utilities.add("ogrzewanie"); + utilities.add("CO"); + utilities.add("śmieci"); + utilities.add("media"); + utilities.add("internet"); + utilities.add("woda"); + + for (String word : utilities) { + Pattern p = Pattern.compile("\\b" + word); + Matcher m = p.matcher(description); + while (m.find()) { + BigDecimal foundCost = checkNeighbor(description, m.end()); + if (!pricesMap.containsKey(word) || pricesMap.get(word) == null) { + pricesMap.put(word, foundCost); + } + } + } + System.out.println(pricesMap); + } + + private BigDecimal checkNeighbor(String description, int end) { + + int i = end + 1; + int counter = 25; + boolean notFound = true; + String digitBuilder = ""; + while (notFound && i <= description.length() - 1 && counter >= 0) { + char ch = description.charAt(i); + if (Character.isDigit(ch)) { + digitBuilder = digitBuilder + ch; + } else { + counter--; + if (digitBuilder.length() > 0) { + return new BigDecimal(digitBuilder); + } + } + + i++; + } + return null; + } }