algo.patch

Index: src/main/java/com/handson/searchengine/crawler/Crawler.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/src/main/java/com/handson/searchengine/crawler/Crawler.java b/src/main/java/com/handson/searchengine/crawler/Crawler.java
new file mode 100644
--- /dev/null	(date 1644411805696)
+++ b/src/main/java/com/handson/searchengine/crawler/Crawler.java	(date 1644411805696)
@@ -0,0 +1,83 @@
+package com.handson.searchengine.crawler;
+
+import com.handson.searchengine.model.CrawlStatus;
+import com.handson.searchengine.model.CrawlerRecord;
+import com.handson.searchengine.model.CrawlerRequest;
+import com.handson.searchengine.model.StopReason;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+@Service
+public class Crawler {
+
+    protected final Log logger = LogFactory.getLog(getClass());
+
+    public static final int MAX_CAPACITY = 100000;
+    private Set<String> visitedUrls = new HashSet<>();
+    private BlockingQueue<CrawlerRecord> queue = new ArrayBlockingQueue<CrawlerRecord>(MAX_CAPACITY);
+    private int curDistance = 0;
+    private long startTime = 0;
+    private StopReason stopReason;
+    public CrawlStatus crawl(String crawlId, CrawlerRequest crawlerRequest) throws InterruptedException, IOException {
+        visitedUrls.clear();
+        queue.clear();
+        curDistance = 0;
+        startTime = System.currentTimeMillis();
+        stopReason = null;
+        queue.put(CrawlerRecord.of(crawlId, crawlerRequest));
+        while (!queue.isEmpty() && getStopReason(queue.peek()) == null) {
+            CrawlerRecord rec = queue.poll();
+            logger.info("crawling url:" + rec.getUrl());
+            Document webPageContent = Jsoup.connect(rec.getUrl()).get();
+            List<String> innerUrls = extractWebPageUrls(rec.getBaseUrl(), webPageContent);
+            addUrlsToQueue(rec, innerUrls, rec.getDistance() +1);
+        }
+        stopReason = queue.isEmpty() ? null : getStopReason(queue.peek());
+        return CrawlStatus.of(curDistance, startTime, visitedUrls.size(), stopReason);
+
+    }
+
+    private StopReason getStopReason(CrawlerRecord rec) {
+        if (rec.getDistance() == rec.getMaxDistance() +1) return StopReason.maxDistance;
+        if (visitedUrls.size() >= rec.getMaxUrls()) return StopReason.maxUrls;
+        if (System.currentTimeMillis() >= rec.getMaxTime()) return StopReason.timeout;
+        return null;
+    }
+
+
+    private void addUrlsToQueue(CrawlerRecord rec, List<String> urls, int distance) throws InterruptedException {
+        logger.info(">> adding urls to queue: distance->" + distance + " amount->" + urls.size());
+        curDistance = distance;
+        for (String url : urls) {
+            if (!visitedUrls.contains(url)) {
+                visitedUrls.add(url);
+                queue.put(CrawlerRecord.of(rec).withUrl(url).withIncDistance()) ;
+            }
+        }
+    }
+
+    private List<String> extractWebPageUrls(String baseUrl, Document webPageContent) {
+        List<String> links = webPageContent.select("a[href]")
+                .eachAttr("abs:href")
+                .stream()
+                .filter(url -> url.startsWith(baseUrl))
+                .collect(Collectors.toList());
+        logger.info(">> extracted->" + links.size() + " links");
+
+        return links;
+    }
+
+
+}
Index: src/main/java/com/handson/searchengine/model/CrawlStatus.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/src/main/java/com/handson/searchengine/model/CrawlStatus.java b/src/main/java/com/handson/searchengine/model/CrawlStatus.java
new file mode 100644
--- /dev/null	(date 1644355865826)
+++ b/src/main/java/com/handson/searchengine/model/CrawlStatus.java	(date 1644355865826)
@@ -0,0 +1,43 @@
+package com.handson.searchengine.model;
+
+public class CrawlStatus {
+    int distance;
+    long startTime;
+    StopReason stopReason;
+    long lastModified;
+    long numPages = 0;
+
+    public static CrawlStatus of(int distance, long startTime, int numPages, StopReason stopReason) {
+        CrawlStatus res = new CrawlStatus();
+        res.distance = distance;
+        res.startTime =  startTime;
+        res.lastModified = System.currentTimeMillis();
+        res.stopReason = stopReason;
+        res.numPages = numPages;
+        return res;
+    }
+
+    public int getDistance() {
+        return distance;
+    }
+
+    public long getLastModified() {
+        return lastModified;
+    }
+
+    public long getStartTime() {
+        return startTime;
+    }
+
+    public StopReason getStopReason() {
+        return stopReason;
+    }
+
+    public long getNumPages() {
+        return numPages;
+    }
+
+    public void setNumPages(long numPages) {
+        this.numPages = numPages;
+    }
+}
Index: src/main/java/com/handson/searchengine/model/CrawlStatusOut.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/src/main/java/com/handson/searchengine/model/CrawlStatusOut.java b/src/main/java/com/handson/searchengine/model/CrawlStatusOut.java
new file mode 100644
--- /dev/null	(date 1644354358224)
+++ b/src/main/java/com/handson/searchengine/model/CrawlStatusOut.java	(date 1644354358224)
@@ -0,0 +1,64 @@
+package com.handson.searchengine.model;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.handson.searchengine.util.Dates;
+import org.joda.time.LocalDateTime;
+
+import java.util.Date;
+
+public class CrawlStatusOut {
+    int distance;
+    long startTime;
+
+    @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd HH:mm:ss")
+    @JsonProperty("startTime")
+    public LocalDateTime calcStartTime() {
+        return Dates.atLocalTime(new Date(startTime));
+    }
+
+    StopReason stopReason;
+    long lastModified;
+    @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd HH:mm:ss")
+    @JsonProperty("lastModified")
+    public LocalDateTime calcLastModified() {
+        return Dates.atLocalTime(new Date(lastModified));
+    }
+
+    long numPages = 0;
+
+    public static CrawlStatusOut of(CrawlStatus in) {
+        CrawlStatusOut res = new CrawlStatusOut();
+        res.distance = in.distance;
+        res.startTime =  in.startTime;
+        res.lastModified = in.lastModified;
+        res.stopReason = in.stopReason;
+        res.numPages = in.numPages;
+        return res;
+    }
+
+    public int getDistance() {
+        return distance;
+    }
+
+    public long getLastModified() {
+        return lastModified;
+    }
+
+    public long getStartTime() {
+        return startTime;
+    }
+
+    public StopReason getStopReason() {
+        return stopReason;
+    }
+
+    public long getNumPages() {
+        return numPages;
+    }
+
+    public void setNumPages(long numPages) {
+        this.numPages = numPages;
+    }
+}
+
Index: src/main/java/com/handson/searchengine/model/StopReason.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/src/main/java/com/handson/searchengine/model/StopReason.java b/src/main/java/com/handson/searchengine/model/StopReason.java
new file mode 100644
--- /dev/null	(date 1644354009505)
+++ b/src/main/java/com/handson/searchengine/model/StopReason.java	(date 1644354009505)
@@ -0,0 +1,5 @@
+package com.handson.searchengine.model;
+
+public enum StopReason {
+    maxUrls, maxDistance, timeout
+}
Index: src/main/java/com/handson/searchengine/model/CrawlerRecord.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/src/main/java/com/handson/searchengine/model/CrawlerRecord.java b/src/main/java/com/handson/searchengine/model/CrawlerRecord.java
new file mode 100644
--- /dev/null	(date 1644357870095)
+++ b/src/main/java/com/handson/searchengine/model/CrawlerRecord.java	(date 1644357870095)
@@ -0,0 +1,81 @@
+package com.handson.searchengine.model;
+
+public class CrawlerRecord {
+    String crawlId;
+    String baseUrl;
+    String url;
+    int distance;
+    int maxDistance;
+    long startTime;
+    long maxTime;
+    int maxUrls;
+
+    public static CrawlerRecord of(String crawlId, CrawlerRequest r) {
+        long startTime = System.currentTimeMillis();
+        CrawlerRecord res = new CrawlerRecord();
+        res.crawlId = crawlId;
+        res.baseUrl = r.url;
+        res.url = r.getUrl();
+        res.distance = 0;
+        res.startTime = startTime;
+        res.maxTime = startTime + 1000L * r.maxSeconds;
+        res.maxDistance = r.maxDistance;
+        res.maxUrls = r.maxUrls;
+        return res;
+    }
+
+    public static CrawlerRecord of(CrawlerRecord r) {
+        CrawlerRecord res = new CrawlerRecord();
+        res.crawlId = r.crawlId;
+        res.baseUrl = r.baseUrl;
+        res.url = r.url;
+        res.distance = r.distance;
+        res.maxTime = r.maxTime;
+        res.startTime = r.startTime;
+        res.maxDistance = r.maxDistance;
+        res.maxUrls = r.maxUrls;
+        return res;
+    }
+    public CrawlerRecord withUrl(String url) {
+        this.url = url;
+        return this;
+    }
+
+
+    public CrawlerRecord withIncDistance() {
+        distance += 1;
+        return this;
+    }
+
+    public String getCrawlId() {
+        return crawlId;
+    }
+
+    public long getStartTime() {
+        return startTime;
+    }
+
+    public String getBaseUrl() {
+        return baseUrl;
+    }
+
+    public String getUrl() {
+        return url;
+    }
+
+    public int getDistance() {
+        return distance;
+    }
+
+    public int getMaxDistance() {
+        return maxDistance;
+    }
+
+    public long getMaxTime() {
+        return maxTime;
+    }
+
+    public int getMaxUrls() {
+        return maxUrls;
+    }
+}
Index: src/main/java/com/handson/searchengine/model/CrawlerRequest.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/src/main/java/com/handson/searchengine/model/CrawlerRequest.java b/src/main/java/com/handson/searchengine/model/CrawlerRequest.java
new file mode 100644
--- /dev/null	(date 1644355865805)
+++ b/src/main/java/com/handson/searchengine/model/CrawlerRequest.java	(date 1644355865805)
@@ -0,0 +1,30 @@
+package com.handson.searchengine.model;
+
+
+public class CrawlerRequest {
+    String url;
+    Integer maxDistance;
+    Integer maxSeconds;
+    Integer maxUrls;
+
+    public void setUrl(String url) {
+        this.url = url;
+    }
+
+    public String getUrl() {
+        return url;
+    }
+
+    public Integer getMaxUrls() {
+        return maxUrls;
+    }
+
+    public Integer getMaxDistance() {
+        return maxDistance;
+    }
+
+    public Integer getMaxSeconds() {
+        return maxSeconds;
+    }
+}
+
Index: src/main/java/com/handson/searchengine/util/Dates.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/src/main/java/com/handson/searchengine/util/Dates.java b/src/main/java/com/handson/searchengine/util/Dates.java
new file mode 100644
--- /dev/null	(date 1644356035799)
+++ b/src/main/java/com/handson/searchengine/util/Dates.java	(date 1644356035799)
@@ -0,0 +1,87 @@
+package com.handson.searchengine.util;
+
+import org.joda.time.*;
+import org.springframework.lang.Nullable;
+
+import java.text.SimpleDateFormat;
+import java.time.OffsetDateTime;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.Objects;
+import java.util.TimeZone;
+
+public class Dates {
+    public static SimpleDateFormat shortDate = new SimpleDateFormat("YYYY-MM-dd");
+    public static TimeZone TIME_ZONE = TimeZone.getTimeZone("Asia/Jerusalem");
+    public static SimpleDateFormat monthFormatter = new SimpleDateFormat("yyyy/MM");
+
+    public Dates() {
+    }
+
+    public static String getCurMonth() {
+        Date date = new Date();
+        return monthFormatter.format(date);
+    }
+
+    public static String dateToStr(@Nullable LocalDate date) {
+        return date == null ? null : shortDate.format(date);
+    }
+
+    public static Date atUtc(LocalDateTime date) {
+        return atUtc(date, TIME_ZONE);
+    }
+
+    public static Date atUtc(LocalDateTime date, TimeZone zone) {
+        if (date == null) return null;
+        Calendar calendar = Calendar.getInstance();
+        calendar.setFirstDayOfWeek(Calendar.SUNDAY);
+        calendar.setTimeZone(zone);
+        calendar.set(date.getYear(), date.getMonthOfYear()-1, date.getDayOfMonth());//convert from locatDateTime to Calender time
+        calendar.set(Calendar.HOUR_OF_DAY, date.getHourOfDay());
+        calendar.set(Calendar.MINUTE, date.getMinuteOfHour());
+        calendar.set(Calendar.SECOND, date.getSecondOfMinute());
+        calendar.set(Calendar.MILLISECOND, 0);
+        return calendar.getTime();
+    }
+
+    public static Date atUtc(@Nullable LocalDate date) {
+        return atUtc(date, TIME_ZONE);
+    }
+
+    public static Date atUtc(@Nullable LocalDate date, TimeZone zone) {
+        return date == null ? null : atUtc(date.toLocalDateTime(LocalTime.MIDNIGHT), zone);
+    }
+
+    public static LocalDateTime atLocalTime(Date date) {
+        return atLocalTime(date, TIME_ZONE);
+    }
+
+    public static LocalDateTime atLocalTime(Date date, TimeZone zone) {
+        if (date == null) return null;
+        var localDate = OffsetDateTime.ofInstant(date.toInstant(), zone.toZoneId()).toLocalDateTime();
+        Calendar c = Calendar.getInstance();
+        c.set(localDate.getYear(), localDate.getMonthValue() - 1, localDate.getDayOfMonth());
+        c.set(Calendar.HOUR_OF_DAY, localDate.getHour());
+        c.set(Calendar.MINUTE, localDate.getMinute());
+        c.set(Calendar.SECOND, localDate.getSecond());
+        c.set(Calendar.MILLISECOND, 0);
+        LocalDateTime res = LocalDateTime.fromCalendarFields(c);
+        return res;
+    }
+
+    public static Date nowUTC() {
+        return DateTime.now().withZone(DateTimeZone.UTC).toDate();
+    }
+
+    public static String getFullDateTime() {
+        return DateTime.now().withZone(DateTimeZone.UTC).toDateTimeISO().toString();
+    }
+
+    public static boolean equals(@Nullable Date date1, @Nullable Date date2) {
+        if (date1 != null && date2 != null) {
+            return date1.getTime() == date2.getTime();
+        } else {
+            return Objects.equals(date1, date2);
+        }
+    }
+}