reverse;
-
- protected abstract B forward(A a);
- protected abstract A backward(B b);
-
- B doForward(A a) {
- if (a != null) return forward(a);
- else return null;
- }
-
- A doBackward(B b) {
- if (b != null) return backward(b);
- else return null;
- }
-
- /**
- * Method used for performing conversions.
- *
- * @return The input value, converted from type {@code A} to {@code B}, or {@code null} if and only if the input is also {@code null}.
- */
- public final B convert(A a) {
- return doForward(a);
- }
-
- /**
- * Returns an {@code Iterable} that applies {@code convert} to each element of {@code fromIterable}. The
- * conversion is done lazily.
- *
- * The returned iterable's iterator supports {@code remove()} if the input iterator does. After
- * a successful {@code remove()} call, {@code fromIterable} no longer contains the corresponding
- * element.
- *
- * @param aIterable The {@code Iterable} whose instances we wish to convert.
- */
- public Iterable convertAll(Iterable aIterable) {
- Objects.requireNonNull(aIterable);
- return () -> new Iterator<>()
- {
- private final Iterator aIterator = aIterable.iterator();
-
- @Override
- public boolean hasNext() {
- return aIterator.hasNext();
- }
-
- @Override
- public B next() {
- return convert(aIterator.next());
- }
-
- @Override
- public void remove() {
- aIterator.remove();
- }
- };
- }
-
- /**
- * Method used for obtaining the reversed view of this converter.
- *
- * @return A {@code Converter} from {@code B} to {@code A}.
- */
- public Converter reverse() {
- if (this.reverse == null) {
- reverse = new ReverseConverter<>(this);
- }
-
- return reverse;
- }
-
- private static final class ReverseConverter extends Converter {
- final Converter original;
-
- ReverseConverter(Converter original) {
- this.original = original;
- }
-
- @Override
- protected A forward(B b) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- protected B backward(A a) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- A doForward(B b) {
- return original.doBackward(b);
- }
-
- @Override
- B doBackward(A a) {
- return original.doForward(a);
- }
-
- @Override
- public Converter reverse() {
- return original;
- }
- }
-}
diff --git a/dl4se-crawler/src/main/java/usi/si/seart/converter/DateToLDTConverter.java b/dl4se-crawler/src/main/java/usi/si/seart/converter/DateToLDTConverter.java
deleted file mode 100644
index ed891f56..00000000
--- a/dl4se-crawler/src/main/java/usi/si/seart/converter/DateToLDTConverter.java
+++ /dev/null
@@ -1,26 +0,0 @@
-package usi.si.seart.converter;
-
-import lombok.AccessLevel;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-
-import java.time.LocalDateTime;
-import java.time.ZoneOffset;
-import java.util.Date;
-
-@NoArgsConstructor(access = AccessLevel.PRIVATE)
-public class DateToLDTConverter extends Converter {
-
- @Getter
- public static final Converter instance = new DateToLDTConverter();
-
- @Override
- protected LocalDateTime forward(Date date) {
- return LocalDateTime.ofInstant(date.toInstant(), ZoneOffset.UTC);
- }
-
- @Override
- protected Date backward(LocalDateTime ldt) {
- return new Date(ldt.toInstant(ZoneOffset.UTC).toEpochMilli());
- }
-}
diff --git a/dl4se-crawler/src/main/java/usi/si/seart/converter/GhsToGitRepoConverter.java b/dl4se-crawler/src/main/java/usi/si/seart/converter/GhsToGitRepoConverter.java
deleted file mode 100644
index 7f84fb5d..00000000
--- a/dl4se-crawler/src/main/java/usi/si/seart/converter/GhsToGitRepoConverter.java
+++ /dev/null
@@ -1,45 +0,0 @@
-package usi.si.seart.converter;
-
-import lombok.AccessLevel;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import usi.si.seart.http.payload.GhsGitRepo;
-import usi.si.seart.model.GitRepo;
-
-import java.time.LocalDateTime;
-
-@NoArgsConstructor(access = AccessLevel.PRIVATE)
-public class GhsToGitRepoConverter extends Converter {
-
- @Getter
- private static final Converter instance = new GhsToGitRepoConverter();
-
- @Override
- protected GitRepo forward(GhsGitRepo ghsGitRepo) {
- GitRepo.GitRepoBuilder builder = GitRepo.builder();
-
- builder.name(ghsGitRepo.getName());
- builder.license(ghsGitRepo.getLicense());
- builder.isFork(ghsGitRepo.getIsFork());
-
- Long commits = ghsGitRepo.getCommits();
- if (commits != null) builder.commits(commits);
- Long contributors = ghsGitRepo.getContributors();
- if (contributors != null) builder.contributors(contributors);
- Long issues = ghsGitRepo.getTotalIssues();
- if (issues != null) builder.issues(issues);
- Long stars = ghsGitRepo.getStargazers();
- if (stars != null) builder.stars(stars);
- LocalDateTime lastCommit = DateToLDTConverter.getInstance().convert(ghsGitRepo.getLastCommit());
- if (lastCommit != null) builder.lastCommit(lastCommit);
- String lastCommitSHA = ghsGitRepo.getLastCommitSHA();
- if (lastCommitSHA != null) builder.lastCommitSHA(lastCommitSHA);
-
- return builder.build();
- }
-
- @Override
- protected GhsGitRepo backward(GitRepo repo) {
- throw new UnsupportedOperationException("Backwards conversion is not supported!");
- }
-}
diff --git a/dl4se-crawler/src/main/java/usi/si/seart/git/GitException.java b/dl4se-crawler/src/main/java/usi/si/seart/git/GitException.java
deleted file mode 100644
index 74a537c6..00000000
--- a/dl4se-crawler/src/main/java/usi/si/seart/git/GitException.java
+++ /dev/null
@@ -1,8 +0,0 @@
-package usi.si.seart.git;
-
-public class GitException extends Exception {
-
- public GitException(String message) {
- super(message);
- }
-}
diff --git a/dl4se-crawler/src/main/java/usi/si/seart/io/PropertiesReader.java b/dl4se-crawler/src/main/java/usi/si/seart/io/PropertiesReader.java
deleted file mode 100644
index 68d70412..00000000
--- a/dl4se-crawler/src/main/java/usi/si/seart/io/PropertiesReader.java
+++ /dev/null
@@ -1,81 +0,0 @@
-package usi.si.seart.io;
-
-import lombok.AccessLevel;
-import lombok.SneakyThrows;
-import lombok.experimental.FieldDefaults;
-
-import java.io.InputStream;
-import java.util.Properties;
-import java.util.function.Function;
-import java.util.function.UnaryOperator;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * Class used for reading {@code Properties} files. It has the added benefit of being able to resolve references to
- * other file properties, system properties and environment variables, with the resolution precedence maintaining the
- * aforementioned order. The property values themselves remain preserved as they were written in the file, with the
- * resolution taking place once {@link #getProperty} is called. Only ${...} is supported for property
- * referencing. Nested resolution (i.e. ${...${...}...}) is not supported.
- *
- * @author dabico
- * @see System#getProperty(String) System.getProperty
- * @see System#getenv(String) System.getenv
- * @see Properties
- * @see Matcher
- * @see Pattern Definition
- */
-@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
-public class PropertiesReader {
-
- static Pattern referencePattern = Pattern.compile("\\$\\{([^}]+)\\}");
-
- Properties properties;
- Function resolver;
-
- @SneakyThrows
- public PropertiesReader(String fileName) {
- this.properties = new Properties();
- InputStream is = getClass().getClassLoader().getResourceAsStream(fileName);
- this.properties.load(is);
-
- UnaryOperator resolver1 = resolver(this.properties::getProperty);
- UnaryOperator resolver2 = resolver(System::getProperty);
- UnaryOperator resolver3 = resolver(System::getenv);
- resolver = resolver1.andThen(resolver2).andThen(resolver3);
- }
-
- /**
- * Searches for the property with the specified key in this property list, performing resolutions if the property
- * value contains references to other properties or variables. The method returns null if the property is not found.
- *
- * @param key The property key.
- * @return The resolved property value.
- */
- public String getProperty(String key) {
- String value = this.properties.getProperty(key);
- if (value != null) return resolver.apply(value);
- else return null;
- }
-
- private UnaryOperator resolver(UnaryOperator mapper) {
- return str -> {
- Matcher matcher = referencePattern.matcher(str);
-
- if (str.matches(".*"+referencePattern.pattern()+".*")) {
- StringBuilder builder = new StringBuilder();
-
- while (matcher.find()) {
- String name = matcher.group(1);
- String value = mapper.apply(name);
- if (value != null) matcher.appendReplacement(builder, Matcher.quoteReplacement(value));
- }
-
- matcher.appendTail(builder);
- return builder.toString();
- }
-
- return str;
- };
- }
-}
diff --git a/dl4se-crawler/src/main/java/usi/si/seart/parser/AbstractParser.java b/dl4se-crawler/src/main/java/usi/si/seart/parser/AbstractParser.java
deleted file mode 100644
index 993cea2f..00000000
--- a/dl4se-crawler/src/main/java/usi/si/seart/parser/AbstractParser.java
+++ /dev/null
@@ -1,33 +0,0 @@
-package usi.si.seart.parser;
-
-import lombok.AccessLevel;
-import lombok.experimental.FieldDefaults;
-import usi.si.seart.model.Language;
-import usi.si.seart.model.code.File;
-import usi.si.seart.model.code.Function;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-
-@FieldDefaults(level = AccessLevel.PROTECTED)
-public abstract class AbstractParser implements Parser {
-
- File.FileBuilder fileBuilder = File.builder();
- List> functionBuilders = new ArrayList<>();
-
- Language language;
-
- protected AbstractParser(Language language) {
- this.language = language;
- }
-
- protected File buildFileAndFunctions() {
- File file = fileBuilder.build();
- List functions = functionBuilders.stream()
- .map(builder -> builder.file(file).isTest(file.getIsTest()).build())
- .collect(Collectors.toList());
- file.setFunctions(functions);
- return file;
- }
-}
diff --git a/dl4se-crawler/src/main/java/usi/si/seart/parser/FallbackParser.java b/dl4se-crawler/src/main/java/usi/si/seart/parser/FallbackParser.java
deleted file mode 100644
index 39bed750..00000000
--- a/dl4se-crawler/src/main/java/usi/si/seart/parser/FallbackParser.java
+++ /dev/null
@@ -1,41 +0,0 @@
-package usi.si.seart.parser;
-
-import lombok.extern.slf4j.Slf4j;
-import usi.si.seart.model.Language;
-import usi.si.seart.model.code.File;
-import usi.si.seart.utils.PathUtils;
-import usi.si.seart.utils.StringUtils;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-
-@Slf4j
-public class FallbackParser extends AbstractParser {
-
- public FallbackParser(Language language) {
- super(language);
- }
-
- @Override
- public File parse(Path path) {
- fileBuilder.isTest(PathUtils.isTestFile(path));
- fileBuilder.language(language);
-
- try {
- String fileContents = Files.readString(path);
- String normalized = StringUtils.normalizeSpace(fileContents);
-
- fileBuilder.content(fileContents);
- fileBuilder.contentHash(StringUtils.sha256(normalized));
- fileBuilder.lines(fileContents.lines().count());
- fileBuilder.characters(fileContents.chars().count());
- fileBuilder.containsNonAscii(StringUtils.containsNonAscii(fileContents));
-
- return buildFileAndFunctions();
- } catch (IOException ex) {
- log.error("Could not read file: " + path, ex);
- return null;
- }
- }
-}
diff --git a/dl4se-crawler/src/main/java/usi/si/seart/parser/JavaParser.java b/dl4se-crawler/src/main/java/usi/si/seart/parser/JavaParser.java
deleted file mode 100644
index ba38e506..00000000
--- a/dl4se-crawler/src/main/java/usi/si/seart/parser/JavaParser.java
+++ /dev/null
@@ -1,212 +0,0 @@
-package usi.si.seart.parser;
-
-import com.github.javaparser.JavaToken;
-import com.github.javaparser.ParseProblemException;
-import com.github.javaparser.StaticJavaParser;
-import com.github.javaparser.ast.CompilationUnit;
-import com.github.javaparser.ast.Node;
-import com.github.javaparser.ast.body.CallableDeclaration;
-import com.github.javaparser.ast.body.ConstructorDeclaration;
-import com.github.javaparser.ast.body.MethodDeclaration;
-import com.github.javaparser.ast.comments.Comment;
-import com.github.javaparser.ast.visitor.VoidVisitorAdapter;
-import com.github.javaparser.printer.XmlPrinter;
-import lombok.extern.slf4j.Slf4j;
-import usi.si.seart.collection.Tuple;
-import usi.si.seart.model.Language;
-import usi.si.seart.model.code.Boilerplate;
-import usi.si.seart.model.code.Code;
-import usi.si.seart.model.code.File;
-import usi.si.seart.model.code.Function;
-import usi.si.seart.utils.PathUtils;
-import usi.si.seart.utils.StringUtils;
-
-import java.io.FileNotFoundException;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Spliterator;
-import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
-
-@Slf4j
-public class JavaParser extends AbstractParser {
-
- private static final XmlPrinter astPrinter = new XmlPrinter(true);
-
- public JavaParser(Language language) {
- super(language);
- }
-
- @Override
- public File parse(Path path) throws ParsingException {
- fileBuilder.isTest(PathUtils.isTestFile(path));
-
- try {
- CompilationUnit compilationUnit = StaticJavaParser.parse(path.toFile());
- fileBuilder.isParsed(true);
- new VoidVisitor().visit(compilationUnit, null);
- } catch (ParseProblemException | FileNotFoundException | StackOverflowError thr) {
- log.error("Parsing failed for: " + path, thr);
- throw new ParsingException(thr.getMessage(), thr.getCause());
- }
-
- return buildFileAndFunctions();
- }
-
- private class VoidVisitor extends VoidVisitorAdapter