Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[v3.0.x] Remove Hadoop dependencies #119

Merged
merged 35 commits into from
Sep 20, 2023
Merged

[v3.0.x] Remove Hadoop dependencies #119

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
ab12ce5
Remove haddop support and refactor workspace
lukfor Aug 23, 2023
1788135
Refactor workspace classes
lukfor Aug 23, 2023
8e6bdc4
Support s3 in Nextflow tasks
lukfor Sep 4, 2023
f432128
Fix settings and cleanup old values
lukfor Sep 4, 2023
7ec922f
Fix uploadLimit
lukfor Sep 4, 2023
11cd4fa
Fix log files of NextflowTasks
lukfor Sep 5, 2023
8e26679
Add reporting and improve Nextflow support
lukfor Sep 5, 2023
4658d04
Fix local workspace and downloads
lukfor Sep 5, 2023
85c0c99
Cleanup engine and remove seconda queue and DAG
lukfor Sep 6, 2023
9c69209
Remove cloudgene-sdk, improve logging and implement reporting
lukfor Sep 7, 2023
a2e9a8c
Refactore NextflowCommand and NextflowCollector
lukfor Sep 7, 2023
d48dfe3
Run Nextflow jobs in workspace directory
lukfor Sep 7, 2023
abe2788
Fix missing command in report
lukfor Sep 7, 2023
772b0e6
Export environment variables when run Nextflow
lukfor Sep 8, 2023
b6c4c4e
Add global nextflow config file and add webui
lukfor Sep 9, 2023
243d9ff
Remove preprocessing queue
lukfor Sep 9, 2023
e4efab2
Add contact details and server url to admin panel
lukfor Sep 9, 2023
efe4b63
Improve application settings and environment variables
lukfor Sep 9, 2023
ffc82ef
Fix issue with job in environment
lukfor Sep 9, 2023
c1795bf
Fix issue with email settings
lukfor Sep 9, 2023
cba304f
Add links to nextflow.log, trace and report
lukfor Sep 9, 2023
53458eb
Fix issue with wrong username in jobs list
lukfor Sep 9, 2023
096e29b
Add workspace to admin panel
lukfor Sep 9, 2023
b3ffd7f
Fix issue with filenames in local workspace
lukfor Sep 9, 2023
7fbf8ab
Clean up downloads and fix logs on job restart
lukfor Sep 10, 2023
bf4a030
Remove delete temp directory on setup job to enable resume
lukfor Sep 11, 2023
9bd1ee6
Add `label` to process config
lukfor Sep 11, 2023
da6f87b
Improve Nextflow feedback rendering
lukfor Sep 11, 2023
11f782b
Add job name to environment
lukfor Sep 11, 2023
240462d
Set java version to 11
lukfor Sep 11, 2023
d8105db
Remove deprecated properties from job
lukfor Sep 12, 2023
cb1d646
Remove deprectated property progress
lukfor Sep 13, 2023
44afc36
Extended binded lists
lukfor Sep 17, 2023
3b26d30
Upload job infos to workspace
lukfor Sep 18, 2023
45011a4
Support cloudgene.report.json on step level
lukfor Sep 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 6 additions & 93 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,9 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<packaging>jar</packaging>
<jdk.version>1.8</jdk.version>
<release.version>8</release.version>
<jdk.version>11</jdk.version>
<release.version>11</release.version>
<java.version>11</java.version>
<micronaut.version>3.3.4</micronaut.version>
<exec.mainClass>cloudgene.mapred.server.Application</exec.mainClass>
<micronaut.runtime>netty</micronaut.runtime>
Expand All @@ -67,15 +68,9 @@
<repositories>

<repository>
<id>jfrog-genepi-maven</id>
<name>jfrog-genepi-maven</name>
<url>https://genepi.jfrog.io/artifactory/maven/</url>
</repository>

<repository>
<id>cloudera</id>
<name>Cloudera repository</name>
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
<id>genepi-maven</id>
<name>genepi-maven</name>
<url>https://genepi.i-med.ac.at/maven</url>
</repository>

</repositories>
Expand All @@ -84,12 +79,6 @@

<dependencies>

<dependency>
<groupId>cloudgene</groupId>
<artifactId>cloudgene-java-sdk</artifactId>
<version>1.0.2</version>
</dependency>

<dependency>
<groupId>genepi</groupId>
<artifactId>genepi-io</artifactId>
Expand All @@ -106,26 +95,6 @@
</exclusions>
</dependency>

<dependency>
<groupId>genepi</groupId>
<artifactId>genepi-hadoop</artifactId>
<version>mr1-1.4.1</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>commons-dbutils</groupId>
<artifactId>commons-dbutils</artifactId>
Expand Down Expand Up @@ -217,62 +186,6 @@
<version>5.8.3</version>
</dependency>

<!-- Test Dependencies -->


<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
<scope>test</scope>
<type>test-jar</type>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-reload4j</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<scope>test</scope>
<type>test-jar</type>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-reload4j</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-test</artifactId>
<version>2.6.0-mr1-cdh5.16.1</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
Expand Down
19 changes: 15 additions & 4 deletions src/main/html/webapp/admin.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,18 @@ import UserListControl from 'components/admin/user/list/';
import JobListControl from 'components/admin/job/list/';
import JobDetailControl from 'components/core/job/detail/';
import AppListControl from 'components/admin/app/list/';
import AppSettingsControl from 'components/admin/app/settings/';

import AppRepositoryControl from 'components/admin/app/repository/';
import SettingsGeneralControl from 'components/admin/settings/general/';
import SettingsNextflowControl from 'components/admin/settings/nextflow/';
import SettingsServerControl from 'components/admin/settings/server/';
import SettingsMailControl from 'components/admin/settings/mail/';
import SettingsTemplatesControl from 'components/admin/settings/templates/';
import SettingsLogsControl from 'components/admin/settings/logs/';


$(document.links).filter(function() {
$(document.links).filter(function () {
return this.hostname != window.location.hostname;
}).attr('target', '_blank');

Expand Down Expand Up @@ -64,6 +67,10 @@ var routes = [{
path: 'pages/admin-apps',
control: AppListControl,
guard: adminGuard
}, {
path: 'pages/admin-apps/{app}',
control: AppSettingsControl,
guard: adminGuard
}, {
path: 'pages/admin-apps-repository',
control: AppRepositoryControl,
Expand All @@ -76,6 +83,10 @@ var routes = [{
path: 'pages/admin-settings-general',
control: SettingsGeneralControl,
guard: adminGuard
}, {
path: 'pages/admin-settings-nextflow',
control: SettingsNextflowControl,
guard: adminGuard
}, {
path: 'pages/admin-settings-mail',
control: SettingsMailControl,
Expand Down Expand Up @@ -106,9 +117,9 @@ function adminGuard(appState) {
}
}

$.ajaxPrefilter(function(options, orig, xhr) {
$.ajaxPrefilter(function (options, orig, xhr) {
if (!options.beforeSend) {
options.beforeSend = function(xhr) {
options.beforeSend = function (xhr) {
if (localStorage.getItem("cloudgene")) {
try {
// get data
Expand All @@ -133,7 +144,7 @@ $.ajaxPrefilter(function(options, orig, xhr) {
});


Server.findOne({}, function(server) {
Server.findOne({}, function (server) {

new LayoutControl("#main", {
appState: server
Expand Down
Loading
Loading