Merge pull request #39 from thiagocoutinhor/v0.1.3-alpha

v0.1.3-alpha

.env

@@ -1,8 +1,9 @@
 MODE=PRODUCTION
 LOG_LEVEL=INFO
 PORT=9099
+LOGIN_TYPE=PASSWORD
 SPARK_HOST=hdcpx02.interno
 MONGO=mongodb://notebook:burning-book@localhost:27017
 USER_BLACKLIST=sods3001,phdpdig
 SPARK_QUEUE=root.digital.users
-SPARK_LIBRARIES=/data2/digital_stage/app/production/scala/Load_digital/lib/commons-csv-1.2.jar,/data2/digital_stage/app/production/scala/Load_digital/lib/spark-csv_2.10-1.5.0.jar
+SPARK_LIBRARIES=/data2/digital_stage/app/production/scala/Load_digital/lib/commons-csv-1.2.jar,/data2/digital_stage/app/production/scala/Load_digital/lib/spark-csv_2.10-1.5.0.jar

CHANGELOG.md

@@ -1,3 +1,27 @@
+v0.1.3-alpha
+===============================================================================
+
+### Bug Fixes
+- Corrects the progress bar calculation (was adding the numbers in parentesis
+before)
+- Progress bars are always marked as done when the chunk is done running
+- Fixes the "run all above" bug where once activatede always runs to the end
+- Fixes the progress bar bug when there where no "=" or ">" character
+- Fixes the receit not following the chunk on move up and down
+
+### Quality of Life
+- Execution tags on the chunks
+- Adds formatted tables to the result card
+- Naming chunks now is possible
+- Spark configuration is now saved in each book
+- Download of the code now is possible
+- When running all above the screen jumps to the receipt of the running chunk
+- Run all above now mark all the chunks above as running
+- Login using SSH Identity file
+
+### Internal
+- Protects the user password in the session store
+
 v0.1.2-alpha
 ===============================================================================
 

README.md

@@ -48,6 +48,12 @@ services:
             MONGO_INITDB_ROOT_PASSWORD: book
 ```
 
+## Connecting with identity files
+
+If you need a identity file to login in your spark host (AWS users, for example), set the
+enviroment variable `LOGIN_TYPE` to `SSH`. Changing it to `SSH` will make the login screen ask
+for your identity file during the login.
+
 ## Enviroment Variables
 
 | Variable          | Default                   | Meaning                                       |
@@ -58,3 +64,4 @@ services:
 | USER_BLACKLIST    |                           | Users to be denied access                     |
 | SPARK_QUEUE       |                           | Default queue of all new spark-shell sessions |
 | SPARK_LIBRARIES   |                           | Libraries in SPARK_HOST to be used            |
+| LOGIN_TYPE        | PASSWORD                  | Type of login, between PASSWORD and SSH       |

api/book/book-model.js

@@ -4,10 +4,16 @@ const newName = 'Novo Book'
 const schema = new mongoose.Schema({
     name: { type: String, required: true },
     commands: [{
+        name: String,
         command: String
     }],
     owner: { type: String, required: true },
-    sharedWith: [String]
+    sharedWith: [String],
+    sparkConfig: {
+        executors: Number,
+        cores: Number,
+        memory: Number
+    }
 })
 
 schema.index({

api/crypt/password-utils.js

@@ -0,0 +1,20 @@
+const crypto = require('crypto')
+
+// Password cryptografy
+const key = crypto.randomBytes(32)
+const iv = crypto.randomBytes(16)
+
+function crush(password) {
+    const cypher = crypto.createCipheriv('aes-256-ctr', key, iv)
+    return Buffer.concat([cypher.update(password), cypher.final()]).toString('hex')
+}
+
+function uncrush(password) {
+    const decript = crypto.createDecipheriv('aes-256-ctr', key, iv)
+    return Buffer.concat([decript.update(Buffer.from(password, 'hex')), decript.final()]).toString()
+}
+
+module.exports = {
+    crush,
+    uncrush
+}

api/socket/book-socket.js

@@ -1,5 +1,7 @@
 const Book = require('../book/book-model').Book
-const books = {} // Controle para evitar dissincronia dos books entre sockets
+// Controle para evitar dissincronia dos books entre sockets
+// TODO impede o crescimento horizontal da aplicação. Rever talvez seja necessário
+const books = {}
 
 module.exports = socket => {
     const usuario = socket.handshake.session.usuario
@@ -26,7 +28,7 @@ module.exports = socket => {
             console.warn(`[IO BOOK - ${usuario.login}] Tentativa de acesso a um book que não existe ${bookId}`)
             socket.emit('exit')
             socket.disconnect()
-            returns
+            return
         }
 
         if (!temAcesso(book)) {
@@ -78,6 +80,34 @@ module.exports = socket => {
             socket.broadcast.to(bookId).emit('update', index, command)
         })
 
+        socket.on('chunk.name', (index, name) => {
+            book.commands[index].name = name
+            book.save()
+            socket.broadcast.to(bookId).emit('chunk.name', index, name)
+        })
+
+        socket.on('chunk.move', (source, destination) => {
+            const comSource = book.commands[source]
+            const comDestination = book.commands[destination]
+            book.commands[source] = comDestination
+            book.commands[destination] = comSource
+            book.markModified('commands')
+            book.save()
+            console.log(comDestination, book.commands[source])
+            console.log(`move ${source} > ${destination}`)
+            socket.broadcast.to(bookId).emit('chunk.move', source, destination)
+        })
+
+        socket.on('spark.config', (executors, cores, memory) => {
+            book.sparkConfig = {
+                executors,
+                cores,
+                memory
+            }
+            book.save()
+            socket.broadcast.to(bookId).emit('spark.config', executors, cores, memory)
+        })
+
         socket.on('disconnect', () => {
             console.info(`[IO BOOK - ${usuario.login}] Desconectou`)
             books[bookId].count--

api/socket/spark-socket.js

@@ -1,8 +1,11 @@
 const SparkShell = require('../spark-shell/spark-shell').SparkSession
 const Stream = require('stream').PassThrough
+const passwordUtils = require('../crypt/password-utils')
 
 module.exports = socket => {
     const usuario = socket.handshake.session.usuario
+    usuario.senha = passwordUtils.uncrush(usuario.senha)
+
     const config = socket.handshake.query
     console.debug(`[IO SPARK - ${usuario.login}] Conectou`)
 

api/spark-shell/spark-shell-stub.js

@@ -3,7 +3,7 @@ const { PassThrough } = require('stream')
 
 // Configuração do mock para facil mudança
 const config = {
-    shellOpenTime: 5 * 1000,
+    shellOpenTime: 2 * 1000,
     mockRunCommand: (user, comando, stream) => {
         console.log(`[SPARK MOCK - ${user}] Run recieved\n${comando}`)
 
@@ -18,8 +18,9 @@ const config = {
 
         // Quantos contadores e em que intervalo enviar
         const stages = [
-            { step: 10, progresso: 0 },
-            { step: 15, progresso: 0 }
+            { step: 50, progresso: 0 },
+            { step: 30, progresso: 0 },
+            { step: 40, progresso: 0, incompleto: true }
         ]
 
         // Envia os contadores de progresso a cada segundo
@@ -28,7 +29,8 @@ const config = {
             const timer = setInterval(() => {
                 stage.progresso += stage.step
                 stream.emit('data', progress(stage.progresso, index))
-                if (stage.progresso >= 100) {
+                const parar = (stage.incompleto && stage.progresso >= 80) || (stage.progresso >= 100)
+                if (parar) {
                     finalizado++
                     clearInterval(timer)
                     if (finalizado == stages.length) {

api/spark-shell/spark-shell.js

@@ -1,5 +1,7 @@
 const Ssh = require('ssh2-promise')
 
+const LOGIN_TYPE = process.env.LOGIN_TYPE ? process.env.LOGIN_TYPE : 'PASSWORD'
+
 // Classe responsável pela conexão e criação de uma nova sessão do spark
 class SparkSession {
 
@@ -14,12 +16,19 @@ class SparkSession {
         this.__user = user
         console.debug(`[SPARK - ${this.__user}] Command:\n\t${this.__startCommand}`)
 
-        this.ssh = new Ssh({
+        const parameters = {
             host: process.env.SPARK_HOST,
             username: user.toLowerCase(),
-            password: password,
             keepaliveInterval: 60 * 1000
-        })
+        }
+
+        if (LOGIN_TYPE === 'PASSWORD') {
+            parameters.password = password
+        } else if (LOGIN_TYPE === 'SSH') {
+            parameters.privateKey = password
+        }
+
+        this.ssh = new Ssh(parameters)
     }
 
     connect() {

dockerfile

@@ -4,6 +4,7 @@ WORKDIR /app
 RUN rm /app/.env
 RUN npm i --production
 ENV LOG_LEVEL=INFO
+ENV LOGIN_TYPE=PASSWORD
 ENV SPARK_HOST=localhost
 ENV MONGO=mongodb://localhost:27017
 ENV USER_BLACKLIST=

package.json

@@ -31,6 +31,7 @@
     "d3": "^5.15.0",
     "dotenv": "^8.2.0",
     "express": "^4.17.1",
+    "express-fileupload": "^1.1.7-alpha.1",
     "express-session": "^1.17.0",
     "express-socket.io-session": "^1.3.5",
     "jquery": "^3.4.1",

web/css/book-list.css

@@ -2,6 +2,10 @@ body {
     background-color: darkslategray
 }
 
+a {
+    cursor: pointer;
+}
+
 .navbar {
     background-color: rgb(88, 131, 131);
     color: white;

web/css/book.css

@@ -4,6 +4,10 @@ body {
     overflow-x: hidden;
 }
 
+a {
+    cursor: pointer;
+}
+
 .pointer {
     cursor: pointer;
 }
@@ -80,10 +84,6 @@ body {
     border-bottom: 1px solid rgb(216, 216, 216);
 }
 
-.command-block.running .card-title {
-    border-bottom: 1px solid rgb(170, 170, 170);
-}
-
 .command-block .card-title .dropdown-menu {
     font-size: 100%;
 }
@@ -110,6 +110,16 @@ body {
     background-color: silver;
 }
 
+.command-block.running .card-title {
+    border-bottom: 1px solid rgb(170, 170, 170);
+}
+
+.command-block.running .bloco::after {
+    content: 'Running...';
+    color: blue;
+    margin-left: 5px;
+}
+
 .command-block.running .button-remove {
     display: none;
 }
@@ -122,6 +132,17 @@ body {
     display: none;
 }
 
+.command-block.done .bloco::after {
+    content: 'Done';
+    color: #0e8913;
+    margin-left: 5px;
+}
+
+.command-block.done .recibo .progress-bar {
+    background-color: #28a745;
+    width: 100%!important;
+}
+
 .recibo {
     font-family: 'Courier New', Courier, monospace;
     border: 1px solid gray;
@@ -138,6 +159,11 @@ body {
     color: white
 }
 
+.recibo .table td,
+.recibo .table th {
+    border: none
+}
+
 button.new-command {
     width: 100%;
     background-color: rgba(0, 0, 0, 0.13);

web/pages/book-list.html

@@ -63,16 +63,12 @@
                         <i class="fa fa-ellipsis-v"></i>
                     </span>
                     <div class="dropdown-menu">
-                        <a class="dropdown-item" href="#" onclick="compartilharScreen('XXX')"data-toggle="modal" data-target="#compartilhar">
+                        <a class="dropdown-item" onclick="compartilharScreen('XXX')"data-toggle="modal" data-target="#compartilhar">
                             <i class="fa fa-share-alt"></i>
                             Compartilhar
                         </a>
-                        <a class="dropdown-item rodar-ate disabled" href="#" onclick="runAllTo(0)">
-                            <i class="fa fa-play"></i>
-                            Rodar todos acima
-                        </a>
                         <div class="dropdown-divider"></div>
-                        <a class="dropdown-item" href="#" onclick="remover('XXXX')">
+                        <a class="dropdown-item" onclick="remover('XXXX')">
                             <i class="fa fa-trash"></i>
                             Remover
                         </a>