Skip to content

Commit

Permalink
7.73
Browse files Browse the repository at this point in the history 
CBC-MAC optimization
  • Loading branch information
tonikelope committed Jan 26, 2023
1 parent 1b076aa commit ee5a5a7
Show file tree
Hide file tree
Showing 9 changed files with 113 additions and 108 deletions.
7 changes: 6 additions & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.tonikelope</groupId>
<artifactId>MegaBasterd</artifactId>
<version>7.72</version>
<version>7.73</version>
<packaging>jar</packaging>
<repositories>
<repository>
Expand All @@ -20,6 +20,11 @@
</repository>
</repositories>
<dependencies>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.11.0</version>
</dependency>
<dependency>
<groupId>org.sejda.webp-imageio</groupId>
<artifactId>webp-imageio-sejda</artifactId>
Expand Down
29 changes: 24 additions & 5 deletions src/main/java/com/tonikelope/megabasterd/ChunkUploader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.crypto.CipherInputStream;
import org.apache.commons.io.input.QueueInputStream;
import org.apache.commons.io.output.QueueOutputStream;

/**
*
Expand Down Expand Up @@ -117,6 +119,8 @@ public void run() {

byte[] buffer = new byte[MainPanel.DEFAULT_BYTE_BUFFER_SIZE];

byte[] buffer_enc = new byte[MainPanel.DEFAULT_BYTE_BUFFER_SIZE];

boolean fatal_error = false;

int conta_error = 0;
Expand All @@ -141,9 +145,9 @@ public void run() {

chunk_id = _upload.nextChunkId();

long chunk_offset = ChunkWriterManager.calculateChunkOffset(chunk_id, Upload.CHUNK_SIZE_MULTI);
long chunk_offset = ChunkWriterManager.calculateChunkOffset(chunk_id, 1);

long chunk_size = ChunkWriterManager.calculateChunkSize(chunk_id, _upload.getFile_size(), chunk_offset, Upload.CHUNK_SIZE_MULTI);
long chunk_size = ChunkWriterManager.calculateChunkSize(chunk_id, _upload.getFile_size(), chunk_offset, 1);

ChunkWriterManager.checkChunkID(chunk_id, _upload.getFile_size(), chunk_offset);

Expand Down Expand Up @@ -194,12 +198,25 @@ public void run() {

f.seek(chunk_offset);

try ( CipherInputStream cis = new CipherInputStream(new BufferedInputStream(Channels.newInputStream(f.getChannel())), genCrypter("AES", "AES/CTR/NoPadding", _upload.getByte_file_key(), forwardMEGALinkKeyIV(_upload.getByte_file_iv(), chunk_offset))); OutputStream out = new ThrottledOutputStream(con.getOutputStream(), _upload.getMain_panel().getStream_supervisor())) {
ByteArrayOutputStream chunk_mac = new ByteArrayOutputStream();

try ( QueueInputStream qis = new QueueInputStream(); QueueOutputStream qos = qis.newQueueOutputStream(); BufferedInputStream bis = new BufferedInputStream(Channels.newInputStream(f.getChannel())); CipherInputStream cis = new CipherInputStream(qis, genCrypter("AES", "AES/CTR/NoPadding", _upload.getByte_file_key(), forwardMEGALinkKeyIV(_upload.getByte_file_iv(), chunk_offset))); OutputStream out = new ThrottledOutputStream(con.getOutputStream(), _upload.getMain_panel().getStream_supervisor())) {

LOG.log(Level.INFO, "{0} Uploading chunk {1} from worker {2} {3}...", new Object[]{Thread.currentThread().getName(), chunk_id, _id, _upload.getFile_name()});

while (!_exit && tot_bytes_up < chunk_size && (reads = cis.read(buffer)) != -1) {
out.write(buffer, 0, reads);
while (!_exit && tot_bytes_up < chunk_size && (reads = bis.read(buffer)) != -1) {

chunk_mac.write(buffer, 0, reads);

for (int i = 0; i < reads; i++) {
qos.write(buffer[i]);
}

for (int i = 0; i < reads; i++) {
buffer_enc[i] = (byte) cis.read();
}

out.write(buffer_enc, 0, reads);

_upload.getPartialProgress().add((long) reads);

Expand All @@ -215,6 +232,8 @@ public void run() {

}
}

_upload.getMac_generator().CHUNK_QUEUE.put(chunk_offset, chunk_mac);
}

if (!_exit) {
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/com/tonikelope/megabasterd/MainPanel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@
*/
public final class MainPanel {

public static final String VERSION = "7.72";
public static final String VERSION = "7.73";
public static final boolean FORCE_SMART_PROXY = false; //TRUE FOR DEBUGING SMART PROXY
public static final int THROTTLE_SLICE_SIZE = 16 * 1024;
public static final int DEFAULT_BYTE_BUFFER_SIZE = 16 * 1024;
Expand Down
55 changes: 16 additions & 39 deletions src/main/java/com/tonikelope/megabasterd/MiscTools.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
Expand All @@ -48,6 +49,7 @@
import java.net.URL;
import java.net.URLDecoder;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.IntBuffer;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
Expand Down Expand Up @@ -239,54 +241,29 @@ public static void setNimbusLookAndFeel() {
public static int[] bin2i32a(byte[] bin) {
int l = (int) (4 * Math.ceil((double) bin.length / 4));

byte[] new_bin = Arrays.copyOfRange(bin, 0, l);
IntBuffer intBuf = ByteBuffer.wrap(bin, 0, l).order(ByteOrder.BIG_ENDIAN).asIntBuffer();

bin = new_bin;
int[] array = new int[intBuf.remaining()];

ByteBuffer bin_buffer = ByteBuffer.wrap(bin);
IntBuffer int_buffer = bin_buffer.asIntBuffer();
intBuf.get(array);

if (int_buffer.hasArray()) {
return int_buffer.array();
} else {
ArrayList<Integer> list = new ArrayList<>();

while (int_buffer.hasRemaining()) {
list.add(int_buffer.get());
}

int[] aux = new int[list.size()];

for (int i = 0; i < aux.length; i++) {
aux[i] = list.get(i);
}

return aux;
}
return array;
}

public static byte[] i32a2bin(int[] i32a) {
ByteBuffer bin_buffer = ByteBuffer.allocate(i32a.length * 4);
IntBuffer int_buffer = bin_buffer.asIntBuffer();
int_buffer.put(i32a);
public static byte[] i32a2bin(int[] values) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();

if (bin_buffer.hasArray()) {
return bin_buffer.array();
} else {
ArrayList<Byte> list = new ArrayList<>();

while (int_buffer.hasRemaining()) {
list.add(bin_buffer.get());
}

byte[] aux = new byte[list.size()];
DataOutputStream dos = new DataOutputStream(baos);

for (int i = 0; i < aux.length; i++) {
aux[i] = list.get(i);
for (int i = 0; i < values.length; ++i) {
try {
dos.writeInt(values[i]);
} catch (IOException ex) {
Logger.getLogger(MiscTools.class.getName()).log(Level.SEVERE, null, ex);
}

return aux;
}

return baos.toByteArray();
}

public static BigInteger mpi2big(byte[] s) {
Expand Down
5 changes: 2 additions & 3 deletions src/main/java/com/tonikelope/megabasterd/Upload.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@
public class Upload implements Transference, Runnable, SecureSingleThreadNotifiable {

public static final int WORKERS_DEFAULT = 6;
public static final int CHUNK_SIZE_MULTI = 1; //Otra cosa da errores al reanudar una subida (investigar)
public static final boolean DEFAULT_THUMBNAILS = true;
public static final boolean UPLOAD_LOG = false;
private static final Logger LOG = Logger.getLogger(Upload.class.getName());
Expand All @@ -62,7 +61,7 @@ public class Upload implements Transference, Runnable, SecureSingleThreadNotifia
private long _last_chunk_id_dispatched;
private final ConcurrentLinkedQueue<Long> _partialProgressQueue;
private final ExecutorService _thread_pool;
private int[] _file_meta_mac;
private volatile int[] _file_meta_mac;
private String _fid;
private boolean _notified;
private volatile String _completion_handler;
Expand Down Expand Up @@ -1305,7 +1304,7 @@ public boolean isStatusError() {
public long calculateLastUploadedChunk(long bytes_read) {

if (bytes_read > 3584 * 1024) {
return 7 + (long) Math.floor((float) (bytes_read - 3584 * 1024) / (1024 * 1024 * Upload.CHUNK_SIZE_MULTI));
return 7 + (long) Math.floor((float) (bytes_read - 3584 * 1024) / (1024 * 1024 * 1));
} else {
long i = 0, tot = 0;

Expand Down
96 changes: 51 additions & 45 deletions src/main/java/com/tonikelope/megabasterd/UploadMACGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@

import static com.tonikelope.megabasterd.CryptTools.*;
import static com.tonikelope.megabasterd.MiscTools.*;
import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.util.HashMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.crypto.BadPaddingException;
Expand All @@ -33,6 +33,7 @@ public class UploadMACGenerator implements Runnable, SecureSingleThreadNotifiabl
private final Object _secure_notify_lock;
private boolean _notified;
private volatile boolean _exit;
public final ConcurrentHashMap<Long, ByteArrayOutputStream> CHUNK_QUEUE = new ConcurrentHashMap<>();

public UploadMACGenerator(Upload upload) {
_secure_notify_lock = new Object();
Expand Down Expand Up @@ -122,36 +123,40 @@ public void run() {

Cipher cryptor = genCrypter("AES", "AES/CBC/NoPadding", _upload.getByte_file_key(), i32a2bin(mac_iv));

try ( BufferedInputStream is = new BufferedInputStream(new FileInputStream(_upload.getFile_name()))) {
int[] chunk_mac = new int[4];
byte[] byte_block = new byte[16];
byte[] chunk_bytes;

if (tot > 0) {
is.skip(tot);
}
try {
while (!_exit && !_upload.isStopped() && !_upload.getMain_panel().isExit()) {

int[] chunk_mac = new int[4];
byte[] byte_block = new byte[16];
int reads;

try {
while (!_exit && !_upload.isStopped() && !_upload.getMain_panel().isExit()) {
long chunk_offset = ChunkWriterManager.calculateChunkOffset(chunk_id, 1);

int reads;
long chunk_size = ChunkWriterManager.calculateChunkSize(chunk_id, _upload.getFile_size(), chunk_offset, 1);

long chunk_offset = ChunkWriterManager.calculateChunkOffset(chunk_id, 1);
ChunkWriterManager.checkChunkID(chunk_id, _upload.getFile_size(), chunk_offset);

long chunk_size = ChunkWriterManager.calculateChunkSize(chunk_id, _upload.getFile_size(), chunk_offset, 1);
while (!CHUNK_QUEUE.containsKey(chunk_offset)) {
MiscTools.pausar(1000);
}

ChunkWriterManager.checkChunkID(chunk_id, _upload.getFile_size(), chunk_offset);
try {

try {
chunk_mac[0] = file_iv[0];
chunk_mac[1] = file_iv[1];
chunk_mac[2] = file_iv[0];
chunk_mac[3] = file_iv[1];

chunk_mac[0] = file_iv[0];
chunk_mac[1] = file_iv[1];
chunk_mac[2] = file_iv[0];
chunk_mac[3] = file_iv[1];
long conta_chunk = 0L;

long conta_chunk = 0L;
try ( ByteArrayOutputStream baos = CHUNK_QUEUE.remove(chunk_offset)) {
chunk_bytes = baos.toByteArray();
}

while (conta_chunk < chunk_size && (reads = is.read(byte_block)) != -1) {
try ( ByteArrayInputStream bais = new ByteArrayInputStream(chunk_bytes)) {
while (conta_chunk < chunk_size && (reads = bais.read(byte_block)) != -1) {

if (reads < byte_block.length) {
for (int i = reads; i < byte_block.length; i++) {
Expand All @@ -172,45 +177,46 @@ public void run() {
tot += reads;

}
}

for (int i = 0; i < file_mac.length; i++) {
file_mac[i] ^= chunk_mac[i];
}
for (int i = 0; i < file_mac.length; i++) {
file_mac[i] ^= chunk_mac[i];
}

file_mac = bin2i32a(cryptor.doFinal(i32a2bin(file_mac)));
file_mac = bin2i32a(cryptor.doFinal(i32a2bin(file_mac)));

} catch (IOException | IllegalBlockSizeException | BadPaddingException ex) {
LOG.log(Level.SEVERE, ex.getMessage());
}
} catch (IllegalBlockSizeException | BadPaddingException ex) {
LOG.log(Level.SEVERE, ex.getMessage());
}

chunk_id++;
chunk_id++;

int new_cbc_per = (int) ((((double) tot) / _upload.getFile_size()) * 100);
int new_cbc_per = (int) ((((double) tot) / _upload.getFile_size()) * 100);

if (new_cbc_per != cbc_per) {
_upload.getView().updateCBC("CBC-MAC " + String.valueOf(new_cbc_per) + "%");
cbc_per = new_cbc_per;
}
if (new_cbc_per != cbc_per) {
_upload.getView().updateCBC("CBC-MAC " + String.valueOf(new_cbc_per) + "%");
cbc_per = new_cbc_per;
}

mac = (tot == _upload.getFile_size());
}

} catch (ChunkInvalidException e) {
mac = (tot == _upload.getFile_size());

mac = true;
}
} catch (ChunkInvalidException e) {

_upload.setTemp_mac_data(String.valueOf(tot) + "#" + String.valueOf(chunk_id) + "#" + Bin2BASE64(i32a2bin(file_mac)));
mac = true;
}

if (mac) {
_upload.setTemp_mac_data(String.valueOf(tot) + "#" + String.valueOf(chunk_id) + "#" + Bin2BASE64(i32a2bin(file_mac)));

int[] meta_mac = {file_mac[0] ^ file_mac[1], file_mac[2] ^ file_mac[3]};
if (mac) {

_upload.setFile_meta_mac(meta_mac);
int[] meta_mac = {file_mac[0] ^ file_mac[1], file_mac[2] ^ file_mac[3]};

LOG.log(Level.INFO, "{0} MAC GENERATOR {1} finished MAC CALCULATION. Waiting workers to finish uploading (if any)...", new Object[]{Thread.currentThread().getName(), getUpload().getFile_name()});
_upload.setFile_meta_mac(meta_mac);

LOG.log(Level.INFO, "{0} MAC GENERATOR {1} finished MAC CALCULATION. Waiting workers to finish uploading (if any)...", new Object[]{Thread.currentThread().getName(), getUpload().getFile_name()});

}
}

while (!_exit && !_upload.isStopped() && !_upload.getChunkworkers().isEmpty()) {
Expand Down
15 changes: 7 additions & 8 deletions src/main/java/com/tonikelope/megabasterd/UploadView.form
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,11 +106,13 @@
<Component id="slot_status_label" min="-2" max="-2" attributes="0"/>
</Group>
<EmptySpace min="-2" max="-2" attributes="0"/>
<Group type="103" groupAlignment="3" attributes="0">
<Component id="folder_link_button" alignment="3" min="-2" max="-2" attributes="0"/>
<Component id="file_link_button" alignment="3" min="-2" max="-2" attributes="0"/>
<Component id="file_size_label" alignment="3" min="-2" max="-2" attributes="0"/>
<Component id="open_browser_button" alignment="0" max="32767" attributes="0"/>
<Group type="103" groupAlignment="0" attributes="0">
<Component id="open_browser_button" alignment="0" max="-2" attributes="0"/>
<Group type="103" groupAlignment="3" attributes="0">
<Component id="folder_link_button" alignment="3" min="-2" max="-2" attributes="0"/>
<Component id="file_link_button" alignment="3" min="-2" max="-2" attributes="0"/>
<Component id="file_size_label" alignment="3" min="-2" max="-2" attributes="0"/>
</Group>
</Group>
<EmptySpace min="-2" max="-2" attributes="0"/>
<Component id="progress_pbar" min="-2" max="-2" attributes="0"/>
Expand Down Expand Up @@ -387,9 +389,6 @@
<Image iconType="3" name="/images/icons8-export-30.png"/>
</Property>
<Property name="text" type="java.lang.String" value="Open folder in browser"/>
<Property name="cursor" type="java.awt.Cursor" editor="org.netbeans.modules.form.editors2.CursorEditor">
<Color id="Cursor de Mano"/>
</Property>
<Property name="doubleBuffered" type="boolean" value="true"/>
</Properties>
<Events>
Expand Down
Loading

0 comments on commit ee5a5a7

Please sign in to comment.