diff --git a/src/freenet/crypt/Util.java b/src/freenet/crypt/Util.java index 5295b79f125..c239aaeb795 100644 --- a/src/freenet/crypt/Util.java +++ b/src/freenet/crypt/Util.java @@ -21,6 +21,7 @@ import java.util.Random; import freenet.crypt.ciphers.Rijndael; +import freenet.support.Fields; import freenet.support.HexUtil; import freenet.support.Loader; import freenet.support.Logger; @@ -394,4 +395,11 @@ public static void readFully(InputStream in, byte[] b, int off, int length) } } + public static double keyDigestAsNormalizedDouble(byte[] digest) { + long asLong = Math.abs(Fields.bytesToLong(digest)); + // Math.abs can actually return negative... + if(asLong == Long.MIN_VALUE) + asLong = Long.MAX_VALUE; + return ((double)asLong)/((double)Long.MAX_VALUE); + } } diff --git a/src/freenet/keys/Key.java b/src/freenet/keys/Key.java index 7a90853ec6a..18011111272 100644 --- a/src/freenet/keys/Key.java +++ b/src/freenet/keys/Key.java @@ -14,6 +14,7 @@ import freenet.crypt.CryptFormatException; import freenet.crypt.DSAPublicKey; import freenet.crypt.SHA256; +import freenet.crypt.Util; import freenet.io.WritableToDataOutputStream; import freenet.support.Fields; import freenet.support.LogThresholdCallback; @@ -134,12 +135,8 @@ public synchronized double toNormalizedDouble() { md.update((byte)TYPE); byte[] digest = md.digest(); SHA256.returnMessageDigest(md); md = null; - long asLong = Math.abs(Fields.bytesToLong(digest)); - // Math.abs can actually return negative... - if(asLong == Long.MIN_VALUE) - asLong = Long.MAX_VALUE; - cachedNormalizedDouble = ((double)asLong)/((double)Long.MAX_VALUE); - return cachedNormalizedDouble; + cachedNormalizedDouble = Util.keyDigestAsNormalizedDouble(digest); + return cachedNormalizedDouble; } /** diff --git a/src/freenet/node/LocationManager.java b/src/freenet/node/LocationManager.java index d25eb25372a..2684e1d67f2 100644 --- a/src/freenet/node/LocationManager.java +++ b/src/freenet/node/LocationManager.java @@ -9,12 +9,19 @@ import java.io.BufferedWriter; import java.io.File; +import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; +import java.net.MalformedURLException; +import java.nio.file.Files; import java.security.MessageDigest; import java.text.DateFormat; +import java.time.Duration; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.Deque; import java.util.Hashtable; @@ -24,19 +31,32 @@ import java.util.Set; import java.util.TimeZone; +import freenet.client.FetchException; +import freenet.client.FetchResult; +import freenet.client.HighLevelSimpleClient; +import freenet.client.InsertBlock; +import freenet.client.InsertException; import freenet.crypt.RandomSource; import freenet.crypt.SHA256; +import freenet.crypt.Util; import freenet.io.comm.ByteCounter; import freenet.io.comm.DMT; import freenet.io.comm.DisconnectedException; import freenet.io.comm.Message; import freenet.io.comm.MessageFilter; import freenet.io.comm.NotConnectedException; +import freenet.keys.ClientCHK; +import freenet.keys.ClientKSK; +import freenet.keys.ClientKey; +import freenet.keys.ClientSSK; +import freenet.keys.FreenetURI; +import freenet.support.Base64; import freenet.support.Fields; import freenet.support.Logger; +import freenet.support.Logger.LogLevel; import freenet.support.ShortBuffer; import freenet.support.TimeSortedHashtable; -import freenet.support.Logger.LogLevel; +import freenet.support.io.ArrayBucket; import freenet.support.io.Closer; import freenet.support.math.BootstrappingDecayingRunningAverage; @@ -48,6 +68,8 @@ */ public class LocationManager implements ByteCounter { + public static final String FOIL_PITCH_BLACK_ATTACK_PREFIX = "mitigate-pitch-black-attack-"; + public class MyCallback extends SendMessageOnErrorCallback { RecentlyForwardedItem item; @@ -146,8 +168,9 @@ public synchronized void updateLocationChangeSession(double newLoc) { * we are not locked. */ public void start() { - if(node.enableSwapping) - node.getTicker().queueTimedJob(sender, STARTUP_DELAY); + if(node.enableSwapping) { + node.getTicker().queueTimedJob(sender, STARTUP_DELAY); + } node.ticker.queueTimedJob(new Runnable() { @Override @@ -161,6 +184,205 @@ public void run() { } }, SECONDS.toMillis(10)); + // Insert key to probe whether its part of the keyspace is operational. If it is not, switch location to it. + node.ticker.queueTimedJob(new Runnable() { + + @Override + public void run() { + node.ticker.queueTimedJob(this, DAYS.toMillis(1)); + LocalDateTime now = LocalDateTime.now(); + String isoDateStringToday = DateTimeFormatter.ISO_DATE + .format(now); + String isoDateStringYesterday = DateTimeFormatter.ISO_DATE + .format(now.minus(Duration.ofDays(1))); + File[] previousInsertFromToday = node.userDir().dir() + .listFiles((file, name) -> name.startsWith(FOIL_PITCH_BLACK_ATTACK_PREFIX + + isoDateStringToday)); + HighLevelSimpleClient highLevelSimpleClient = node.clientCore.makeClient( + RequestStarter.INTERACTIVE_PRIORITY_CLASS, + true, + false); + + if (previousInsertFromToday != null + && previousInsertFromToday.length == 0) { + byte[] randomContentForKSK = new byte[20]; + node.secureRandom.nextBytes(randomContentForKSK); + String randomPart = Base64.encode(randomContentForKSK); + String nameForInsert = + FOIL_PITCH_BLACK_ATTACK_PREFIX + isoDateStringToday + "-" + randomPart; + tryToInsertPitchBlackCheck(highLevelSimpleClient, nameForInsert); + } + + File[] foilPitchBlackStatusFiles = node.userDir().dir() + .listFiles((file, name) -> name.startsWith(FOIL_PITCH_BLACK_ATTACK_PREFIX)); + if (foilPitchBlackStatusFiles != null) { + File[] successfulInsertFromYesterday = Arrays.stream(foilPitchBlackStatusFiles) + .filter(file -> file.getName().contains(isoDateStringYesterday)) + .toArray(File[]::new); + for (File f : successfulInsertFromYesterday) { + tryToRequestPitchBlackCheckFromYesterday( + highLevelSimpleClient, + successfulInsertFromYesterday[0] + ); + // cleanup file, regardless of success + if (!f.delete()) { + f.deleteOnExit(); + } + } + // delete files from more than one day ago + File[] leftoverFiles = Arrays.stream(foilPitchBlackStatusFiles) + .filter(file -> !file.getName().contains(isoDateStringToday)) + .toArray(File[]::new); + for (File f : leftoverFiles) { + if (!f.delete()) { + f.deleteOnExit(); + } + } + } + } + }, SECONDS.toMillis(60)); + } + + private void tryToRequestPitchBlackCheckFromYesterday( + HighLevelSimpleClient highLevelSimpleClient, + File insertInfoFromYesterday) { + ClientKSK insertFromYesterday = ClientKSK.create(insertInfoFromYesterday.getName()); + byte[] expectedContent; + try { + expectedContent = Files.readAllBytes(insertInfoFromYesterday.toPath()); + } catch (FileNotFoundException e) { + Logger.warning( + e, + "Could not read from insert info file from yesterday because the file was not found: " + + insertInfoFromYesterday.getName()); + return; + } catch (IOException e) { + Logger.warning( + e, + "Could not read from insert info file from yesterday: " + + insertInfoFromYesterday.getName()); + return; + } + // check the SSK + FetchResult sskFetchResult = null; + try { + sskFetchResult = highLevelSimpleClient.fetch(insertFromYesterday.getURI()); + if (!Arrays.equals(expectedContent, sskFetchResult.asByteArray())) { + switchLocationToDefendAgainstPitchBlackAttack(insertFromYesterday); + } + } catch (FetchException e) { + if (isRequestExceptionBecauseUriIsNotAvailable(e)) { + switchLocationToDefendAgainstPitchBlackAttack(insertFromYesterday); + } + return; + } catch (IOException e) { + Logger.warning( + e, + "Could not convert fetched data into byteArray. fetch: " + + sskFetchResult); + return; + } + // check the CHK + ArrayBucket randomBucketToInsert = new ArrayBucket(expectedContent); + InsertBlock chkInsertBlock = new InsertBlock( + randomBucketToInsert, + null, + FreenetURI.EMPTY_CHK_URI); + FreenetURI calculatedChkUri; + try { + calculatedChkUri = highLevelSimpleClient.insert(chkInsertBlock, true, null); + } catch (InsertException e) { + Logger.error( + e, + "Could not create CHK for expected content."); + return; + } + try { + highLevelSimpleClient.fetch(calculatedChkUri); + } catch (FetchException e) { + if (isRequestExceptionBecauseUriIsNotAvailable(e)) { + try { + switchLocationToDefendAgainstPitchBlackAttack(new ClientCHK(calculatedChkUri)); + } catch (MalformedURLException exception) { + Logger.error( + exception, + "Could not create ClientCHK from CHKUri for calculated CHK URI:" + + calculatedChkUri); + return; + } + } + } + + } + + private void switchLocationToDefendAgainstPitchBlackAttack(ClientKey insertFromYesterday) { + double probedLocationFromYesterday = insertFromYesterday + .getNodeKey() + .toNormalizedDouble(); + if (insertFromYesterday instanceof ClientSSK) { + // decide between SSK and pubkey at random, because they always break together. + if (node.fastWeakRandom.nextBoolean()) { + probedLocationFromYesterday = Util.keyDigestAsNormalizedDouble( + ((ClientSSK) insertFromYesterday).getPubKey().getRoutingKey()); + } + } + Logger.warning( + this, + "could not fetch the insert from yesterday: " + + insertFromYesterday.getURI().toString() + + ", assuming we are under attack: switching location to failed location: " + + probedLocationFromYesterday); + setLocation(probedLocationFromYesterday); + } + + private void tryToInsertPitchBlackCheck( + HighLevelSimpleClient highLevelSimpleClient, + String nameForInsert) { + // create some random data of up to 1021 bytes to insert to the KSK + byte[] contentLengthSource = new byte[2]; + node.fastWeakRandom.nextBytes(contentLengthSource); + // bytes are -127 to 128, + // so this gives us 253 to 1021 bytes of size + int contentLength = (5 * 127) + + (3 * contentLengthSource[0]) + + contentLengthSource[1] / 64; // -1 to 2 + byte[] randomContentToInsert = new byte[contentLength]; + node.fastWeakRandom.nextBytes(randomContentToInsert); + ArrayBucket randomBucketToInsert = new ArrayBucket(randomContentToInsert); + // create the KSK + ClientKSK insertForToday = (ClientKSK.create(nameForInsert)); + InsertBlock kskInsertBlock = new InsertBlock( + randomBucketToInsert, + null, + insertForToday.getInsertURI()); + // create the CHK + InsertBlock chkInsertBlock = new InsertBlock( + randomBucketToInsert, + null, + FreenetURI.EMPTY_CHK_URI); + try { + highLevelSimpleClient.insert(kskInsertBlock, false, null); + highLevelSimpleClient.insert(chkInsertBlock, false, null); + // create a file to check on the next run tomorrow + File succeededInsertFile = node.userDir().file(nameForInsert); + try (FileOutputStream fileOutputStream = new FileOutputStream(succeededInsertFile)) { + fileOutputStream.write(randomContentToInsert); + } catch (IOException e) { + Logger.error( + e, + "Could not write successful insert content to file: " + nameForInsert); + } + } catch (InsertException e) { + Logger.error( + this, + "could not insert pitch black detection data to KSK for today: " + + insertForToday.getURI().toString() + + ", trying again tomorrow."); + } + } + + private static boolean isRequestExceptionBecauseUriIsNotAvailable(FetchException fetchException) { + return FetchException.FetchExceptionMode.DATA_NOT_FOUND.equals(fetchException.getMode()); } /** @@ -171,7 +393,7 @@ public class SwapRequestSender implements Runnable { @Override public void run() { - freenet.support.Logger.OSThread.logPID(this); + freenet.support.Logger.OSThread.logPID(this); Thread.currentThread().setName("SwapRequestSender"); while(true) { try { @@ -263,7 +485,7 @@ public boolean swappingDisabled() { // Swapping on opennet nodes, even hybrid nodes, causes significant and unnecessary location churn. // Simulations show significantly improved performance if all opennet enabled nodes don't participate in swapping. // FIXME: Investigate the possibility of enabling swapping on hybrid nodes with mostly darknet peers (more simulation needed). - // FIXME: Hybrid nodes with all darknet peeers who haven't upgraded to HIGH. + // FIXME: Hybrid nodes with all darknet peers who haven't upgraded to HIGH. // Probably we should have a useralert for this to get the user to do the right thing ... but we could auto-detect // it and start swapping... however, we should not start swapping just because we temporarily have no opennet peers // on startup. diff --git a/src/freenet/node/simulator/RealNodeRoutingTest.java b/src/freenet/node/simulator/RealNodeRoutingTest.java index 38e684c6e7e..46a33bd8d75 100644 --- a/src/freenet/node/simulator/RealNodeRoutingTest.java +++ b/src/freenet/node/simulator/RealNodeRoutingTest.java @@ -167,7 +167,7 @@ static void waitForPingAverage(double accuracy, Node[] nodes, RandomSource rando Logger.error(RealNodeRoutingTest.class, "Caught " + t, t); } } - System.err.println("Average path length for successful requests: "+totalHopsTaken/successes); + System.err.println("Average path length for successful requests: "+((double)totalHopsTaken)/successes); if(pings > 10 && avg.currentValue() > accuracy && ((double) successes / ((double) (failures + successes)) > accuracy)) { System.err.println(); System.err.println("Reached " + (accuracy * 100) + "% accuracy.");