Merge remote-tracking branch 'origin/fix-pitch-black' into next

src/freenet/crypt/Util.java

@@ -21,6 +21,7 @@
 import java.util.Random;
 
 import freenet.crypt.ciphers.Rijndael;
+import freenet.support.Fields;
 import freenet.support.HexUtil;
 import freenet.support.Loader;
 import freenet.support.Logger;
@@ -394,4 +395,11 @@ public static void readFully(InputStream in, byte[] b, int off, int length)
 		}
 	}
 
+	public static double keyDigestAsNormalizedDouble(byte[] digest) {
+		long asLong = Math.abs(Fields.bytesToLong(digest));
+		// Math.abs can actually return negative...
+		if(asLong == Long.MIN_VALUE)
+				asLong = Long.MAX_VALUE;
+		return ((double)asLong)/((double)Long.MAX_VALUE);
+	}
 }

src/freenet/keys/Key.java

@@ -14,6 +14,7 @@
 import freenet.crypt.CryptFormatException;
 import freenet.crypt.DSAPublicKey;
 import freenet.crypt.SHA256;
+import freenet.crypt.Util;
 import freenet.io.WritableToDataOutputStream;
 import freenet.support.Fields;
 import freenet.support.LogThresholdCallback;
@@ -134,12 +135,8 @@ public synchronized double toNormalizedDouble() {
         md.update((byte)TYPE);
         byte[] digest = md.digest();
         SHA256.returnMessageDigest(md); md = null;
-        long asLong = Math.abs(Fields.bytesToLong(digest));
-        // Math.abs can actually return negative...
-        if(asLong == Long.MIN_VALUE)
-            asLong = Long.MAX_VALUE;
-        cachedNormalizedDouble = ((double)asLong)/((double)Long.MAX_VALUE);
-        return cachedNormalizedDouble;
+			cachedNormalizedDouble = Util.keyDigestAsNormalizedDouble(digest);
+			return cachedNormalizedDouble;
     }
 
 	/**

src/freenet/node/LocationManager.java

@@ -9,12 +9,19 @@
 
 import java.io.BufferedWriter;
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.OutputStreamWriter;
+import java.net.MalformedURLException;
+import java.nio.file.Files;
 import java.security.MessageDigest;
 import java.text.DateFormat;
+import java.time.Duration;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Date;
 import java.util.Deque;
 import java.util.Hashtable;
@@ -24,19 +31,32 @@
 import java.util.Set;
 import java.util.TimeZone;
 
+import freenet.client.FetchException;
+import freenet.client.FetchResult;
+import freenet.client.HighLevelSimpleClient;
+import freenet.client.InsertBlock;
+import freenet.client.InsertException;
 import freenet.crypt.RandomSource;
 import freenet.crypt.SHA256;
+import freenet.crypt.Util;
 import freenet.io.comm.ByteCounter;
 import freenet.io.comm.DMT;
 import freenet.io.comm.DisconnectedException;
 import freenet.io.comm.Message;
 import freenet.io.comm.MessageFilter;
 import freenet.io.comm.NotConnectedException;
+import freenet.keys.ClientCHK;
+import freenet.keys.ClientKSK;
+import freenet.keys.ClientKey;
+import freenet.keys.ClientSSK;
+import freenet.keys.FreenetURI;
+import freenet.support.Base64;
 import freenet.support.Fields;
 import freenet.support.Logger;
+import freenet.support.Logger.LogLevel;
 import freenet.support.ShortBuffer;
 import freenet.support.TimeSortedHashtable;
-import freenet.support.Logger.LogLevel;
+import freenet.support.io.ArrayBucket;
 import freenet.support.io.Closer;
 import freenet.support.math.BootstrappingDecayingRunningAverage;
 
@@ -48,6 +68,8 @@
  */
 public class LocationManager implements ByteCounter {
 
+    public static final String FOIL_PITCH_BLACK_ATTACK_PREFIX = "mitigate-pitch-black-attack-";
+
     public class MyCallback extends SendMessageOnErrorCallback {
 
         RecentlyForwardedItem item;
@@ -146,8 +168,9 @@ public synchronized void updateLocationChangeSession(double newLoc) {
      * we are not locked.
      */
     public void start() {
-    	if(node.enableSwapping)
-    		node.getTicker().queueTimedJob(sender, STARTUP_DELAY);
+    	if(node.enableSwapping) {
+          node.getTicker().queueTimedJob(sender, STARTUP_DELAY);
+      }
 		node.ticker.queueTimedJob(new Runnable() {
 
 			@Override
@@ -161,6 +184,205 @@ public void run() {
 			}
 
 		}, SECONDS.toMillis(10));
+    	// Insert key to probe whether its part of the keyspace is operational. If it is not, switch location to it.
+        node.ticker.queueTimedJob(new Runnable() {
+
+            @Override
+            public void run() {
+                node.ticker.queueTimedJob(this, DAYS.toMillis(1));
+                LocalDateTime now = LocalDateTime.now();
+                String isoDateStringToday = DateTimeFormatter.ISO_DATE
+                    .format(now);
+                String isoDateStringYesterday = DateTimeFormatter.ISO_DATE
+                    .format(now.minus(Duration.ofDays(1)));
+                File[] previousInsertFromToday = node.userDir().dir()
+                    .listFiles((file, name) -> name.startsWith(FOIL_PITCH_BLACK_ATTACK_PREFIX
+                        + isoDateStringToday));
+                HighLevelSimpleClient highLevelSimpleClient = node.clientCore.makeClient(
+                    RequestStarter.INTERACTIVE_PRIORITY_CLASS,
+                    true,
+                    false);
+
+                if (previousInsertFromToday != null
+                    && previousInsertFromToday.length == 0) {
+                    byte[] randomContentForKSK = new byte[20];
+                    node.secureRandom.nextBytes(randomContentForKSK);
+                    String randomPart = Base64.encode(randomContentForKSK);
+                    String nameForInsert =
+                        FOIL_PITCH_BLACK_ATTACK_PREFIX + isoDateStringToday + "-" + randomPart;
+                    tryToInsertPitchBlackCheck(highLevelSimpleClient, nameForInsert);
+                }
+
+                File[] foilPitchBlackStatusFiles = node.userDir().dir()
+                    .listFiles((file, name) -> name.startsWith(FOIL_PITCH_BLACK_ATTACK_PREFIX));
+                if (foilPitchBlackStatusFiles != null) {
+                    File[] successfulInsertFromYesterday = Arrays.stream(foilPitchBlackStatusFiles)
+                        .filter(file -> file.getName().contains(isoDateStringYesterday))
+                        .toArray(File[]::new);
+                    for (File f : successfulInsertFromYesterday) {
+                        tryToRequestPitchBlackCheckFromYesterday(
+                            highLevelSimpleClient,
+                            successfulInsertFromYesterday[0]
+                        );
+                        // cleanup file, regardless of success
+                        if (!f.delete()) {
+                            f.deleteOnExit();
+                        }
+                    }
+                    // delete files from more than one day ago
+                    File[] leftoverFiles = Arrays.stream(foilPitchBlackStatusFiles)
+                        .filter(file -> !file.getName().contains(isoDateStringToday))
+                        .toArray(File[]::new);
+                    for (File f : leftoverFiles) {
+                        if (!f.delete()) {
+                            f.deleteOnExit();
+                        }
+                    }
+                }
+            }
+        }, SECONDS.toMillis(60));
+    }
+
+    private void tryToRequestPitchBlackCheckFromYesterday(
+        HighLevelSimpleClient highLevelSimpleClient,
+        File insertInfoFromYesterday) {
+        ClientKSK insertFromYesterday = ClientKSK.create(insertInfoFromYesterday.getName());
+        byte[] expectedContent;
+        try {
+            expectedContent = Files.readAllBytes(insertInfoFromYesterday.toPath());
+        } catch (FileNotFoundException e) {
+            Logger.warning(
+                e,
+                "Could not read from insert info file from yesterday because the file was not found: "
+                    + insertInfoFromYesterday.getName());
+            return;
+        } catch (IOException e) {
+            Logger.warning(
+                e,
+                "Could not read from insert info file from yesterday: "
+                    + insertInfoFromYesterday.getName());
+            return;
+        }
+        // check the SSK
+        FetchResult sskFetchResult = null;
+        try {
+            sskFetchResult = highLevelSimpleClient.fetch(insertFromYesterday.getURI());
+            if (!Arrays.equals(expectedContent, sskFetchResult.asByteArray())) {
+                switchLocationToDefendAgainstPitchBlackAttack(insertFromYesterday);
+            }
+        } catch (FetchException e) {
+            if (isRequestExceptionBecauseUriIsNotAvailable(e)) {
+                switchLocationToDefendAgainstPitchBlackAttack(insertFromYesterday);
+            }
+            return;
+        } catch (IOException e) {
+            Logger.warning(
+                e,
+                "Could not convert fetched data into byteArray. fetch: "
+                    + sskFetchResult);
+            return;
+        }
+        // check the CHK
+        ArrayBucket randomBucketToInsert = new ArrayBucket(expectedContent);
+        InsertBlock chkInsertBlock = new InsertBlock(
+            randomBucketToInsert,
+            null,
+            FreenetURI.EMPTY_CHK_URI);
+        FreenetURI calculatedChkUri;
+        try {
+            calculatedChkUri = highLevelSimpleClient.insert(chkInsertBlock, true, null);
+        } catch (InsertException e) {
+            Logger.error(
+                e,
+                "Could not create CHK for expected content.");
+            return;
+        }
+        try {
+            highLevelSimpleClient.fetch(calculatedChkUri);
+        } catch (FetchException e) {
+            if (isRequestExceptionBecauseUriIsNotAvailable(e)) {
+                try {
+                    switchLocationToDefendAgainstPitchBlackAttack(new ClientCHK(calculatedChkUri));
+                } catch (MalformedURLException exception) {
+                    Logger.error(
+                        exception,
+                        "Could not create ClientCHK from CHKUri for calculated CHK URI:"
+                            + calculatedChkUri);
+                    return;
+                }
+            }
+        }
+
+    }
+
+    private void switchLocationToDefendAgainstPitchBlackAttack(ClientKey insertFromYesterday) {
+        double probedLocationFromYesterday = insertFromYesterday
+            .getNodeKey()
+            .toNormalizedDouble();
+        if (insertFromYesterday instanceof ClientSSK) {
+            // decide between SSK and pubkey at random, because they always break together.
+            if (node.fastWeakRandom.nextBoolean()) {
+                probedLocationFromYesterday = Util.keyDigestAsNormalizedDouble(
+                    ((ClientSSK) insertFromYesterday).getPubKey().getRoutingKey());
+            }
+        }
+        Logger.warning(
+            this,
+            "could not fetch the insert from yesterday: "
+                + insertFromYesterday.getURI().toString()
+                + ", assuming we are under attack: switching location to failed location: "
+                + probedLocationFromYesterday);
+        setLocation(probedLocationFromYesterday);
+    }
+
+    private void tryToInsertPitchBlackCheck(
+        HighLevelSimpleClient highLevelSimpleClient,
+        String nameForInsert) {
+        // create some random data of up to 1021 bytes to insert to the KSK
+        byte[] contentLengthSource = new byte[2];
+        node.fastWeakRandom.nextBytes(contentLengthSource);
+        // bytes are -127 to 128,
+        // so this gives us 253 to 1021 bytes of size
+        int contentLength = (5 * 127)
+            + (3 * contentLengthSource[0])
+            + contentLengthSource[1] / 64; // -1 to 2
+        byte[] randomContentToInsert = new byte[contentLength];
+        node.fastWeakRandom.nextBytes(randomContentToInsert);
+        ArrayBucket randomBucketToInsert = new ArrayBucket(randomContentToInsert);
+        // create the KSK
+        ClientKSK insertForToday = (ClientKSK.create(nameForInsert));
+        InsertBlock kskInsertBlock = new InsertBlock(
+            randomBucketToInsert,
+            null,
+            insertForToday.getInsertURI());
+        // create the CHK
+        InsertBlock chkInsertBlock = new InsertBlock(
+            randomBucketToInsert,
+            null,
+            FreenetURI.EMPTY_CHK_URI);
+        try {
+            highLevelSimpleClient.insert(kskInsertBlock, false, null);
+            highLevelSimpleClient.insert(chkInsertBlock, false, null);
+            // create a file to check on the next run tomorrow
+            File succeededInsertFile = node.userDir().file(nameForInsert);
+            try (FileOutputStream fileOutputStream = new FileOutputStream(succeededInsertFile)) {
+                fileOutputStream.write(randomContentToInsert);
+            } catch (IOException e) {
+                Logger.error(
+                    e,
+                    "Could not write successful insert content to file: " + nameForInsert);
+            }
+        } catch (InsertException e) {
+            Logger.error(
+                this,
+                "could not insert pitch black detection data to KSK for today: "
+                    + insertForToday.getURI().toString()
+                    + ", trying again tomorrow.");
+        }
+    }
+
+    private static boolean isRequestExceptionBecauseUriIsNotAvailable(FetchException fetchException) {
+        return FetchException.FetchExceptionMode.DATA_NOT_FOUND.equals(fetchException.getMode());
     }
 
     /**
@@ -171,7 +393,7 @@ public class SwapRequestSender implements Runnable {
 
         @Override
         public void run() {
-		    freenet.support.Logger.OSThread.logPID(this);
+            freenet.support.Logger.OSThread.logPID(this);
 		    Thread.currentThread().setName("SwapRequestSender");
             while(true) {
                 try {
@@ -263,7 +485,7 @@ public boolean swappingDisabled() {
     	// Swapping on opennet nodes, even hybrid nodes, causes significant and unnecessary location churn.
     	// Simulations show significantly improved performance if all opennet enabled nodes don't participate in swapping.
     	// FIXME: Investigate the possibility of enabling swapping on hybrid nodes with mostly darknet peers (more simulation needed).
-    	// FIXME: Hybrid nodes with all darknet peeers who haven't upgraded to HIGH.
+    	// FIXME: Hybrid nodes with all darknet peers who haven't upgraded to HIGH.
     	// Probably we should have a useralert for this to get the user to do the right thing ... but we could auto-detect
     	// it and start swapping... however, we should not start swapping just because we temporarily have no opennet peers
     	// on startup.

src/freenet/node/simulator/RealNodeRoutingTest.java

@@ -167,7 +167,7 @@ static void waitForPingAverage(double accuracy, Node[] nodes, RandomSource rando
 					Logger.error(RealNodeRoutingTest.class, "Caught " + t, t);
 				}
 			}
-			System.err.println("Average path length for successful requests: "+totalHopsTaken/successes);
+			System.err.println("Average path length for successful requests: "+((double)totalHopsTaken)/successes);
 			if(pings > 10 && avg.currentValue() > accuracy && ((double) successes / ((double) (failures + successes)) > accuracy)) {
 				System.err.println();
 				System.err.println("Reached " + (accuracy * 100) + "% accuracy.");