From 152d2d44907cd4fac0ed96170db98ac465442637 Mon Sep 17 00:00:00 2001 From: Chayim Date: Wed, 17 Jan 2024 14:38:31 +0200 Subject: [PATCH 01/10] Running doctests also on emb-examples (#3693) Given how these examples are generated, we intended to run and validate CI, ongoingly on emb-examples as well. As a result, this PR ensures that direct commits there, will also run them. --- .github/workflows/doctests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/doctests.yml b/.github/workflows/doctests.yml index daa8858b89..0062c2da24 100644 --- a/.github/workflows/doctests.yml +++ b/.github/workflows/doctests.yml @@ -4,6 +4,8 @@ on: push: tags-ignore: - '*' + branches: + - 'emb-examples' pull_request: workflow_dispatch: From 64b5aac03bc26a1278796718a49c264d0005164f Mon Sep 17 00:00:00 2001 From: ren ran <35724820+stillerrr@users.noreply.github.com> Date: Sat, 27 Jan 2024 02:07:18 +0800 Subject: [PATCH 02/10] Avoid NPE in MultiNodePipelineBase (#3697) Should get connection first and then create new pipeline queue, otherwise it would cause NPE when timeout for getting connection and call sync() method. If there is timeout for getting connection, the pipeline queue would be create for node, but connections map has no connection for this node. Once executing sync() method, it would throw NPE on connection.getMany() (line 112, connection is null) --- .../java/redis/clients/jedis/MultiNodePipelineBase.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/redis/clients/jedis/MultiNodePipelineBase.java b/src/main/java/redis/clients/jedis/MultiNodePipelineBase.java index a01ff4d1bb..13f2730ab4 100644 --- a/src/main/java/redis/clients/jedis/MultiNodePipelineBase.java +++ b/src/main/java/redis/clients/jedis/MultiNodePipelineBase.java @@ -63,9 +63,6 @@ protected final Response appendCommand(CommandObject commandObject) { queue = pipelinedResponses.get(nodeKey); connection = connections.get(nodeKey); } else { - pipelinedResponses.putIfAbsent(nodeKey, new LinkedList<>()); - queue = pipelinedResponses.get(nodeKey); - Connection newOne = getConnection(nodeKey); connections.putIfAbsent(nodeKey, newOne); connection = connections.get(nodeKey); @@ -73,6 +70,9 @@ protected final Response appendCommand(CommandObject commandObject) { log.debug("Duplicate connection to {}, closing it.", nodeKey); IOUtils.closeQuietly(newOne); } + + pipelinedResponses.putIfAbsent(nodeKey, new LinkedList<>()); + queue = pipelinedResponses.get(nodeKey); } connection.sendCommand(commandObject.getArguments()); From e4a0c783b11f1ddda28e605e2a70f6e016eff514 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Feb 2024 11:44:29 +0600 Subject: [PATCH 03/10] Bump com.kohlschutter.junixsocket:junixsocket-core from 2.8.3 to 2.9.0 (#3724) Bumps [com.kohlschutter.junixsocket:junixsocket-core](https://github.com/kohlschutter/junixsocket) from 2.8.3 to 2.9.0. - [Release notes](https://github.com/kohlschutter/junixsocket/releases) - [Commits](https://github.com/kohlschutter/junixsocket/compare/junixsocket-2.8.3...junixsocket-2.9.0) --- updated-dependencies: - dependency-name: com.kohlschutter.junixsocket:junixsocket-core dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c9fd15d2cb..d52ea40e26 100644 --- a/pom.xml +++ b/pom.xml @@ -79,7 +79,7 @@ com.kohlschutter.junixsocket junixsocket-core - 2.8.3 + 2.9.0 pom test From f86e1badb56f8f6e36b565259a61073491322aad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Feb 2024 12:13:39 +0600 Subject: [PATCH 04/10] Bump org.json:json from 20231013 to 20240205 (#3706) Bumps [org.json:json](https://github.com/douglascrockford/JSON-java) from 20231013 to 20240205. - [Release notes](https://github.com/douglascrockford/JSON-java/releases) - [Changelog](https://github.com/stleary/JSON-java/blob/master/docs/RELEASES.md) - [Commits](https://github.com/douglascrockford/JSON-java/commits) --- updated-dependencies: - dependency-name: org.json:json dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d52ea40e26..a90614bd36 100644 --- a/pom.xml +++ b/pom.xml @@ -67,7 +67,7 @@ org.json json - 20231013 + 20240205 com.google.code.gson From 65d431f549502364bacbc8c25c85e973278c437a Mon Sep 17 00:00:00 2001 From: Charles Chang Date: Mon, 19 Feb 2024 19:01:06 +0800 Subject: [PATCH 05/10] Check for thread interrupt in subscribe process of PubSub (#3726) - adding thread interrupted check in `JedisShardedPubSubBase::process` - also in `JedisPubSubBase::process` Fixes #3725 --------- Co-authored-by: charles.chang Co-authored-by: M Sazzadul Hoque <7600764+sazzad16@users.noreply.github.com> --- .../redis/clients/jedis/JedisPubSubBase.java | 2 +- .../clients/jedis/JedisShardedPubSubBase.java | 2 +- .../clients/jedis/JedisPubSubBaseTest.java | 59 +++++++++++++++++++ .../jedis/JedisShardedPubSubBaseTest.java | 56 ++++++++++++++++++ 4 files changed, 117 insertions(+), 2 deletions(-) create mode 100644 src/test/java/redis/clients/jedis/JedisPubSubBaseTest.java create mode 100644 src/test/java/redis/clients/jedis/JedisShardedPubSubBaseTest.java diff --git a/src/main/java/redis/clients/jedis/JedisPubSubBase.java b/src/main/java/redis/clients/jedis/JedisPubSubBase.java index 7092680e33..552310e4de 100644 --- a/src/main/java/redis/clients/jedis/JedisPubSubBase.java +++ b/src/main/java/redis/clients/jedis/JedisPubSubBase.java @@ -172,7 +172,7 @@ private void process() { } else { throw new JedisException("Unknown message type: " + reply); } - } while (isSubscribed()); + } while (!Thread.currentThread().isInterrupted() && isSubscribed()); // /* Invalidate instance since this thread is no longer listening */ // this.client = null; diff --git a/src/main/java/redis/clients/jedis/JedisShardedPubSubBase.java b/src/main/java/redis/clients/jedis/JedisShardedPubSubBase.java index f0a251f61f..2b2ce944fe 100644 --- a/src/main/java/redis/clients/jedis/JedisShardedPubSubBase.java +++ b/src/main/java/redis/clients/jedis/JedisShardedPubSubBase.java @@ -99,7 +99,7 @@ private void process() { } else { throw new JedisException("Unknown message type: " + reply); } - } while (isSubscribed()); + } while (!Thread.currentThread().isInterrupted() && isSubscribed()); // /* Invalidate instance since this thread is no longer listening */ // this.client = null; diff --git a/src/test/java/redis/clients/jedis/JedisPubSubBaseTest.java b/src/test/java/redis/clients/jedis/JedisPubSubBaseTest.java new file mode 100644 index 0000000000..a7910bd6a3 --- /dev/null +++ b/src/test/java/redis/clients/jedis/JedisPubSubBaseTest.java @@ -0,0 +1,59 @@ +package redis.clients.jedis; + +import junit.framework.TestCase; +import redis.clients.jedis.util.SafeEncoder; + +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static redis.clients.jedis.Protocol.ResponseKeyword.MESSAGE; +import static redis.clients.jedis.Protocol.ResponseKeyword.SUBSCRIBE; + +public class JedisPubSubBaseTest extends TestCase { + + public void testProceed_givenThreadInterrupt_exitLoop() throws InterruptedException { + // setup + final JedisPubSubBase pubSub = new JedisPubSubBase() { + + @Override + public void onMessage(String channel, String message) { + fail("this should not happen when thread is interrupted"); + } + + @Override + protected String encode(byte[] raw) { + return SafeEncoder.encode(raw); + } + }; + + final Connection mockConnection = mock(Connection.class); + final List mockSubscribe = Arrays.asList( + SUBSCRIBE.getRaw(), "channel".getBytes(), 1L + ); + final List mockResponse = Arrays.asList( + MESSAGE.getRaw(), "channel".getBytes(), "message".getBytes() + ); + + when(mockConnection.getUnflushedObject()). + + thenReturn(mockSubscribe, mockResponse); + + + final CountDownLatch countDownLatch = new CountDownLatch(1); + // action + final Thread thread = new Thread(() -> { + Thread.currentThread().interrupt(); + pubSub.proceed(mockConnection, "channel"); + + countDownLatch.countDown(); + }); + thread.start(); + + assertTrue(countDownLatch.await(10, TimeUnit.MILLISECONDS)); + + } +} diff --git a/src/test/java/redis/clients/jedis/JedisShardedPubSubBaseTest.java b/src/test/java/redis/clients/jedis/JedisShardedPubSubBaseTest.java new file mode 100644 index 0000000000..fb1ecdd87a --- /dev/null +++ b/src/test/java/redis/clients/jedis/JedisShardedPubSubBaseTest.java @@ -0,0 +1,56 @@ +package redis.clients.jedis; + +import junit.framework.TestCase; + +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static redis.clients.jedis.Protocol.ResponseKeyword.SMESSAGE; +import static redis.clients.jedis.Protocol.ResponseKeyword.SSUBSCRIBE; + +public class JedisShardedPubSubBaseTest extends TestCase { + + public void testProceed_givenThreadInterrupt_exitLoop() throws InterruptedException { + // setup + final JedisShardedPubSubBase pubSub = new JedisShardedPubSubBase() { + + @Override + public void onSMessage(String channel, String message) { + fail("this should not happen when thread is interrupted"); + } + + @Override + protected String encode(byte[] raw) { + return new String(raw); + } + + }; + + final Connection mockConnection = mock(Connection.class); + final List mockSubscribe = Arrays.asList( + SSUBSCRIBE.getRaw(), "channel".getBytes(), 1L + ); + final List mockResponse = Arrays.asList( + SMESSAGE.getRaw(), "channel".getBytes(), "message".getBytes() + ); + when(mockConnection.getUnflushedObject()).thenReturn(mockSubscribe, mockResponse); + + + final CountDownLatch countDownLatch = new CountDownLatch(1); + // action + final Thread thread = new Thread(() -> { + Thread.currentThread().interrupt(); + pubSub.proceed(mockConnection, "channel"); + + countDownLatch.countDown(); + }); + thread.start(); + + assertTrue(countDownLatch.await(10, TimeUnit.MILLISECONDS)); + + } +} \ No newline at end of file From 6f3db461932fe6751175d779ed0133f9eea3f46f Mon Sep 17 00:00:00 2001 From: Tyshchenko Leonid <32874828+Lcarrot@users.noreply.github.com> Date: Thu, 22 Feb 2024 09:36:12 +0300 Subject: [PATCH 06/10] Support issuing Latency commands (#3729) * add LATENCY LATEST, LATENCY HISTORY and LATENCY RESET commands * add LatencyEvent class --- .../redis/clients/jedis/BuilderFactory.java | 47 ++++++++++++++++ src/main/java/redis/clients/jedis/Jedis.java | 21 ++++++++ .../java/redis/clients/jedis/Protocol.java | 2 +- .../clients/jedis/args/LatencyEvent.java | 24 +++++++++ .../jedis/commands/ServerCommands.java | 12 +++++ .../jedis/resps/LatencyHistoryInfo.java | 38 +++++++++++++ .../jedis/resps/LatencyLatestInfo.java | 53 +++++++++++++++++++ .../commands/jedis/ControlCommandsTest.java | 20 +++++++ 8 files changed, 216 insertions(+), 1 deletion(-) create mode 100644 src/main/java/redis/clients/jedis/args/LatencyEvent.java create mode 100644 src/main/java/redis/clients/jedis/resps/LatencyHistoryInfo.java create mode 100644 src/main/java/redis/clients/jedis/resps/LatencyLatestInfo.java diff --git a/src/main/java/redis/clients/jedis/BuilderFactory.java b/src/main/java/redis/clients/jedis/BuilderFactory.java index c7349f3e89..18bcc5a31c 100644 --- a/src/main/java/redis/clients/jedis/BuilderFactory.java +++ b/src/main/java/redis/clients/jedis/BuilderFactory.java @@ -997,6 +997,53 @@ public Map build(Object data) { } }; + public static final Builder> LATENCY_LATEST_RESPONSE = new Builder>() { + @Override + public Map build(Object data) { + if (data == null) { + return null; + } + + List rawList = (List) data; + Map map = new HashMap<>(rawList.size()); + + for (Object rawLatencyLatestInfo : rawList) { + if (rawLatencyLatestInfo == null) { + continue; + } + + LatencyLatestInfo latestInfo = LatencyLatestInfo.LATENCY_LATEST_BUILDER.build(rawLatencyLatestInfo); + String name = latestInfo.getCommand(); + map.put(name, latestInfo); + } + + return map; + } + }; + + public static final Builder> LATENCY_HISTORY_RESPONSE = new Builder>() { + @Override + public List build(Object data) { + if (data == null) { + return null; + } + + List rawList = (List) data; + List response = new ArrayList<>(rawList.size()); + + for (Object rawLatencyHistoryInfo : rawList) { + if (rawLatencyHistoryInfo == null) { + continue; + } + + LatencyHistoryInfo historyInfo = LatencyHistoryInfo.LATENCY_HISTORY_BUILDER.build(rawLatencyHistoryInfo); + response.add(historyInfo); + } + + return response; + } + }; + private static final Builder>> CLUSTER_SHARD_SLOTS_RANGES = new Builder>>() { @Override diff --git a/src/main/java/redis/clients/jedis/Jedis.java b/src/main/java/redis/clients/jedis/Jedis.java index 181fe279a2..8595a14de7 100644 --- a/src/main/java/redis/clients/jedis/Jedis.java +++ b/src/main/java/redis/clients/jedis/Jedis.java @@ -14,6 +14,7 @@ import java.util.Map.Entry; import java.util.Set; import java.util.stream.Collectors; +import java.util.Arrays; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.SSLParameters; @@ -9280,6 +9281,26 @@ public String latencyDoctor() { return connection.getBulkReply(); } + public Map latencyLatest() { + checkIsInMultiOrPipeline(); + connection.sendCommand(LATENCY, LATEST); + return BuilderFactory.LATENCY_LATEST_RESPONSE.build(connection.getOne()); + } + + public List latencyHistory(LatencyEvent event) { + checkIsInMultiOrPipeline(); + connection.sendCommand(new CommandArguments(LATENCY).add(HISTORY).add(event)); + return BuilderFactory.LATENCY_HISTORY_RESPONSE.build(connection.getOne()); + } + + public long latencyReset(LatencyEvent... events) { + checkIsInMultiOrPipeline(); + CommandArguments arguments = new CommandArguments(LATENCY).add(Keyword.RESET); + Arrays.stream(events).forEach(arguments::add); + connection.sendCommand(arguments); + return connection.getIntegerReply(); + } + @Override public StreamEntryID xadd(final String key, final StreamEntryID id, final Map hash) { checkIsInMultiOrPipeline(); diff --git a/src/main/java/redis/clients/jedis/Protocol.java b/src/main/java/redis/clients/jedis/Protocol.java index cba60115b3..24830637fb 100644 --- a/src/main/java/redis/clients/jedis/Protocol.java +++ b/src/main/java/redis/clients/jedis/Protocol.java @@ -299,7 +299,7 @@ public static enum Keyword implements Rawable { REV, WITHCOORD, WITHDIST, WITHHASH, ANY, FROMMEMBER, FROMLONLAT, BYRADIUS, BYBOX, BYLEX, BYSCORE, STOREDIST, TO, FORCE, TIMEOUT, DB, UNLOAD, ABORT, IDX, MINMATCHLEN, WITHMATCHLEN, FULL, DELETE, LIBRARYNAME, WITHCODE, DESCRIPTION, GETKEYS, GETKEYSANDFLAGS, DOCS, FILTERBY, DUMP, - MODULE, ACLCAT, PATTERN, DOCTOR, USAGE, SAMPLES, PURGE, STATS, LOADEX, CONFIG, ARGS, RANK, + MODULE, ACLCAT, PATTERN, DOCTOR, LATEST, HISTORY, USAGE, SAMPLES, PURGE, STATS, LOADEX, CONFIG, ARGS, RANK, NOW, VERSION, ADDR, SKIPME, USER, LADDR, CHANNELS, NUMPAT, NUMSUB, SHARDCHANNELS, SHARDNUMSUB; diff --git a/src/main/java/redis/clients/jedis/args/LatencyEvent.java b/src/main/java/redis/clients/jedis/args/LatencyEvent.java new file mode 100644 index 0000000000..bd11d6a104 --- /dev/null +++ b/src/main/java/redis/clients/jedis/args/LatencyEvent.java @@ -0,0 +1,24 @@ +package redis.clients.jedis.args; + +import redis.clients.jedis.util.SafeEncoder; + +public enum LatencyEvent implements Rawable { + + ACTIVE_DEFRAG_CYCLE("active-defrag-cycle"), AOF_FSYNC_ALWAYS("aof-fsync-always"), AOF_STAT("aof-stat"), + AOF_REWRITE_DIFF_WRITE("aof-rewrite-diff-write"), AOF_RENAME("aof-rename"), AOF_WRITE("aof-write"), + AOF_WRITE_ACTIVE_CHILD("aof-write-active-child"), AOF_WRITE_ALONE("aof-write-alone"), + AOF_WRITE_PENDING_FSYNC("aof-write-pending-fsync"), COMMAND("command"), EXPIRE_CYCLE("expire-cycle"), + EVICTION_CYCLE("eviction-cycle"), EVICTION_DEL("eviction-del"), FAST_COMMAND("fast-command"), + FORK("fork"), RDB_UNLINK_TEMP_FILE("rdb-unlink-temp-file"); + + private final byte[] raw; + + private LatencyEvent(String s) { + raw = SafeEncoder.encode(s); + } + + @Override + public byte[] getRaw() { + return raw; + } +} diff --git a/src/main/java/redis/clients/jedis/commands/ServerCommands.java b/src/main/java/redis/clients/jedis/commands/ServerCommands.java index 511357ac1d..3aeff1153b 100644 --- a/src/main/java/redis/clients/jedis/commands/ServerCommands.java +++ b/src/main/java/redis/clients/jedis/commands/ServerCommands.java @@ -1,12 +1,18 @@ package redis.clients.jedis.commands; import redis.clients.jedis.args.FlushMode; +import redis.clients.jedis.args.LatencyEvent; import redis.clients.jedis.args.SaveMode; import redis.clients.jedis.exceptions.JedisException; import redis.clients.jedis.params.LolwutParams; import redis.clients.jedis.params.ShutdownParams; +import redis.clients.jedis.resps.LatencyHistoryInfo; +import redis.clients.jedis.resps.LatencyLatestInfo; import redis.clients.jedis.util.KeyValue; +import java.util.List; +import java.util.Map; + public interface ServerCommands { /** @@ -246,4 +252,10 @@ default void shutdown(SaveMode saveMode) throws JedisException { * @return the report */ String latencyDoctor(); + + Map latencyLatest(); + + List latencyHistory(LatencyEvent events); + + long latencyReset(LatencyEvent... events); } diff --git a/src/main/java/redis/clients/jedis/resps/LatencyHistoryInfo.java b/src/main/java/redis/clients/jedis/resps/LatencyHistoryInfo.java new file mode 100644 index 0000000000..68867e74e6 --- /dev/null +++ b/src/main/java/redis/clients/jedis/resps/LatencyHistoryInfo.java @@ -0,0 +1,38 @@ +package redis.clients.jedis.resps; + +import redis.clients.jedis.Builder; + +import java.util.List; + +import static redis.clients.jedis.BuilderFactory.LONG; + +public class LatencyHistoryInfo { + + private final long timestamp; + private final long latency; + + public LatencyHistoryInfo(long timestamp, long latency) { + this.timestamp = timestamp; + this.latency = latency; + } + + public long getTimestamp() { + return timestamp; + } + + public long getLatency() { + return latency; + } + + public static final Builder LATENCY_HISTORY_BUILDER = new Builder() { + @Override + public LatencyHistoryInfo build(Object data) { + List commandData = (List) data; + + long timestamp = LONG.build(commandData.get(0)); + long latency = LONG.build(commandData.get(1)); + + return new LatencyHistoryInfo(timestamp, latency); + } + }; +} diff --git a/src/main/java/redis/clients/jedis/resps/LatencyLatestInfo.java b/src/main/java/redis/clients/jedis/resps/LatencyLatestInfo.java new file mode 100644 index 0000000000..b441ea9d75 --- /dev/null +++ b/src/main/java/redis/clients/jedis/resps/LatencyLatestInfo.java @@ -0,0 +1,53 @@ +package redis.clients.jedis.resps; + +import redis.clients.jedis.Builder; + +import java.util.List; + +import static redis.clients.jedis.BuilderFactory.LONG; +import static redis.clients.jedis.BuilderFactory.STRING; + +public class LatencyLatestInfo { + + private final String command; + private final long timestamp; + private final long lastEventLatency; + private final long maxEventLatency; + + public LatencyLatestInfo(String command, long timestamp, long lastEventLatency, long maxEventLatency) { + this.command = command; + this.timestamp = timestamp; + this.lastEventLatency = lastEventLatency; + this.maxEventLatency = maxEventLatency; + } + + public String getCommand() { + return command; + } + + public long getTimestamp() { + return timestamp; + } + + public long getLastEventLatency() { + return lastEventLatency; + } + + public long getMaxEventLatency() { + return maxEventLatency; + } + + public static final Builder LATENCY_LATEST_BUILDER = new Builder() { + @Override + public LatencyLatestInfo build(Object data) { + List commandData = (List) data; + + String command = STRING.build(commandData.get(0)); + long timestamp = LONG.build(commandData.get(1)); + long lastEventLatency = LONG.build(commandData.get(2)); + long maxEventLatency = LONG.build(commandData.get(3)); + + return new LatencyLatestInfo(command, timestamp, lastEventLatency, maxEventLatency); + } + }; +} diff --git a/src/test/java/redis/clients/jedis/commands/jedis/ControlCommandsTest.java b/src/test/java/redis/clients/jedis/commands/jedis/ControlCommandsTest.java index df3fef6886..cb95a9489b 100644 --- a/src/test/java/redis/clients/jedis/commands/jedis/ControlCommandsTest.java +++ b/src/test/java/redis/clients/jedis/commands/jedis/ControlCommandsTest.java @@ -30,12 +30,15 @@ import redis.clients.jedis.JedisMonitor; import redis.clients.jedis.Protocol; import redis.clients.jedis.args.ClientPauseMode; +import redis.clients.jedis.args.LatencyEvent; import redis.clients.jedis.exceptions.JedisDataException; import redis.clients.jedis.HostAndPorts; import redis.clients.jedis.params.CommandListFilterByParams; import redis.clients.jedis.params.LolwutParams; import redis.clients.jedis.resps.CommandDocument; import redis.clients.jedis.resps.CommandInfo; +import redis.clients.jedis.resps.LatencyHistoryInfo; +import redis.clients.jedis.resps.LatencyLatestInfo; import redis.clients.jedis.util.AssertUtil; import redis.clients.jedis.util.KeyValue; import redis.clients.jedis.util.SafeEncoder; @@ -440,6 +443,23 @@ public void latencyDoctor() { assertNotNull(report); } + @Test + public void latencyLatest() { + Map report = jedis.latencyLatest(); + assertNotNull(report); + } + + @Test + public void latencyHistoryFork() { + List report = jedis.latencyHistory(LatencyEvent.FORK); + assertNotNull(report); + } + + @Test + public void latencyReset() { + assertTrue(jedis.latencyReset() >= 0); + } + @Test public void commandCount() { assertTrue(jedis.commandCount() > 100); From f64dbb4f66bec271923d3591f8c46ec106abdeb7 Mon Sep 17 00:00:00 2001 From: Gabriel Erzse Date: Tue, 27 Feb 2024 18:15:24 +0200 Subject: [PATCH 07/10] Add support for the NOVALUES option of HSCAN (#3741) Issue #3730 The NOVALUES option makes HSCAN return only the keys, without their associated values. This will become available with Redis 7.6. Co-authored-by: Gabriel Erzse --- .../redis/clients/jedis/CommandObjects.java | 8 + src/main/java/redis/clients/jedis/Jedis.java | 17 +- .../java/redis/clients/jedis/Protocol.java | 2 +- .../redis/clients/jedis/UnifiedJedis.java | 10 ++ .../jedis/commands/HashBinaryCommands.java | 6 + .../clients/jedis/commands/HashCommands.java | 6 + .../commands/jedis/HashesCommandsTest.java | 164 +++++++++++++++-- .../unified/HashesCommandsTestBase.java | 166 ++++++++++++++++-- 8 files changed, 345 insertions(+), 34 deletions(-) diff --git a/src/main/java/redis/clients/jedis/CommandObjects.java b/src/main/java/redis/clients/jedis/CommandObjects.java index da773d5782..2f5e4d16a0 100644 --- a/src/main/java/redis/clients/jedis/CommandObjects.java +++ b/src/main/java/redis/clients/jedis/CommandObjects.java @@ -1128,6 +1128,10 @@ public final CommandObject>> hscan(String k return new CommandObject<>(commandArguments(HSCAN).key(key).add(cursor).addParams(params), BuilderFactory.HSCAN_RESPONSE); } + public final CommandObject> hscanNoValues(String key, String cursor, ScanParams params) { + return new CommandObject<>(commandArguments(HSCAN).key(key).add(cursor).addParams(params).add(NOVALUES), BuilderFactory.SCAN_RESPONSE); + } + public final CommandObject hstrlen(String key, String field) { return new CommandObject<>(commandArguments(HSTRLEN).key(key).add(field), BuilderFactory.LONG); } @@ -1136,6 +1140,10 @@ public final CommandObject>> hscan(byte[] k return new CommandObject<>(commandArguments(HSCAN).key(key).add(cursor).addParams(params), BuilderFactory.HSCAN_BINARY_RESPONSE); } + public final CommandObject> hscanNoValues(byte[] key, byte[] cursor, ScanParams params) { + return new CommandObject<>(commandArguments(HSCAN).key(key).add(cursor).addParams(params).add(NOVALUES), BuilderFactory.SCAN_BINARY_RESPONSE); + } + public final CommandObject hstrlen(byte[] key, byte[] field) { return new CommandObject<>(commandArguments(HSTRLEN).key(key).add(field), BuilderFactory.LONG); } diff --git a/src/main/java/redis/clients/jedis/Jedis.java b/src/main/java/redis/clients/jedis/Jedis.java index 8595a14de7..1e25f2ffcb 100644 --- a/src/main/java/redis/clients/jedis/Jedis.java +++ b/src/main/java/redis/clients/jedis/Jedis.java @@ -4413,11 +4413,6 @@ public ScanResult scan(final byte[] cursor, final ScanParams params, fin return connection.executeCommand(commandObjects.scan(cursor, params, type)); } - @Override - public ScanResult> hscan(final byte[] key, final byte[] cursor) { - return hscan(key, cursor, new ScanParams()); - } - @Override public ScanResult> hscan(final byte[] key, final byte[] cursor, final ScanParams params) { @@ -4425,6 +4420,12 @@ public ScanResult> hscan(final byte[] key, final byte[ return connection.executeCommand(commandObjects.hscan(key, cursor, params)); } + @Override + public ScanResult hscanNoValues(final byte[] key, final byte[] cursor, final ScanParams params) { + checkIsInMultiOrPipeline(); + return connection.executeCommand(commandObjects.hscanNoValues(key, cursor, params)); + } + @Override public ScanResult sscan(final byte[] key, final byte[] cursor) { return sscan(key, cursor, new ScanParams()); @@ -8617,6 +8618,12 @@ public ScanResult> hscan(final String key, final Strin return connection.executeCommand(commandObjects.hscan(key, cursor, params)); } + @Override + public ScanResult hscanNoValues(final String key, final String cursor, final ScanParams params) { + checkIsInMultiOrPipeline(); + return connection.executeCommand(commandObjects.hscanNoValues(key, cursor, params)); + } + @Override public ScanResult sscan(final String key, final String cursor, final ScanParams params) { checkIsInMultiOrPipeline(); diff --git a/src/main/java/redis/clients/jedis/Protocol.java b/src/main/java/redis/clients/jedis/Protocol.java index 24830637fb..31a93e3ebb 100644 --- a/src/main/java/redis/clients/jedis/Protocol.java +++ b/src/main/java/redis/clients/jedis/Protocol.java @@ -301,7 +301,7 @@ public static enum Keyword implements Rawable { DELETE, LIBRARYNAME, WITHCODE, DESCRIPTION, GETKEYS, GETKEYSANDFLAGS, DOCS, FILTERBY, DUMP, MODULE, ACLCAT, PATTERN, DOCTOR, LATEST, HISTORY, USAGE, SAMPLES, PURGE, STATS, LOADEX, CONFIG, ARGS, RANK, NOW, VERSION, ADDR, SKIPME, USER, LADDR, - CHANNELS, NUMPAT, NUMSUB, SHARDCHANNELS, SHARDNUMSUB; + CHANNELS, NUMPAT, NUMSUB, SHARDCHANNELS, SHARDNUMSUB, NOVALUES; private final byte[] raw; diff --git a/src/main/java/redis/clients/jedis/UnifiedJedis.java b/src/main/java/redis/clients/jedis/UnifiedJedis.java index b628be6933..eab452a1f1 100644 --- a/src/main/java/redis/clients/jedis/UnifiedJedis.java +++ b/src/main/java/redis/clients/jedis/UnifiedJedis.java @@ -1549,6 +1549,11 @@ public ScanResult> hscan(String key, String cursor, Sc return executeCommand(commandObjects.hscan(key, cursor, params)); } + @Override + public ScanResult hscanNoValues(String key, String cursor, ScanParams params) { + return executeCommand(commandObjects.hscanNoValues(key, cursor, params)); + } + @Override public long hstrlen(String key, String field) { return executeCommand(commandObjects.hstrlen(key, field)); @@ -1574,6 +1579,11 @@ public ScanResult> hscan(byte[] key, byte[] cursor, Sc return executeCommand(commandObjects.hscan(key, cursor, params)); } + @Override + public ScanResult hscanNoValues(byte[] key, byte[] cursor, ScanParams params) { + return executeCommand(commandObjects.hscanNoValues(key, cursor, params)); + } + @Override public long hstrlen(byte[] key, byte[] field) { return executeCommand(commandObjects.hstrlen(key, field)); diff --git a/src/main/java/redis/clients/jedis/commands/HashBinaryCommands.java b/src/main/java/redis/clients/jedis/commands/HashBinaryCommands.java index 196f7512a8..d0695302b3 100644 --- a/src/main/java/redis/clients/jedis/commands/HashBinaryCommands.java +++ b/src/main/java/redis/clients/jedis/commands/HashBinaryCommands.java @@ -47,8 +47,14 @@ default ScanResult> hscan(byte[] key, byte[] cursor) { return hscan(key, cursor, new ScanParams()); } + default ScanResult hscanNoValues(byte[] key, byte[] cursor) { + return hscanNoValues(key, cursor, new ScanParams()); + } + ScanResult> hscan(byte[] key, byte[] cursor, ScanParams params); + ScanResult hscanNoValues(byte[] key, byte[] cursor, ScanParams params); + long hstrlen(byte[] key, byte[] field); } diff --git a/src/main/java/redis/clients/jedis/commands/HashCommands.java b/src/main/java/redis/clients/jedis/commands/HashCommands.java index d319c4fa39..ad3bed4e32 100644 --- a/src/main/java/redis/clients/jedis/commands/HashCommands.java +++ b/src/main/java/redis/clients/jedis/commands/HashCommands.java @@ -47,7 +47,13 @@ default ScanResult> hscan(String key, String cursor) { return hscan(key, cursor, new ScanParams()); } + default ScanResult hscanNoValues(String key, String cursor) { + return hscanNoValues(key, cursor, new ScanParams()); + } + ScanResult> hscan(String key, String cursor, ScanParams params); + ScanResult hscanNoValues(String key, String cursor, ScanParams params); + long hstrlen(String key, String field); } diff --git a/src/test/java/redis/clients/jedis/commands/jedis/HashesCommandsTest.java b/src/test/java/redis/clients/jedis/commands/jedis/HashesCommandsTest.java index c592d65afa..a30740dcf6 100644 --- a/src/test/java/redis/clients/jedis/commands/jedis/HashesCommandsTest.java +++ b/src/test/java/redis/clients/jedis/commands/jedis/HashesCommandsTest.java @@ -1,5 +1,7 @@ package redis.clients.jedis.commands.jedis; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -10,6 +12,7 @@ import static redis.clients.jedis.params.ScanParams.SCAN_POINTER_START_BINARY; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; @@ -17,6 +20,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import org.junit.Test; @@ -339,13 +343,20 @@ public void hgetAllPipeline() { @Test public void hscan() { - jedis.hset("foo", "b", "b"); - jedis.hset("foo", "a", "a"); + jedis.hset("foo", "b", "y"); + jedis.hset("foo", "a", "x"); ScanResult> result = jedis.hscan("foo", SCAN_POINTER_START); assertEquals(SCAN_POINTER_START, result.getCursor()); - assertFalse(result.getResult().isEmpty()); + assertEquals(2, result.getResult().size()); + + assertThat( + result.getResult().stream().map(Map.Entry::getKey).collect(Collectors.toList()), + containsInAnyOrder("a", "b")); + assertThat( + result.getResult().stream().map(Map.Entry::getValue).collect(Collectors.toList()), + containsInAnyOrder("x", "y")); // binary jedis.hset(bfoo, bbar, bcar); @@ -353,7 +364,14 @@ public void hscan() { ScanResult> bResult = jedis.hscan(bfoo, SCAN_POINTER_START_BINARY); assertArrayEquals(SCAN_POINTER_START_BINARY, bResult.getCursorAsBytes()); - assertFalse(bResult.getResult().isEmpty()); + assertEquals(1, bResult.getResult().size()); + + assertThat( + bResult.getResult().stream().map(Map.Entry::getKey).collect(Collectors.toList()), + containsInAnyOrder(bbar)); + assertThat( + bResult.getResult().stream().map(Map.Entry::getValue).collect(Collectors.toList()), + containsInAnyOrder(bcar)); } @Test @@ -361,13 +379,20 @@ public void hscanMatch() { ScanParams params = new ScanParams(); params.match("a*"); - jedis.hset("foo", "b", "b"); - jedis.hset("foo", "a", "a"); - jedis.hset("foo", "aa", "aa"); + jedis.hset("foo", "b", "y"); + jedis.hset("foo", "a", "x"); + jedis.hset("foo", "aa", "xx"); ScanResult> result = jedis.hscan("foo", SCAN_POINTER_START, params); assertEquals(SCAN_POINTER_START, result.getCursor()); - assertFalse(result.getResult().isEmpty()); + assertEquals(2, result.getResult().size()); + + assertThat( + result.getResult().stream().map(Map.Entry::getKey).collect(Collectors.toList()), + containsInAnyOrder("a", "aa")); + assertThat( + result.getResult().stream().map(Map.Entry::getValue).collect(Collectors.toList()), + containsInAnyOrder("x", "xx")); // binary params = new ScanParams(); @@ -379,10 +404,17 @@ public void hscanMatch() { jedis.hset(bfoo, bbar3, bcar); ScanResult> bResult = jedis.hscan(bfoo, SCAN_POINTER_START_BINARY, - params); + params); assertArrayEquals(SCAN_POINTER_START_BINARY, bResult.getCursorAsBytes()); - assertFalse(bResult.getResult().isEmpty()); + assertEquals(4, bResult.getResult().size()); + + assertThat( + bResult.getResult().stream().map(Map.Entry::getKey).collect(Collectors.toList()), + containsInAnyOrder(bbar, bbar1, bbar2, bbar3)); + assertThat( + bResult.getResult().stream().map(Map.Entry::getValue).collect(Collectors.toList()), + containsInAnyOrder(bcar, bcar, bcar, bcar)); } @Test @@ -391,13 +423,20 @@ public void hscanCount() { params.count(2); for (int i = 0; i < 10; i++) { - jedis.hset("foo", "a" + i, "a" + i); + jedis.hset("foo", "a" + i, "x" + i); } ScanResult> result = jedis.hscan("foo", SCAN_POINTER_START, params); assertFalse(result.getResult().isEmpty()); + assertThat( + result.getResult().stream().map(Map.Entry::getKey).map(s -> s.substring(0, 1)).collect(Collectors.toSet()), + containsInAnyOrder("a")); + assertThat( + result.getResult().stream().map(Map.Entry::getValue).map(s -> s.substring(0, 1)).collect(Collectors.toSet()), + containsInAnyOrder("x")); + // binary params = new ScanParams(); params.count(2); @@ -407,10 +446,109 @@ public void hscanCount() { jedis.hset(bfoo, bbar2, bcar); jedis.hset(bfoo, bbar3, bcar); - ScanResult> bResult = jedis.hscan(bfoo, SCAN_POINTER_START_BINARY, - params); + ScanResult> bResult = jedis.hscan(bfoo, SCAN_POINTER_START_BINARY, params); + + assertFalse(bResult.getResult().isEmpty()); + + assertThat( + bResult.getResult().stream().map(Map.Entry::getKey) + .map(a -> Arrays.copyOfRange(a, 0, 4)).map(Arrays::toString).collect(Collectors.toSet()), + containsInAnyOrder(Arrays.toString(bbar))); + assertThat( + bResult.getResult().stream().map(Map.Entry::getValue) + .map(a -> Arrays.copyOfRange(a, 0, 4)).map(Arrays::toString).collect(Collectors.toSet()), + containsInAnyOrder(Arrays.toString(bcar))); + } + + @Test + public void hscanNoValues() { + jedis.hset("foo", "b", "y"); + jedis.hset("foo", "a", "x"); + + ScanResult result = jedis.hscanNoValues("foo", SCAN_POINTER_START); + + assertEquals(SCAN_POINTER_START, result.getCursor()); + assertEquals(2, result.getResult().size()); + + assertThat(result.getResult(), containsInAnyOrder("a", "b")); + + // binary + jedis.hset(bfoo, bbar, bcar); + + ScanResult bResult = jedis.hscanNoValues(bfoo, SCAN_POINTER_START_BINARY); + + assertArrayEquals(SCAN_POINTER_START_BINARY, bResult.getCursorAsBytes()); + assertEquals(1, bResult.getResult().size()); + + assertThat(bResult.getResult(), containsInAnyOrder(bbar)); + } + + @Test + public void hscanNoValuesMatch() { + ScanParams params = new ScanParams(); + params.match("a*"); + + jedis.hset("foo", "b", "y"); + jedis.hset("foo", "a", "x"); + jedis.hset("foo", "aa", "xx"); + ScanResult result = jedis.hscanNoValues("foo", SCAN_POINTER_START, params); + + assertEquals(SCAN_POINTER_START, result.getCursor()); + assertEquals(2, result.getResult().size()); + + assertThat(result.getResult(), containsInAnyOrder("a", "aa")); + + // binary + params = new ScanParams(); + params.match(bbarstar); + + jedis.hset(bfoo, bbar, bcar); + jedis.hset(bfoo, bbar1, bcar); + jedis.hset(bfoo, bbar2, bcar); + jedis.hset(bfoo, bbar3, bcar); + + ScanResult bResult = jedis.hscanNoValues(bfoo, SCAN_POINTER_START_BINARY, params); + + assertArrayEquals(SCAN_POINTER_START_BINARY, bResult.getCursorAsBytes()); + assertEquals(4, bResult.getResult().size()); + + assertThat(bResult.getResult(), containsInAnyOrder(bbar, bbar1, bbar2, bbar3)); + } + + @Test + public void hscanNoValuesCount() { + ScanParams params = new ScanParams(); + params.count(2); + + for (int i = 0; i < 10; i++) { + jedis.hset("foo", "a" + i, "a" + i); + } + + ScanResult result = jedis.hscanNoValues("foo", SCAN_POINTER_START, params); + + assertFalse(result.getResult().isEmpty()); + + assertThat( + result.getResult().stream().map(s -> s.substring(0, 1)).collect(Collectors.toSet()), + containsInAnyOrder("a")); + + // binary + params = new ScanParams(); + params.count(2); + + jedis.hset(bfoo, bbar, bcar); + jedis.hset(bfoo, bbar1, bcar); + jedis.hset(bfoo, bbar2, bcar); + jedis.hset(bfoo, bbar3, bcar); + + ScanResult bResult = jedis.hscanNoValues(bfoo, SCAN_POINTER_START_BINARY, params); assertFalse(bResult.getResult().isEmpty()); + + assertThat( + bResult.getResult().stream() + .map(a -> Arrays.copyOfRange(a, 0, 4)).map(Arrays::toString).collect(Collectors.toSet()), + containsInAnyOrder(Arrays.toString(bbar))); } @Test diff --git a/src/test/java/redis/clients/jedis/commands/unified/HashesCommandsTestBase.java b/src/test/java/redis/clients/jedis/commands/unified/HashesCommandsTestBase.java index cac2606984..d06edf38ec 100644 --- a/src/test/java/redis/clients/jedis/commands/unified/HashesCommandsTestBase.java +++ b/src/test/java/redis/clients/jedis/commands/unified/HashesCommandsTestBase.java @@ -1,9 +1,10 @@ package redis.clients.jedis.commands.unified; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertNull; @@ -11,6 +12,7 @@ import static redis.clients.jedis.params.ScanParams.SCAN_POINTER_START_BINARY; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; @@ -18,6 +20,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import org.junit.Test; @@ -316,13 +319,20 @@ public void hgetAll() { @Test public void hscan() { - jedis.hset("foo", "b", "b"); - jedis.hset("foo", "a", "a"); + jedis.hset("foo", "b", "y"); + jedis.hset("foo", "a", "x"); ScanResult> result = jedis.hscan("foo", SCAN_POINTER_START); assertEquals(SCAN_POINTER_START, result.getCursor()); - assertFalse(result.getResult().isEmpty()); + assertEquals(2, result.getResult().size()); + + assertThat( + result.getResult().stream().map(Map.Entry::getKey).collect(Collectors.toList()), + containsInAnyOrder("a", "b")); + assertThat( + result.getResult().stream().map(Map.Entry::getValue).collect(Collectors.toList()), + containsInAnyOrder("x", "y")); // binary jedis.hset(bfoo, bbar, bcar); @@ -330,7 +340,14 @@ public void hscan() { ScanResult> bResult = jedis.hscan(bfoo, SCAN_POINTER_START_BINARY); assertArrayEquals(SCAN_POINTER_START_BINARY, bResult.getCursorAsBytes()); - assertFalse(bResult.getResult().isEmpty()); + assertEquals(1, bResult.getResult().size()); + + assertThat( + bResult.getResult().stream().map(Map.Entry::getKey).collect(Collectors.toList()), + containsInAnyOrder(bbar)); + assertThat( + bResult.getResult().stream().map(Map.Entry::getValue).collect(Collectors.toList()), + containsInAnyOrder(bcar)); } @Test @@ -338,13 +355,20 @@ public void hscanMatch() { ScanParams params = new ScanParams(); params.match("a*"); - jedis.hset("foo", "b", "b"); - jedis.hset("foo", "a", "a"); - jedis.hset("foo", "aa", "aa"); + jedis.hset("foo", "b", "y"); + jedis.hset("foo", "a", "x"); + jedis.hset("foo", "aa", "xx"); ScanResult> result = jedis.hscan("foo", SCAN_POINTER_START, params); assertEquals(SCAN_POINTER_START, result.getCursor()); - assertFalse(result.getResult().isEmpty()); + assertEquals(2, result.getResult().size()); + + assertThat( + result.getResult().stream().map(Map.Entry::getKey).collect(Collectors.toList()), + containsInAnyOrder("a", "aa")); + assertThat( + result.getResult().stream().map(Map.Entry::getValue).collect(Collectors.toList()), + containsInAnyOrder("x", "xx")); // binary params = new ScanParams(); @@ -355,11 +379,17 @@ public void hscanMatch() { jedis.hset(bfoo, bbar2, bcar); jedis.hset(bfoo, bbar3, bcar); - ScanResult> bResult = jedis.hscan(bfoo, SCAN_POINTER_START_BINARY, - params); + ScanResult> bResult = jedis.hscan(bfoo, SCAN_POINTER_START_BINARY, params); assertArrayEquals(SCAN_POINTER_START_BINARY, bResult.getCursorAsBytes()); - assertFalse(bResult.getResult().isEmpty()); + assertEquals(4, bResult.getResult().size()); + + assertThat( + bResult.getResult().stream().map(Map.Entry::getKey).collect(Collectors.toList()), + containsInAnyOrder(bbar, bbar1, bbar2, bbar3)); + assertThat( + bResult.getResult().stream().map(Map.Entry::getValue).collect(Collectors.toList()), + containsInAnyOrder(bcar, bcar, bcar, bcar)); } @Test @@ -368,13 +398,20 @@ public void hscanCount() { params.count(2); for (int i = 0; i < 10; i++) { - jedis.hset("foo", "a" + i, "a" + i); + jedis.hset("foo", "a" + i, "x" + i); } ScanResult> result = jedis.hscan("foo", SCAN_POINTER_START, params); assertFalse(result.getResult().isEmpty()); + assertThat( + result.getResult().stream().map(Map.Entry::getKey).map(s -> s.substring(0, 1)).collect(Collectors.toSet()), + containsInAnyOrder("a")); + assertThat( + result.getResult().stream().map(Map.Entry::getValue).map(s -> s.substring(0, 1)).collect(Collectors.toSet()), + containsInAnyOrder("x")); + // binary params = new ScanParams(); params.count(2); @@ -384,10 +421,109 @@ public void hscanCount() { jedis.hset(bfoo, bbar2, bcar); jedis.hset(bfoo, bbar3, bcar); - ScanResult> bResult = jedis.hscan(bfoo, SCAN_POINTER_START_BINARY, - params); + ScanResult> bResult = jedis.hscan(bfoo, SCAN_POINTER_START_BINARY, params); assertFalse(bResult.getResult().isEmpty()); + + assertThat( + bResult.getResult().stream().map(Map.Entry::getKey) + .map(a -> Arrays.copyOfRange(a, 0, 4)).map(Arrays::toString).collect(Collectors.toSet()), + containsInAnyOrder(Arrays.toString(bbar))); + assertThat( + bResult.getResult().stream().map(Map.Entry::getValue) + .map(a -> Arrays.copyOfRange(a, 0, 4)).map(Arrays::toString).collect(Collectors.toSet()), + containsInAnyOrder(Arrays.toString(bcar))); + } + + @Test + public void hscanNoValues() { + jedis.hset("foo", "b", "y"); + jedis.hset("foo", "a", "x"); + + ScanResult result = jedis.hscanNoValues("foo", SCAN_POINTER_START); + + assertEquals(SCAN_POINTER_START, result.getCursor()); + assertEquals(2, result.getResult().size()); + + assertThat(result.getResult(), containsInAnyOrder("a", "b")); + + // binary + jedis.hset(bfoo, bbar, bcar); + + ScanResult bResult = jedis.hscanNoValues(bfoo, SCAN_POINTER_START_BINARY); + + assertArrayEquals(SCAN_POINTER_START_BINARY, bResult.getCursorAsBytes()); + assertEquals(1, bResult.getResult().size()); + + assertThat(bResult.getResult(), containsInAnyOrder(bbar)); + } + + @Test + public void hscanNoValuesMatch() { + ScanParams params = new ScanParams(); + params.match("a*"); + + jedis.hset("foo", "b", "y"); + jedis.hset("foo", "a", "x"); + jedis.hset("foo", "aa", "xx"); + ScanResult result = jedis.hscanNoValues("foo", SCAN_POINTER_START, params); + + assertEquals(SCAN_POINTER_START, result.getCursor()); + assertEquals(2, result.getResult().size()); + + assertThat(result.getResult(), containsInAnyOrder("a", "aa")); + + // binary + params = new ScanParams(); + params.match(bbarstar); + + jedis.hset(bfoo, bbar, bcar); + jedis.hset(bfoo, bbar1, bcar); + jedis.hset(bfoo, bbar2, bcar); + jedis.hset(bfoo, bbar3, bcar); + + ScanResult bResult = jedis.hscanNoValues(bfoo, SCAN_POINTER_START_BINARY, params); + + assertArrayEquals(SCAN_POINTER_START_BINARY, bResult.getCursorAsBytes()); + assertEquals(4, bResult.getResult().size()); + + assertThat(bResult.getResult(), containsInAnyOrder(bbar, bbar1, bbar2, bbar3)); + } + + @Test + public void hscanNoValuesCount() { + ScanParams params = new ScanParams(); + params.count(2); + + for (int i = 0; i < 10; i++) { + jedis.hset("foo", "a" + i, "a" + i); + } + + ScanResult result = jedis.hscanNoValues("foo", SCAN_POINTER_START, params); + + assertFalse(result.getResult().isEmpty()); + + assertThat( + result.getResult().stream().map(s -> s.substring(0, 1)).collect(Collectors.toSet()), + containsInAnyOrder("a")); + + // binary + params = new ScanParams(); + params.count(2); + + jedis.hset(bfoo, bbar, bcar); + jedis.hset(bfoo, bbar1, bcar); + jedis.hset(bfoo, bbar2, bcar); + jedis.hset(bfoo, bbar3, bcar); + + ScanResult bResult = jedis.hscanNoValues(bfoo, SCAN_POINTER_START_BINARY, params); + + assertFalse(bResult.getResult().isEmpty()); + + assertThat( + bResult.getResult().stream() + .map(a -> Arrays.copyOfRange(a, 0, 4)).map(Arrays::toString).collect(Collectors.toSet()), + containsInAnyOrder(Arrays.toString(bbar))); } @Test From ab6f0581323adf9ef7e0ca1c9f55c03b3c90eb00 Mon Sep 17 00:00:00 2001 From: M Sazzadul Hoque <7600764+sazzad16@users.noreply.github.com> Date: Wed, 28 Feb 2024 23:02:25 +0600 Subject: [PATCH 08/10] Polish #3741 (#3746) Add HSCAN NOVALUES in pipelines and transactions --- src/main/java/redis/clients/jedis/PipeliningBase.java | 10 ++++++++++ .../clients/jedis/commands/HashBinaryCommands.java | 4 ++-- .../redis/clients/jedis/commands/HashCommands.java | 4 ++-- .../jedis/commands/HashPipelineBinaryCommands.java | 6 ++++++ .../clients/jedis/commands/HashPipelineCommands.java | 6 ++++++ 5 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/main/java/redis/clients/jedis/PipeliningBase.java b/src/main/java/redis/clients/jedis/PipeliningBase.java index 5c6cd45f07..bf6711b632 100644 --- a/src/main/java/redis/clients/jedis/PipeliningBase.java +++ b/src/main/java/redis/clients/jedis/PipeliningBase.java @@ -706,6 +706,11 @@ public Response>> hscan(String key, String return appendCommand(commandObjects.hscan(key, cursor, params)); } + @Override + public Response> hscanNoValues(String key, String cursor, ScanParams params) { + return appendCommand(commandObjects.hscanNoValues(key, cursor, params)); + } + @Override public Response hstrlen(String key, String field) { return appendCommand(commandObjects.hstrlen(key, field)); @@ -1972,6 +1977,11 @@ public Response>> hscan(byte[] key, byte[] return appendCommand(commandObjects.hscan(key, cursor, params)); } + @Override + public Response> hscanNoValues(byte[] key, byte[] cursor, ScanParams params) { + return appendCommand(commandObjects.hscanNoValues(key, cursor, params)); + } + @Override public Response hstrlen(byte[] key, byte[] field) { return appendCommand(commandObjects.hstrlen(key, field)); diff --git a/src/main/java/redis/clients/jedis/commands/HashBinaryCommands.java b/src/main/java/redis/clients/jedis/commands/HashBinaryCommands.java index d0695302b3..15a462c9b7 100644 --- a/src/main/java/redis/clients/jedis/commands/HashBinaryCommands.java +++ b/src/main/java/redis/clients/jedis/commands/HashBinaryCommands.java @@ -47,12 +47,12 @@ default ScanResult> hscan(byte[] key, byte[] cursor) { return hscan(key, cursor, new ScanParams()); } + ScanResult> hscan(byte[] key, byte[] cursor, ScanParams params); + default ScanResult hscanNoValues(byte[] key, byte[] cursor) { return hscanNoValues(key, cursor, new ScanParams()); } - ScanResult> hscan(byte[] key, byte[] cursor, ScanParams params); - ScanResult hscanNoValues(byte[] key, byte[] cursor, ScanParams params); long hstrlen(byte[] key, byte[] field); diff --git a/src/main/java/redis/clients/jedis/commands/HashCommands.java b/src/main/java/redis/clients/jedis/commands/HashCommands.java index ad3bed4e32..ef18e34aee 100644 --- a/src/main/java/redis/clients/jedis/commands/HashCommands.java +++ b/src/main/java/redis/clients/jedis/commands/HashCommands.java @@ -47,12 +47,12 @@ default ScanResult> hscan(String key, String cursor) { return hscan(key, cursor, new ScanParams()); } + ScanResult> hscan(String key, String cursor, ScanParams params); + default ScanResult hscanNoValues(String key, String cursor) { return hscanNoValues(key, cursor, new ScanParams()); } - ScanResult> hscan(String key, String cursor, ScanParams params); - ScanResult hscanNoValues(String key, String cursor, ScanParams params); long hstrlen(String key, String field); diff --git a/src/main/java/redis/clients/jedis/commands/HashPipelineBinaryCommands.java b/src/main/java/redis/clients/jedis/commands/HashPipelineBinaryCommands.java index e87dcacdb3..cf104f7a33 100644 --- a/src/main/java/redis/clients/jedis/commands/HashPipelineBinaryCommands.java +++ b/src/main/java/redis/clients/jedis/commands/HashPipelineBinaryCommands.java @@ -50,6 +50,12 @@ default Response>> hscan(byte[] key, byte[] Response>> hscan(byte[] key, byte[] cursor, ScanParams params); + default Response> hscanNoValues(byte[] key, byte[] cursor) { + return hscanNoValues(key, cursor, new ScanParams()); + } + + Response> hscanNoValues(byte[] key, byte[] cursor, ScanParams params); + Response hstrlen(byte[] key, byte[] field); } diff --git a/src/main/java/redis/clients/jedis/commands/HashPipelineCommands.java b/src/main/java/redis/clients/jedis/commands/HashPipelineCommands.java index 3aa256cb78..dabcac4e90 100644 --- a/src/main/java/redis/clients/jedis/commands/HashPipelineCommands.java +++ b/src/main/java/redis/clients/jedis/commands/HashPipelineCommands.java @@ -50,5 +50,11 @@ default Response>> hscan(String key, String Response>> hscan(String key, String cursor, ScanParams params); + default Response> hscanNoValues(String key, String cursor) { + return hscanNoValues(key, cursor, new ScanParams()); + } + + Response> hscanNoValues(String key, String cursor, ScanParams params); + Response hstrlen(String key, String field); } From 3dfa82d4432d2ed8442f66d5416e0269fd9de4f3 Mon Sep 17 00:00:00 2001 From: M Sazzadul Hoque <7600764+sazzad16@users.noreply.github.com> Date: Thu, 29 Feb 2024 19:14:24 +0600 Subject: [PATCH 09/10] Reduce the log level of validateObject to WARN (#3750) - Reduce the log level of `JedisFactory#validateObject` to `WARN` - Reduce the log level of `ConnectionFactory#validateObject` to `WARN` --- src/main/java/redis/clients/jedis/ConnectionFactory.java | 2 +- src/main/java/redis/clients/jedis/JedisFactory.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/redis/clients/jedis/ConnectionFactory.java b/src/main/java/redis/clients/jedis/ConnectionFactory.java index d286462347..3500a21172 100644 --- a/src/main/java/redis/clients/jedis/ConnectionFactory.java +++ b/src/main/java/redis/clients/jedis/ConnectionFactory.java @@ -76,7 +76,7 @@ public boolean validateObject(PooledObject pooledConnection) { // check HostAndPort ?? return jedis.isConnected() && jedis.ping(); } catch (final Exception e) { - logger.error("Error while validating pooled Connection object.", e); + logger.warn("Error while validating pooled Connection object.", e); return false; } } diff --git a/src/main/java/redis/clients/jedis/JedisFactory.java b/src/main/java/redis/clients/jedis/JedisFactory.java index b84e2b05ae..0e07ccc286 100644 --- a/src/main/java/redis/clients/jedis/JedisFactory.java +++ b/src/main/java/redis/clients/jedis/JedisFactory.java @@ -197,7 +197,7 @@ public boolean validateObject(PooledObject pooledJedis) { && jedis.getConnection().isConnected() && jedis.ping().equals("PONG"); } catch (final Exception e) { - logger.error("Error while validating pooled Jedis object.", e); + logger.warn("Error while validating pooled Jedis object.", e); return false; } } From 25acc8d5b1519d1404fd04b094638a794a054caf Mon Sep 17 00:00:00 2001 From: M Sazzadul Hoque <7600764+sazzad16@users.noreply.github.com> Date: Wed, 6 Mar 2024 13:29:41 +0600 Subject: [PATCH 10/10] Stop connection fetching before sync/exec (#3756) in multi cluster failover mode --- .../clients/jedis/AbstractTransaction.java | 4 +++ .../jedis/MultiClusterClientConfig.java | 3 +- .../redis/clients/jedis/TransactionBase.java | 4 +++ .../redis/clients/jedis/UnifiedJedis.java | 4 +-- .../jedis/mcf/MultiClusterPipeline.java | 6 ++++ .../jedis/mcf/MultiClusterTransaction.java | 17 ++++++++++ .../jedis/misc/AutomaticFailoverTest.java | 32 +++++++++++++------ 7 files changed, 57 insertions(+), 13 deletions(-) diff --git a/src/main/java/redis/clients/jedis/AbstractTransaction.java b/src/main/java/redis/clients/jedis/AbstractTransaction.java index ed6f397caa..2a551224fa 100644 --- a/src/main/java/redis/clients/jedis/AbstractTransaction.java +++ b/src/main/java/redis/clients/jedis/AbstractTransaction.java @@ -9,6 +9,10 @@ protected AbstractTransaction() { super(new CommandObjects()); } + protected AbstractTransaction(CommandObjects commandObjects) { + super(commandObjects); + } + public abstract void multi(); /** diff --git a/src/main/java/redis/clients/jedis/MultiClusterClientConfig.java b/src/main/java/redis/clients/jedis/MultiClusterClientConfig.java index 980bbb91f9..15956ebed4 100644 --- a/src/main/java/redis/clients/jedis/MultiClusterClientConfig.java +++ b/src/main/java/redis/clients/jedis/MultiClusterClientConfig.java @@ -40,8 +40,7 @@ public final class MultiClusterClientConfig { private static final float CIRCUIT_BREAKER_SLOW_CALL_RATE_THRESHOLD_DEFAULT = 100.0f; // measured as percentage private static final List CIRCUIT_BREAKER_INCLUDED_EXCEPTIONS_DEFAULT = Arrays.asList(JedisConnectionException.class); - private static final List> FALLBACK_EXCEPTIONS_DEFAULT = - Arrays.asList(CallNotPermittedException.class, JedisConnectionException.class); + private static final List> FALLBACK_EXCEPTIONS_DEFAULT = Arrays.asList(CallNotPermittedException.class); private final ClusterConfig[] clusterConfigs; diff --git a/src/main/java/redis/clients/jedis/TransactionBase.java b/src/main/java/redis/clients/jedis/TransactionBase.java index 805a1120c4..efdf332700 100644 --- a/src/main/java/redis/clients/jedis/TransactionBase.java +++ b/src/main/java/redis/clients/jedis/TransactionBase.java @@ -9,4 +9,8 @@ public abstract class TransactionBase extends AbstractTransaction { protected TransactionBase() { super(); } + + protected TransactionBase(CommandObjects commandObjects) { + super(commandObjects); + } } diff --git a/src/main/java/redis/clients/jedis/UnifiedJedis.java b/src/main/java/redis/clients/jedis/UnifiedJedis.java index eab452a1f1..a43ffc1090 100644 --- a/src/main/java/redis/clients/jedis/UnifiedJedis.java +++ b/src/main/java/redis/clients/jedis/UnifiedJedis.java @@ -4849,7 +4849,7 @@ public PipelineBase pipelined() { if (provider == null) { throw new IllegalStateException("It is not allowed to create Pipeline from this " + getClass()); } else if (provider instanceof MultiClusterPooledConnectionProvider) { - return new MultiClusterPipeline((MultiClusterPooledConnectionProvider) provider); + return new MultiClusterPipeline((MultiClusterPooledConnectionProvider) provider, commandObjects); } else { return new Pipeline(provider.getConnection(), true); } @@ -4859,7 +4859,7 @@ public AbstractTransaction multi() { if (provider == null) { throw new IllegalStateException("It is not allowed to create Pipeline from this " + getClass()); } else if (provider instanceof MultiClusterPooledConnectionProvider) { - return new MultiClusterTransaction((MultiClusterPooledConnectionProvider) provider); + return new MultiClusterTransaction((MultiClusterPooledConnectionProvider) provider, true, commandObjects); } else { return new Transaction(provider.getConnection(), true, true); } diff --git a/src/main/java/redis/clients/jedis/mcf/MultiClusterPipeline.java b/src/main/java/redis/clients/jedis/mcf/MultiClusterPipeline.java index d4052dae7b..00c0ba1d91 100644 --- a/src/main/java/redis/clients/jedis/mcf/MultiClusterPipeline.java +++ b/src/main/java/redis/clients/jedis/mcf/MultiClusterPipeline.java @@ -20,6 +20,7 @@ public class MultiClusterPipeline extends PipelineBase implements Closeable { private final CircuitBreakerFailoverConnectionProvider failoverProvider; private final Queue>> commands = new LinkedList<>(); + @Deprecated public MultiClusterPipeline(MultiClusterPooledConnectionProvider pooledProvider) { super(new CommandObjects()); @@ -31,6 +32,11 @@ public MultiClusterPipeline(MultiClusterPooledConnectionProvider pooledProvider) } } + public MultiClusterPipeline(MultiClusterPooledConnectionProvider pooledProvider, CommandObjects commandObjects) { + super(commandObjects); + this.failoverProvider = new CircuitBreakerFailoverConnectionProvider(pooledProvider); + } + @Override protected final Response appendCommand(CommandObject commandObject) { CommandArguments args = commandObject.getArguments(); diff --git a/src/main/java/redis/clients/jedis/mcf/MultiClusterTransaction.java b/src/main/java/redis/clients/jedis/mcf/MultiClusterTransaction.java index 540911f2d6..d759ce1da0 100644 --- a/src/main/java/redis/clients/jedis/mcf/MultiClusterTransaction.java +++ b/src/main/java/redis/clients/jedis/mcf/MultiClusterTransaction.java @@ -38,6 +38,7 @@ public class MultiClusterTransaction extends TransactionBase { * called with this object. * @param provider */ + @Deprecated public MultiClusterTransaction(MultiClusterPooledConnectionProvider provider) { this(provider, true); } @@ -49,6 +50,7 @@ public MultiClusterTransaction(MultiClusterPooledConnectionProvider provider) { * @param provider * @param doMulti {@code false} should be set to enable manual WATCH, UNWATCH and MULTI */ + @Deprecated public MultiClusterTransaction(MultiClusterPooledConnectionProvider provider, boolean doMulti) { this.failoverProvider = new CircuitBreakerFailoverConnectionProvider(provider); @@ -60,6 +62,21 @@ public MultiClusterTransaction(MultiClusterPooledConnectionProvider provider, bo if (doMulti) multi(); } + /** + * A user wanting to WATCH/UNWATCH keys followed by a call to MULTI ({@link #multi()}) it should + * be {@code doMulti=false}. + * + * @param provider + * @param doMulti {@code false} should be set to enable manual WATCH, UNWATCH and MULTI + * @param commandObjects command objects + */ + public MultiClusterTransaction(MultiClusterPooledConnectionProvider provider, boolean doMulti, CommandObjects commandObjects) { + super(commandObjects); + this.failoverProvider = new CircuitBreakerFailoverConnectionProvider(provider); + + if (doMulti) multi(); + } + @Override public final void multi() { appendCommand(new CommandObject<>(new CommandArguments(MULTI), NO_OP_BUILDER)); diff --git a/src/test/java/redis/clients/jedis/misc/AutomaticFailoverTest.java b/src/test/java/redis/clients/jedis/misc/AutomaticFailoverTest.java index dc03c52f39..c6b25d764b 100644 --- a/src/test/java/redis/clients/jedis/misc/AutomaticFailoverTest.java +++ b/src/test/java/redis/clients/jedis/misc/AutomaticFailoverTest.java @@ -26,6 +26,7 @@ import redis.clients.jedis.MultiClusterClientConfig; import redis.clients.jedis.UnifiedJedis; import redis.clients.jedis.exceptions.JedisAccessControlException; +import redis.clients.jedis.exceptions.JedisConnectionException; import redis.clients.jedis.providers.MultiClusterPooledConnectionProvider; import redis.clients.jedis.util.IOUtils; @@ -68,7 +69,7 @@ public void pipelineWithSwitch() { AbstractPipeline pipe = client.pipelined(); pipe.set("pstr", "foobar"); pipe.hset("phash", "foo", "bar"); - //provider.incrementActiveMultiClusterIndex(); + provider.incrementActiveMultiClusterIndex(); pipe.sync(); } @@ -85,7 +86,7 @@ public void transactionWithSwitch() { AbstractTransaction tx = client.multi(); tx.set("tstr", "foobar"); tx.hset("thash", "foo", "bar"); - //provider.incrementActiveMultiClusterIndex(); + provider.incrementActiveMultiClusterIndex(); assertEquals(Arrays.asList("OK", Long.valueOf(1L)), tx.exec()); } @@ -109,9 +110,19 @@ public void commandFailover() { UnifiedJedis jedis = new UnifiedJedis(cacheProvider); - assertFalse(failoverReporter.failedOver); - log.info("Starting calls to Redis"); String key = "hash-" + System.nanoTime(); + log.info("Starting calls to Redis"); + assertFalse(failoverReporter.failedOver); + for (int attempt = 0; attempt < 10; attempt++) { + try { + jedis.hset(key, "f1", "v1"); + } catch (JedisConnectionException jce) { + // + } + assertFalse(failoverReporter.failedOver); + } + + // should failover now jedis.hset(key, "f1", "v1"); assertTrue(failoverReporter.failedOver); @@ -129,7 +140,8 @@ public void pipelineFailover() { MultiClusterClientConfig.Builder builder = new MultiClusterClientConfig.Builder( getClusterConfigs(clientConfig, hostPort_1, hostPort_2)) .circuitBreakerSlidingWindowMinCalls(slidingWindowMinCalls) - .circuitBreakerSlidingWindowSize(slidingWindowSize); + .circuitBreakerSlidingWindowSize(slidingWindowSize) + .fallbackExceptionList(Arrays.asList(JedisConnectionException.class)); RedisFailoverReporter failoverReporter = new RedisFailoverReporter(); MultiClusterPooledConnectionProvider cacheProvider = new MultiClusterPooledConnectionProvider(builder.build()); @@ -137,11 +149,13 @@ public void pipelineFailover() { UnifiedJedis jedis = new UnifiedJedis(cacheProvider); - assertFalse(failoverReporter.failedOver); + String key = "hash-" + System.nanoTime(); log.info("Starting calls to Redis"); + assertFalse(failoverReporter.failedOver); AbstractPipeline pipe = jedis.pipelined(); - String key = "hash-" + System.nanoTime(); + assertFalse(failoverReporter.failedOver); pipe.hset(key, "f1", "v1"); + assertFalse(failoverReporter.failedOver); pipe.sync(); assertTrue(failoverReporter.failedOver); @@ -168,9 +182,9 @@ public void failoverFromAuthError() { UnifiedJedis jedis = new UnifiedJedis(cacheProvider); - assertFalse(failoverReporter.failedOver); - log.info("Starting calls to Redis"); String key = "hash-" + System.nanoTime(); + log.info("Starting calls to Redis"); + assertFalse(failoverReporter.failedOver); jedis.hset(key, "f1", "v1"); assertTrue(failoverReporter.failedOver);