Skip to content

Commit

Permalink
extstore tests: further tuning of tests
Browse files Browse the repository at this point in the history 
enforce actually waiting for extstore to flush items before moving on, remove
some pacing that was making things worse. The compactor is rescuing the
canary items occasionally.

Detunes the compactor on start, then ramps it up before the compactor
specific tests. This seems to fix the flakiness, at least enough that it's
been passing in a loop on fast and slow systems for me.
  • Loading branch information
@dormando
dormando committed May 25, 2018
1 parent def2f9f commit 3c270bb
Show file tree
Hide file tree
Showing 2 changed files with 61 additions and 26 deletions.
56 changes: 35 additions & 21 deletions t/chunked-extstore.t
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,26 @@ if (!supports_extstore()) {

$ext_path = "/tmp/extstore.$$";

my $server = new_memcached("-m 64 -U 0 -o ext_page_size=8,ext_page_count=8,ext_wbuf_size=2,ext_threads=1,ext_io_depth=2,ext_item_size=512,ext_item_age=2,ext_recache_rate=10000,ext_max_frag=0.9,ext_path=$ext_path,slab_chunk_max=16384,slab_automove=0");
my $server = new_memcached("-m 64 -U 0 -o ext_page_size=8,ext_page_count=8,ext_wbuf_size=2,ext_threads=1,ext_io_depth=2,ext_item_size=512,ext_item_age=2,ext_recache_rate=10000,ext_max_frag=0.9,ext_path=$ext_path,slab_chunk_max=16384,slab_automove=0,ext_compact_under=1");
my $sock = $server->sock;

# Wait until all items have flushed
sub wait_for_ext {
my $sum = 1;
while ($sum != 0) {
my $s = mem_stats($sock, "items");
$sum = 0;
for my $key (keys %$s) {
if ($key =~ m/items:(\d+):number/) {
# Ignore classes which can contain extstore items
next if $1 < 3;
$sum += $s->{$key};
}
}
sleep 1 if $sum != 0;
}
}

# We're testing to ensure item chaining doesn't corrupt or poorly overlap
# data, so create a non-repeating pattern.
my @parts = ();
Expand Down Expand Up @@ -58,7 +75,7 @@ for (1..5) {
print $sock "set toast$_ 0 0 $biglen\r\n$big\r\n";
is(scalar <$sock>, "STORED\r\n", "stored big");
}
sleep 4;
wait_for_ext();

for (1..40) {
mem_get_is($sock, "toast$_", $big);
Expand All @@ -76,18 +93,19 @@ for (1..5) {

# fill to eviction
{
my $keycount = 2000;
my $keycount = 1250;
for (1 .. $keycount) {
print $sock "set mfoo$_ 0 0 $plen noreply\r\n$pattern\r\n";
sleep 1 if ($_ % 250 == 0);
wait_for_ext() if $_ % 500 == 0;
}
sleep 1;
# because item_age is set to 2s.
wait_for_ext();

my $stats = mem_stats($sock);
cmp_ok($stats->{extstore_page_evictions}, '>', 0, 'at least one page evicted');
cmp_ok($stats->{extstore_objects_evicted}, '>', 0, 'at least one object evicted');
cmp_ok($stats->{extstore_bytes_evicted}, '>', 0, 'some bytes evicted');
is($stats->{extstore_pages_free}, 1, '1 page is free');
cmp_ok($stats->{extstore_pages_free}, '<', 2, 'most pages are used');
is($stats->{miss_from_extstore}, 0, 'no misses');

# original "pattern" key should be gone.
Expand All @@ -107,14 +125,18 @@ for (1..5) {
print $sock "delete toast$_ noreply\r\n" if $_ % 2 == 0;
}

for (1..1000) {
for (1..1250) {
# Force a read so objects don't get skipped.
mem_get_is($sock, "mfoo$_", $pattern) if $_ % 2 == 1;
print $sock "add mfoo$_ 0 0 1 noreply\r\n1\r\n" if $_ % 2 == 1;
}
for (1..1000) {
# Force a read so objects don't get skipped.
for (1..1250) {
# Delete lots of objects to trigger compaction.
print $sock "delete mfoo$_ noreply\r\n" if $_ % 2 == 0;
}
print $sock "extstore compact_under 4\r\n";
my $res = <$sock>;
print $sock "extstore drop_under 3\r\n";
$res = <$sock>;

sleep 4;

Expand All @@ -123,7 +145,7 @@ for (1..5) {
cmp_ok($stats->{extstore_compact_rescues}, '>', 0, 'some compaction rescues happened');

# Some of the early items got evicted
for (100..1000) {
for (750..1250) {
# everything should validate properly.
mem_get_is($sock, "mfoo$_", $pattern) if $_ % 2 == 1;
}
Expand All @@ -134,20 +156,12 @@ for (1..5) {
print $sock "extstore recache_rate 1\r\n";
is(scalar <$sock>, "OK\r\n", "upped recache rate");

for (800..1000) {
for (1150..1250) {
mem_get_is($sock, "mfoo$_", $pattern) if $_ % 2 == 1;
}

my $stats = mem_stats($sock);
cmp_ok($stats->{recache_from_extstore}, '>', 100, 'recaching happening');

for (800..1000) {
mem_get_is($sock, "mfoo$_", $pattern) if $_ % 2 == 1;
}

my $stats2 = mem_stats($sock);
is($stats->{recache_from_extstore}, $stats2->{recache_from_extstore},
'values already recached');
cmp_ok($stats->{recache_from_extstore}, '>', 25, 'recaching happening');
}

done_testing();
Expand Down
31 changes: 26 additions & 5 deletions t/extstore.t
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,26 @@ if (!supports_extstore()) {

$ext_path = "/tmp/extstore.$$";

my $server = new_memcached("-m 64 -U 0 -o ext_page_size=8,ext_page_count=8,ext_wbuf_size=2,ext_threads=1,ext_io_depth=2,ext_item_size=512,ext_item_age=2,ext_recache_rate=10000,ext_max_frag=0.9,ext_path=$ext_path,slab_automove=0");
my $server = new_memcached("-m 64 -U 0 -o ext_page_size=8,ext_page_count=8,ext_wbuf_size=2,ext_threads=1,ext_io_depth=2,ext_item_size=512,ext_item_age=2,ext_recache_rate=10000,ext_max_frag=0.9,ext_path=$ext_path,slab_automove=0,ext_compact_under=1");
my $sock = $server->sock;

# Wait until all items have flushed
sub wait_for_ext {
my $sum = 1;
while ($sum != 0) {
my $s = mem_stats($sock, "items");
$sum = 0;
for my $key (keys %$s) {
if ($key =~ m/items:(\d+):number/) {
# Ignore classes which can contain extstore items
next if $1 < 3;
$sum += $s->{$key};
}
}
sleep 1 if $sum != 0;
}
}

my $value;
{
my @chars = ("C".."Z");
Expand Down Expand Up @@ -47,7 +64,7 @@ mem_get_is($sock, "foo", "hi");
print $sock "set nfoo$_ 0 0 20000 noreply\r\n$value\r\n";
}
# wait for a flush
sleep 4;
wait_for_ext();
# fetch
# TODO: Fetch back all values
mem_get_is($sock, "nfoo1", $value);
Expand Down Expand Up @@ -86,9 +103,9 @@ mem_get_is($sock, "foo", "hi");
my $keycount = 4000;
for (1 .. $keycount) {
print $sock "set mfoo$_ 0 0 20000 noreply\r\n$value\r\n";
sleep 1 if ($_ % 1000 == 0);
}
sleep 1;
# because item_age is set to 2s
wait_for_ext();
my $stats = mem_stats($sock);
is($stats->{miss_from_extstore}, 0, 'no misses');
mem_get_is($sock, "canary", undef);
Expand All @@ -98,7 +115,7 @@ mem_get_is($sock, "foo", "hi");
cmp_ok($stats->{extstore_page_evictions}, '>', 0, 'at least one page evicted');
cmp_ok($stats->{extstore_objects_evicted}, '>', 0, 'at least one object evicted');
cmp_ok($stats->{extstore_bytes_evicted}, '>', 0, 'some bytes evicted');
is($stats->{extstore_pages_free}, 0, '0 pages are free');
cmp_ok($stats->{extstore_pages_free}, '<', 2, 'few pages are free');
is($stats->{miss_from_extstore}, 1, 'exactly one miss');

# refresh some keys so rescues happen while drop_unread == 1.
Expand All @@ -112,6 +129,10 @@ mem_get_is($sock, "foo", "hi");
my $res = <$sock>;
print $sock "extstore max_frag 0\r\n";
$res = <$sock>;
print $sock "extstore compact_under 4\r\n";
$res = <$sock>;
print $sock "extstore drop_under 3\r\n";
$res = <$sock>;
for (1 .. $keycount) {
next unless $_ % 2 == 0;
print $sock "delete mfoo$_ noreply\r\n";
Expand Down

0 comments on commit 3c270bb

Please sign in to comment.