Jp/perf 2.9 2025013

.github/workflows/test.yml

@@ -221,10 +221,13 @@ jobs:
             ar_block,
             ar_block_cache,
             ar_chain_stats,
+            ar_chunk_copy,
             ar_chunk_storage,
             ar_data_sync_worker_master,
             ar_deep_hash,
+            ar_device_lock,
             ar_diff_dag,
+            ar_entropy_storage,
             ar_ets_intervals,
             ar_events,
             ar_inflation,

apps/arweave/e2e/ar_e2e.erl

@@ -6,7 +6,7 @@
 -export([delayed_print/2, packing_type_to_packing/2,
 	start_source_node/3, source_node_storage_modules/3, max_chunk_offset/1,
 	assert_block/2, assert_syncs_range/3, assert_does_not_sync_range/3, 
-	assert_chunks/3, assert_no_chunks/2, assert_partition_size/4, assert_empty_partition/3]).
+	assert_chunks/3, assert_no_chunks/2, assert_partition_size/3, assert_empty_partition/3]).
 
 -include_lib("arweave/include/ar.hrl").
 -include_lib("arweave/include/ar_config.hrl").
@@ -70,6 +70,8 @@ packing_type_to_packing(PackingType, Address) ->
 	end.
 
 start_source_node(Node, unpacked, _WalletFixture) ->
+	?LOG_INFO("Starting source node ~p with packing type ~p and wallet fixture ~p",
+		[Node, unpacked, _WalletFixture]),
 	TempNode = case Node of
 		peer1 -> peer2;
 		peer2 -> peer1
@@ -85,11 +87,16 @@ start_source_node(Node, unpacked, _WalletFixture) ->
 		auto_join = true
 	}, true),
 
-	ar_e2e:assert_partition_size(Node, 0, unpacked, ?PARTITION_SIZE),
-	ar_e2e:assert_partition_size(Node, 1, unpacked, ?PARTITION_SIZE),
+	?LOG_INFO("Source node ~p started.", [Node]),
+
+	ar_e2e:assert_partition_size(Node, 0, unpacked),
+	ar_e2e:assert_partition_size(Node, 1, unpacked),
 
 	ar_e2e:assert_syncs_range(Node, ?PARTITION_SIZE, 2*?PARTITION_SIZE),
 	ar_e2e:assert_chunks(Node, unpacked, Chunks),
+
+	?LOG_INFO("Source node ~p assertions passed.", [Node]),
+
 	ar_test_node:stop(TempNode),
 	{Blocks, undefined, Chunks};
 start_source_node(Node, PackingType, WalletFixture) ->
@@ -110,6 +117,8 @@ start_source_node(Node, PackingType, WalletFixture) ->
 		}, true)
 	),
 
+	?LOG_INFO("Source node ~p started.", [Node]),
+
 	%% Note: small chunks will be padded to 256 KiB. So B1 actually contains 3 chunks of data
 	%% and B2 starts at a chunk boundary and contains 1 chunk of data.
 	B1 = mine_block(Node, Wallet, floor(2.5 * ?DATA_CHUNK_SIZE)),
@@ -130,14 +139,16 @@ start_source_node(Node, PackingType, WalletFixture) ->
 		{B3, ?PARTITION_SIZE + (8*?DATA_CHUNK_SIZE), ?DATA_CHUNK_SIZE}
 	],
 
-	?LOG_INFO("Source node ~p started.", [Node]),
+	?LOG_INFO("Source node ~p blocks mined.", [Node]),
 
 	SourcePacking = ar_e2e:packing_type_to_packing(PackingType, RewardAddr),
 
-	ar_e2e:assert_partition_size(Node, 0, SourcePacking, ?PARTITION_SIZE),
-	ar_e2e:assert_partition_size(Node, 1, SourcePacking, ?PARTITION_SIZE),
+	ar_e2e:assert_partition_size(Node, 0, SourcePacking),
+	ar_e2e:assert_partition_size(Node, 1, SourcePacking),
 
-	ar_e2e:assert_syncs_range(Node, ?PARTITION_SIZE, 2*?PARTITION_SIZE),
+	ar_e2e:assert_syncs_range(Node, 
+		?PARTITION_SIZE,
+		2*?PARTITION_SIZE + ar_storage_module:get_overlap(SourcePacking)),
 	ar_e2e:assert_chunks(Node, SourcePacking, Chunks),
 
 	?LOG_INFO("Source node ~p assertions passed.", [Node]),
@@ -211,14 +222,21 @@ assert_block({replica_2_9, Address}, MinedBlock) ->
 
 
 assert_syncs_range(Node, StartOffset, EndOffset) ->
-	?assert(
-		ar_util:do_until(
-			fun() -> has_range(Node, StartOffset, EndOffset) end,
-			100,
-			60_000
-		),
-		iolist_to_binary(io_lib:format(
-			"~s Failed to sync range ~p - ~p", [Node, StartOffset, EndOffset]))).
+	HasRange = ar_util:do_until(
+		fun() -> has_range(Node, StartOffset, EndOffset) end,
+		100,
+		60_000
+	),
+	case HasRange of
+		true ->
+			ok;
+		false ->
+			SyncRecord = ar_http_iface_client:get_sync_record(Node, json),
+			?assert(false, 
+				iolist_to_binary(io_lib:format(
+					"~s failed to sync range ~p - ~p. Sync record: ~p", 
+					[Node, StartOffset, EndOffset, SyncRecord])))
+	end.
 
 assert_does_not_sync_range(Node, StartOffset, EndOffset) ->
 	ar_util:do_until(
@@ -231,7 +249,10 @@ assert_does_not_sync_range(Node, StartOffset, EndOffset) ->
 			"~s synced range when it should not have: ~p - ~p", 
 			[Node, StartOffset, EndOffset]))).
 
-assert_partition_size(Node, PartitionNumber, Packing, Size) ->
+assert_partition_size(Node, PartitionNumber, Packing) ->
+	Size = ?PARTITION_SIZE,
+	?LOG_INFO("~p: Asserting partition ~p,~p is size ~p",
+		[Node, PartitionNumber, ar_serialize:encode_packing(Packing, true), Size]),
 	?assert(
 		ar_util:do_until(
 			fun() -> 
@@ -330,17 +351,18 @@ assert_chunk(Node, Packing, Block, EndOffset, ChunkSize) ->
 	{ok, ExpectedPackedChunk} = ar_e2e:load_chunk_fixture(Packing, EndOffset),
 	?assertEqual(ExpectedPackedChunk, Chunk,
 		iolist_to_binary(io_lib:format(
-			"Chunk at offset ~p, size ~p does not match previously packed chunk",
-			[EndOffset, ChunkSize]))),
+			"~p: Chunk at offset ~p, size ~p does not match previously packed chunk",
+			[Node, EndOffset, ChunkSize]))),
 
 	{ok, UnpackedChunk} = ar_packing_server:unpack(
 		Packing, EndOffset, Block#block.tx_root, Chunk, ?DATA_CHUNK_SIZE),
-	UnpaddedChunk = ar_packing_server:unpad_chunk(Packing, UnpackedChunk, ChunkSize, byte_size(Chunk)),
+	UnpaddedChunk = ar_packing_server:unpad_chunk(
+		Packing, UnpackedChunk, ChunkSize, byte_size(Chunk)),
 	ExpectedUnpackedChunk = ar_test_node:get_genesis_chunk(EndOffset),
 	?assertEqual(ExpectedUnpackedChunk, UnpaddedChunk,
 		iolist_to_binary(io_lib:format(
-			"Chunk at offset ~p, size ~p does not match unpacked chunk",
-			[EndOffset, ChunkSize]))).
+			"~p: Chunk at offset ~p, size ~p does not match unpacked chunk",
+			[Node, EndOffset, ChunkSize]))).
 
 assert_no_chunks(Node, Chunks) ->
 	lists:foreach(fun({_Block, EndOffset, _ChunkSize}) ->

apps/arweave/e2e/ar_repack_in_place_mine_tests.erl

@@ -15,25 +15,9 @@
 repack_in_place_mine_test_() ->
 	Timeout = ?REPACK_IN_PLACE_MINE_TEST_TIMEOUT,
 	[
-		% XXX {timeout, Timeout, {with, {unpacked, replica_2_9}, [fun test_repack_in_place_mine/1]}},
-		% XXX {timeout, Timeout, {with, {unpacked, spora_2_6}, [fun test_repack_in_place_mine/1]}},
-		% XXX {timeout, Timeout, {with, {unpacked, composite_1}, [fun test_repack_in_place_mine/1]}},
-		% XXX {timeout, Timeout, {with, {unpacked, composite_2}, [fun test_repack_in_place_mine/1]}},
+		% {timeout, Timeout, {with, {unpacked, replica_2_9}, [fun test_repack_in_place_mine/1]}},
 		{timeout, Timeout, {with, {spora_2_6, replica_2_9}, [fun test_repack_in_place_mine/1]}},
-		{timeout, Timeout, {with, {spora_2_6, spora_2_6}, [fun test_repack_in_place_mine/1]}},
-		{timeout, Timeout, {with, {spora_2_6, composite_1}, [fun test_repack_in_place_mine/1]}},
-		{timeout, Timeout, {with, {spora_2_6, composite_2}, [fun test_repack_in_place_mine/1]}},
-		% % % % % XXX {timeout, Timeout, {with, {spora_2_6, unpacked}, [fun test_repack_in_place_mine/1]}},
-		{timeout, Timeout, {with, {composite_1, replica_2_9}, [fun test_repack_in_place_mine/1]}},
-		{timeout, Timeout, {with, {composite_1, spora_2_6}, [fun test_repack_in_place_mine/1]}},
-		{timeout, Timeout, {with, {composite_1, composite_1}, [fun test_repack_in_place_mine/1]}},
-		{timeout, Timeout, {with, {composite_1, composite_2}, [fun test_repack_in_place_mine/1]}},
-		% % % % % XXX {timeout, Timeout, {with, {composite_1, unpacked}, [fun test_repack_in_place_mine/1]}},
-		{timeout, Timeout, {with, {composite_2, replica_2_9}, [fun test_repack_in_place_mine/1]}},
-		{timeout, Timeout, {with, {composite_2, spora_2_6}, [fun test_repack_in_place_mine/1]}},
-		{timeout, Timeout, {with, {composite_2, composite_1}, [fun test_repack_in_place_mine/1]}},
-		{timeout, Timeout, {with, {composite_2, composite_2}, [fun test_repack_in_place_mine/1]}}
-		% % % % % XXX {timeout, Timeout, {with, {composite_2, unpacked}, [fun test_repack_in_place_mine/1]}}
+		{timeout, Timeout, {with, {composite_1, replica_2_9}, [fun test_repack_in_place_mine/1]}}
 	].
 
 %% --------------------------------------------------------------------------------------------
@@ -69,8 +53,8 @@ test_repack_in_place_mine({FromPackingType, ToPackingType}) ->
 	}),
 	ar_test_node:restart(RepackerNode),
 
-	ar_e2e:assert_partition_size(RepackerNode, 0, ToPacking, ?PARTITION_SIZE),
-	ar_e2e:assert_partition_size(RepackerNode, 1, ToPacking, ?PARTITION_SIZE),
+	ar_e2e:assert_partition_size(RepackerNode, 0, ToPacking),
+	ar_e2e:assert_partition_size(RepackerNode, 1, ToPacking),
 
 	ar_test_node:stop(RepackerNode),