Skip to content

Commit

Permalink
Merge pull request #399 from umccr/fix/filemanager-etag
Browse files Browse the repository at this point in the history 
fix: filemanager e_tag quoting
  • Loading branch information
@mmalenic
mmalenic authored Jul 9, 2024
2 parents 6b990f7 + 329bdd8 commit 3294cab
Show file tree
Hide file tree
Showing 6 changed files with 229 additions and 162 deletions.
37 changes: 16 additions & 21 deletions lib/workload/stateless/stacks/filemanager/filemanager/src/database/aws/ingester.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,11 +135,11 @@ pub(crate) mod tests {

use crate::database::aws::migration::tests::MIGRATOR;
use crate::database::{Client, Ingest};
use crate::events::aws::message::EventType;
use crate::events::aws::message::EventType::{Created, Deleted};
use crate::events::aws::message::{default_version_id, EventType};
use crate::events::aws::tests::{
expected_events_simple, expected_events_simple_delete_marker, expected_flat_events_simple,
EXPECTED_E_TAG, EXPECTED_SEQUENCER_CREATED_ONE, EXPECTED_SEQUENCER_CREATED_ZERO,
EXPECTED_QUOTED_E_TAG, EXPECTED_SEQUENCER_CREATED_ONE, EXPECTED_SEQUENCER_CREATED_ZERO,
EXPECTED_SEQUENCER_DELETED_ONE, EXPECTED_SEQUENCER_DELETED_TWO, EXPECTED_SHA256,
EXPECTED_VERSION_ID,
};
Expand Down Expand Up @@ -546,7 +546,7 @@ pub(crate) mod tests {
assert_ingest_events(
&s3_object_results[0],
&s3_object_results[1],
&FlatS3EventMessage::default_version_id(),
&default_version_id(),
);
}

Expand Down Expand Up @@ -577,7 +577,7 @@ pub(crate) mod tests {
assert_ingest_events(
&s3_object_results[0],
&s3_object_results[1],
&FlatS3EventMessage::default_version_id(),
&default_version_id(),
);
}

Expand Down Expand Up @@ -617,7 +617,7 @@ pub(crate) mod tests {
assert_ingest_events(
&s3_object_results[0],
&s3_object_results[1],
&FlatS3EventMessage::default_version_id(),
&default_version_id(),
);
}

Expand Down Expand Up @@ -645,7 +645,7 @@ pub(crate) mod tests {
assert_ingest_events(
&s3_object_results[0],
&s3_object_results[1],
&FlatS3EventMessage::default_version_id(),
&default_version_id(),
);
}

Expand Down Expand Up @@ -826,7 +826,7 @@ pub(crate) mod tests {
assert_missing_deleted(
&s3_object_results[0],
&s3_object_results[1],
&FlatS3EventMessage::default_version_id(),
&default_version_id(),
);
}

Expand All @@ -851,7 +851,7 @@ pub(crate) mod tests {
assert_missing_deleted(
&s3_object_results[0],
&s3_object_results[1],
&FlatS3EventMessage::default_version_id(),
&default_version_id(),
);
}

Expand All @@ -877,7 +877,7 @@ pub(crate) mod tests {
assert_missing_created(
&s3_object_results[0],
&s3_object_results[1],
&FlatS3EventMessage::default_version_id(),
&default_version_id(),
);
}

Expand All @@ -902,7 +902,7 @@ pub(crate) mod tests {
assert_missing_created(
&s3_object_results[0],
&s3_object_results[1],
&FlatS3EventMessage::default_version_id(),
&default_version_id(),
);
}

Expand Down Expand Up @@ -1115,15 +1115,10 @@ pub(crate) mod tests {
.with_sequencer(Some("1".to_string())),
];

let message = expected_message(
None,
FlatS3EventMessage::default_version_id(),
false,
Created,
)
.with_sha256(None)
.with_e_tag(None)
.with_last_modified_date(None);
let message = expected_message(None, default_version_id(), false, Created)
.with_sha256(None)
.with_e_tag(None)
.with_last_modified_date(None);
// 720 permutations
run_permutation_test(&pool, event_permutations, 6, |s3_object_results| {
assert_row(
Expand Down Expand Up @@ -1359,7 +1354,7 @@ pub(crate) mod tests {
events
.version_ids
.iter_mut()
.for_each(|version_id| *version_id = FlatS3EventMessage::default_version_id());
.for_each(|version_id| *version_id = default_version_id());

events
}
Expand Down Expand Up @@ -1469,7 +1464,7 @@ pub(crate) mod tests {
.with_size(size)
.with_version_id(version_id)
.with_last_modified_date(Some(DateTime::<Utc>::default()))
.with_e_tag(Some(EXPECTED_E_TAG.to_string()))
.with_e_tag(Some(EXPECTED_QUOTED_E_TAG.to_string()))
.with_sha256(Some(EXPECTED_SHA256.to_string()))
.with_is_delete_marker(is_delete_marker)
.with_event_type(event_type)
Expand Down
45 changes: 19 additions & 26 deletions lib/workload/stateless/stacks/filemanager/filemanager/src/database/aws/ingester_paired.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,12 +272,14 @@ pub(crate) mod tests {
use crate::database::aws::ingester::tests::fetch_results;
use crate::database::aws::migration::tests::MIGRATOR;
use crate::database::{Client, Ingest};
use crate::events::aws::message::default_version_id;
use crate::events::aws::message::EventType::{Created, Deleted};
use crate::events::aws::tests::{
expected_event_record_simple, expected_flat_events, expected_flat_events_simple,
EXPECTED_QUOTED_E_TAG,
};
use crate::events::aws::tests::{
EXPECTED_E_TAG, EXPECTED_SEQUENCER_CREATED_ONE, EXPECTED_SEQUENCER_CREATED_ZERO,
EXPECTED_SEQUENCER_CREATED_ONE, EXPECTED_SEQUENCER_CREATED_ZERO,
EXPECTED_SEQUENCER_DELETED_ONE, EXPECTED_SEQUENCER_DELETED_TWO, EXPECTED_SHA256,
EXPECTED_VERSION_ID,
};
Expand Down Expand Up @@ -684,10 +686,7 @@ pub(crate) mod tests {
assert_eq!(object_results.len(), 1);
assert_eq!(s3_object_results.len(), 1);
assert_eq!(0, s3_object_results[0].get::<i64, _>("number_reordered"));
assert_ingest_events(
&s3_object_results[0],
&FlatS3EventMessage::default_version_id(),
);
assert_ingest_events(&s3_object_results[0], &default_version_id());
}

#[sqlx::test(migrator = "MIGRATOR")]
Expand All @@ -710,10 +709,7 @@ pub(crate) mod tests {
2,
s3_object_results[0].get::<i64, _>("number_duplicate_events")
);
assert_ingest_events(
&s3_object_results[0],
&FlatS3EventMessage::default_version_id(),
);
assert_ingest_events(&s3_object_results[0], &default_version_id());
}

#[sqlx::test(migrator = "MIGRATOR")]
Expand Down Expand Up @@ -747,10 +743,7 @@ pub(crate) mod tests {
2,
s3_object_results[0].get::<i64, _>("number_duplicate_events")
);
assert_ingest_events(
&s3_object_results[0],
&FlatS3EventMessage::default_version_id(),
);
assert_ingest_events(&s3_object_results[0], &default_version_id());
}

#[sqlx::test(migrator = "MIGRATOR")]
Expand Down Expand Up @@ -782,7 +775,7 @@ pub(crate) mod tests {
Some(0),
Some(EXPECTED_SEQUENCER_CREATED_ONE.to_string()),
Some(EXPECTED_SEQUENCER_DELETED_ONE.to_string()),
FlatS3EventMessage::default_version_id(),
default_version_id(),
Some(Default::default()),
Some(Default::default()),
);
Expand Down Expand Up @@ -987,7 +980,7 @@ pub(crate) mod tests {
assert_missing_deleted(
&s3_object_results[0],
&s3_object_results[1],
&FlatS3EventMessage::default_version_id(),
&default_version_id(),
);
}

Expand Down Expand Up @@ -1017,7 +1010,7 @@ pub(crate) mod tests {
assert_missing_deleted(
&s3_object_results[1],
&s3_object_results[0],
&FlatS3EventMessage::default_version_id(),
&default_version_id(),
);
}

Expand Down Expand Up @@ -1046,7 +1039,7 @@ pub(crate) mod tests {
assert_missing_created(
&s3_object_results[0],
&s3_object_results[1],
&FlatS3EventMessage::default_version_id(),
&default_version_id(),
);
}

Expand Down Expand Up @@ -1076,7 +1069,7 @@ pub(crate) mod tests {
assert_missing_created(
&s3_object_results[1],
&s3_object_results[0],
&FlatS3EventMessage::default_version_id(),
&default_version_id(),
);
}

Expand Down Expand Up @@ -1331,7 +1324,7 @@ pub(crate) mod tests {
&s3_object_results,
"key",
"bucket",
&FlatS3EventMessage::default_version_id(),
&default_version_id(),
Some("1"),
Some("2"),
)
Expand All @@ -1340,7 +1333,7 @@ pub(crate) mod tests {
&s3_object_results,
"key",
"bucket",
&FlatS3EventMessage::default_version_id(),
&default_version_id(),
None,
Some("3"),
)
Expand All @@ -1349,7 +1342,7 @@ pub(crate) mod tests {
&s3_object_results,
"key",
"bucket",
&FlatS3EventMessage::default_version_id(),
&default_version_id(),
Some("4"),
None,
)
Expand All @@ -1358,7 +1351,7 @@ pub(crate) mod tests {
&s3_object_results,
"key",
"bucket",
&FlatS3EventMessage::default_version_id(),
&default_version_id(),
Some("5"),
None,
)
Expand All @@ -1367,7 +1360,7 @@ pub(crate) mod tests {
&s3_object_results,
"key1",
"bucket",
&FlatS3EventMessage::default_version_id(),
&default_version_id(),
Some("1"),
None,
)
Expand Down Expand Up @@ -1624,12 +1617,12 @@ pub(crate) mod tests {
.object_deleted
.version_ids
.iter_mut()
.for_each(|version_id| *version_id = FlatS3EventMessage::default_version_id());
.for_each(|version_id| *version_id = default_version_id());
events
.object_created
.version_ids
.iter_mut()
.for_each(|version_id| *version_id = FlatS3EventMessage::default_version_id());
.for_each(|version_id| *version_id = default_version_id());

events
}
Expand Down Expand Up @@ -1728,7 +1721,7 @@ pub(crate) mod tests {
.with_size(size)
.with_version_id(version_id)
.with_last_modified_date(Some(DateTime::<Utc>::default()))
.with_e_tag(Some(EXPECTED_E_TAG.to_string()))
.with_e_tag(Some(EXPECTED_QUOTED_E_TAG.to_string()))
.with_sha256(Some(EXPECTED_SHA256.to_string()))
.with_is_delete_marker(is_delete_marker)
}
Expand Down
Loading

0 comments on commit 3294cab

Please sign in to comment.