Skip to content

Commit

Permalink
Fix S3 test, cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
joepio committed Jan 18, 2024
1 parent da4a05b commit c1ade8c
Show file tree
Hide file tree
Showing 4 changed files with 23 additions and 29 deletions.
1 change: 1 addition & 0 deletions docs/src/atomicserver/installation.md
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ ATOMIC_SERVER_URL=https://example.com
```

### Configuring S3 for File Storage

You can configure atomic-server to use S3 (and compatible services) for file storage via environment variables or command line arguments when starting atomic-server.

Credentials can either be found in the standard location for AWS credentials on your OS (e.g. `~/.aws/credentials` on UNIX systems) or by using the environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`.
Expand Down
16 changes: 8 additions & 8 deletions server/src/files.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ impl FileStore {
if let FileStore::FS(config) = self {
let fs_file_id = file_id.strip_prefix(Self::FS_PREFIX).unwrap_or(file_id);
let mut file_path = config.path.clone();
file_path.push(fs_file_id.to_string());
file_path.push(fs_file_id);
Ok(file_path)
} else {
Err("Wrong FileStore passed to get_fs_file_path".into())
Expand All @@ -82,8 +82,8 @@ impl FileStore {

pub async fn upload_file(&self, file_id: &str, field: Field) -> AtomicServerResult<i64> {
match self {
FileStore::S3(_) => s3_upload(self, &file_id, field).await,
FileStore::FS(config) => fs_upload(self, &config, &file_id, field).await,
FileStore::S3(_) => s3_upload(self, file_id, field).await,
FileStore::FS(config) => fs_upload(self, config, file_id, field).await,
}
}
}
Expand Down Expand Up @@ -130,8 +130,8 @@ async fn s3_upload(
if let FileStore::S3(config) = file_store {
builder.bucket(&config.bucket);
builder.root(&config.path);
config.region.as_ref().map(|r| builder.region(&r));
config.endpoint.as_ref().map(|e| builder.endpoint(&e));
config.region.as_ref().map(|r| builder.region(r));
config.endpoint.as_ref().map(|e| builder.endpoint(e));
} else {
return Err("Uploading to S3 but no S3 config provided".into());
}
Expand All @@ -141,7 +141,7 @@ async fn s3_upload(
let mut len = 0;
while let Some(chunk) = field.next().await {
let data = chunk.map_err(|e| format!("Error while reading multipart data. {}", e))?;
len = len + data.len();
len += data.len();
w.write(data).await?;
}

Expand All @@ -160,8 +160,8 @@ pub async fn get_s3_signed_url(
if let FileStore::S3(config) = file_store {
builder.bucket(&config.bucket);
builder.root(&config.path);
config.region.as_ref().map(|r| builder.region(&r));
config.endpoint.as_ref().map(|e| builder.endpoint(&e));
config.region.as_ref().map(|r| builder.region(r));
config.endpoint.as_ref().map(|e| builder.endpoint(e));
} else {
return Err("Downloading from S3 but no S3 config provided".into());
}
Expand Down
2 changes: 1 addition & 1 deletion server/src/serve.rs
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ pub async fn serve(config: crate::config::Config) -> AtomicServerResult<()> {
tracing::info!("Binding HTTP server to endpoint {}", endpoint);
println!("{}", message);
server
.bind(&format!("{}:{}", config.opts.ip, config.opts.port))
.bind(&endpoint)
.map_err(|e| format!("Cannot bind to endpoint {}: {}", &endpoint, e))?
.shutdown_timeout(TIMEOUT)
.run()
Expand Down
33 changes: 13 additions & 20 deletions server/src/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,9 @@ fn build_request_authenticated(path: &str, appstate: &AppState) -> TestRequest {
prereq.insert_header(("Accept", "application/ad+json"))
}

#[actix_rt::test]
async fn server_tests() {
let unique_string = atomic_lib::utils::random_string(10);
fn build_test_config() -> crate::config::Config {
use clap::Parser;
let unique_string = atomic_lib::utils::random_string(10);
let opts = Opts::parse_from([
"atomic-server",
"--initialize",
Expand All @@ -46,8 +45,16 @@ async fn server_tests() {
let mut config = config::build_config(opts)
.map_err(|e| format!("Initialization failed: {}", e))
.expect("failed init config");

// This prevents folder access issues when running concurrent tests
config.search_index_path = format!("./.temp/{}/search_index", unique_string).into();
config
}

#[actix_rt::test]
async fn server_tests() {
let unique_string = atomic_lib::utils::random_string(10);
let config = build_test_config();

let appstate = crate::appstate::init(config.clone()).expect("failed init appstate");
let data = Data::new(appstate.clone());
Expand Down Expand Up @@ -166,20 +173,7 @@ fn get_body(resp: ServiceResponse) -> String {

#[actix_rt::test]
async fn file_store_tests() {
let unique_string = atomic_lib::utils::random_string(10);
use clap::Parser;
let mut opts = Opts::parse_from([
"atomic-server",
"--initialize",
"--data-dir",
&format!("./.temp/{}/db", unique_string),
"--config-dir",
&format!("./.temp/{}/config", unique_string),
]);

let mut config = config::build_config(opts.clone())
.map_err(|e| format!("Initialization failed: {}", e))
.expect("failed init config");
let mut config = build_test_config();

let fs_store = FileStore::init_fs_from_config(&config);
if let FileStore::FS(fs_config) = &fs_store {
Expand All @@ -206,9 +200,8 @@ async fn file_store_tests() {
.contains("uploads/my-great-file"));

// FileStore::S3 init
opts.s3_bucket = Some("test-bucket".to_string());
opts.s3_path = Some("uploads".to_string());
config.opts = opts;
config.opts.s3_bucket = Some("test-bucket".to_string());
config.opts.s3_path = Some("uploads".to_string());
let appstate = crate::appstate::init(config.clone()).expect("failed init appstate");

let s3_store = FileStore::init_from_config(&config, fs_store.clone());
Expand Down

0 comments on commit c1ade8c

Please sign in to comment.