From 5d72798046a7947ed048ac4df8f10868b5175eb1 Mon Sep 17 00:00:00 2001 From: David Waring Date: Tue, 22 Oct 2024 12:54:27 -0400 Subject: [PATCH 01/19] [WIP] Multi-Trial Generic Upload --- .../Plugin/MultipleTrialDesignGeneric.pm | 498 ++++++++++++++++++ lib/SGN/Controller/AJAX/Trial.pm | 13 +- 2 files changed, 510 insertions(+), 1 deletion(-) create mode 100644 lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm diff --git a/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm b/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm new file mode 100644 index 0000000000..5af4f36c72 --- /dev/null +++ b/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm @@ -0,0 +1,498 @@ +package CXGN::Trial::ParseUpload::Plugin::MultipleTrialDesignGeneric; + +use Moose::Role; +use CXGN::File::Parse; +use CXGN::Stock::StockLookup; +use SGN::Model::Cvterm; +use Data::Dumper; +use CXGN::List::Validate; +use CXGN::Stock::Seedlot; +use CXGN::Calendar; +use CXGN::Trial; + +my @REQUIRED_COLUMNS = qw|trial_name breeding_program location year design_type description accession_name plot_number block_number|; +my @OPTIONAL_COLUMNS = qw|plot_name trial_type plot_width plot_length field_size planting_date transplanting_date harvest_date is_a_control rep_number range_number row_number col_number seedlot_name num_seed_per_plot weight_gram_seed_per_plot entry_number|; +# Any additional columns that are not required or optional will be used as a treatment + +sub _validate_with_plugin { + my $self = shift; + my $filename = $self->get_filename(); + my $schema = $self->get_chado_schema(); + + my %errors; + my @error_messages; + my %warnings; + my @warning_messages; + my %missing_accessions; + + my $calendar_funcs = CXGN::Calendar->new({}); + my $validator = CXGN::List::Validate->new(); + + # Read and parse the upload file + my $parser = CXGN::File::Parse->new( + file => $filename, + required_columns => \@REQUIRED_COLUMNS, + optional_columns => \@OPTIONAL_COLUMNS + ); + my $parsed = $parser->parse(); + my $parsed_errors = $parsed->{errors}; + my $parsed_data = $parsed->{data}; + my $parsed_values = $parsed->{values}; + my $treatments = $parsed->{additional_columns}; + + # Return file parsing errors + if ( $parsed_errors && scalar(@$parsed_errors) > 0 ) { + $errors{'error_messages'} = $parsed_errors; + $self->_set_parse_errors(\%errors); + return; + } + + ## + ## OVERALL VALIDATION + ## These are checks on the unique values of different columns + ## + + # Trial Name: cannot contain spaces or slashes or already exist in the database + my @already_used_trial_names; + my @missing_trial_names = @{$validator->validate($schema,'trials',$parsed_values->{'trial_name'})->{'missing'}}; + my %unused_trial_names = map { $missing_trial_names[$_] => $_ } 0..$#missing_trial_names; + foreach (@{$parsed_values->{'trial_name'}}) { + push(@already_used_trial_names, $_) unless exists $unused_trial_names{$_}; + if ($_ =~ /\s/) { + push @error_messages, "trial_name $_ must not contain spaces."; + } + elsif ($_ =~ /\// || $_ =~ /\\/) { + push @warning_messages, "trial_name $_ contains slashes. Note that slashes can cause problems for third-party applications; however, plotnames can be saved with slashes."; + } + } + if (scalar(@already_used_trial_names) > 0) { + push @error_messages, "Trial name(s) ".join(',',@already_used_trial_names)." are invalid because they are already used in the database."; + } + + # Breeding Program: must already exist in the database + my $breeding_programs_missing = $validator->validate($schema,'breeding_programs',$parsed_values->{'breeding_program'})->{'missing'}; + my @breeding_programs_missing = @{$breeding_programs_missing}; + if (scalar(@breeding_programs_missing) > 0) { + push @error_messages, "Breeding program(s) ".join(',',@breeding_programs_missing)." are not in the database."; + } + + # Location: must already exist in the database + # Transform location abbreviations to full names + my $locations_hashref = $validator->validate($schema,'locations',$parsed_values->{'location'}); + + # Find valid location codes + my @codes = @{$locations_hashref->{'codes'}}; + my %location_code_map; + foreach my $code (@codes) { + my $location_code = $code->[0]; + my $found_location_name = $code->[1]; + $location_code_map{$location_code} = $found_location_name; + push @warning_messages, "File location '$location_code' matches the code for the location named '$found_location_name' and will be substituted if you ignore warnings."; + } + $self->_set_location_code_map(\%location_code_map); + + # Check the missing locations, ignoring matched codes + my @locations_missing = @{$locations_hashref->{'missing'}}; + my @locations_missing_no_codes = grep { !exists $location_code_map{$_} } @locations_missing; + if (scalar(@locations_missing_no_codes) > 0) { + push @error_messages, "Location(s) ".join(',',@locations_missing_no_codes)." are not in the database."; + } + + + + # Year: must be a 4 digit number + foreach (@{$parsed_values->{'year'}}) { + if (!($_ =~ /^\d{4}$/)) { + push @error_messages, "year $_ is not a valid year, must be a 4 digit positive integer."; + } + } + + + + + + + + + # Dates: must be YYYY-MM-DD format + foreach (@{$parsed_values->{'transplanting_date'}}) { + unless ($calendar_funcs->check_value_format($_)) { + push @error_messages, "transplanting_date $_ must be in the format YYYY-MM-DD."; + } + } + foreach (@{$parsed_values->{'planting_date'}}) { + unless ($calendar_funcs->check_value_format($_)) { + push @error_messages, "planting_date $_ must be in the format YYYY-MM-DD."; + } + } + foreach (@{$parsed_values->{'harvest_date'}}) { + unless ($calendar_funcs->check_value_format($_)) { + push @error_messages, "harvest_date $_ must be in the format YYYY-MM-DD."; + } + } + + + + + ## + ## ROW BY ROW VALIDATION + ## These are checks on the individual plot-level data + ## + + + # NOT FINISHED! + push(@error_messages, "Generic Trial Upload not fully implemented!"); + + + if (scalar(@warning_messages) >= 1) { + $warnings{'warning_messages'} = \@warning_messages; + $self->_set_parse_warnings(\%warnings); + } + if (scalar(@error_messages) >= 1) { + $errors{'error_messages'} = \@error_messages; + $self->_set_parse_errors(\%errors); + return; + } + + return 1; #returns true if validation is passed +} + + +sub _parse_with_plugin { + my $self = shift; + my $filename = $self->get_filename(); + my $schema = $self->get_chado_schema(); + + # Match a dot, extension .xls / .xlsx + my ($extension) = $filename =~ /(\.[^.]+)$/; + my $parser; + + if ($extension eq '.xlsx') { + $parser = Spreadsheet::ParseXLSX->new(); + } + else { + $parser = Spreadsheet::ParseExcel->new(); + } + + my $excel_obj; + my $worksheet; + + $excel_obj = $parser->parse($filename); + if ( !$excel_obj ) { + return; + } + + $worksheet = ( $excel_obj->worksheets() )[0]; + my ( $row_min, $row_max ) = $worksheet->row_range(); + + my $headers = _parse_headers($worksheet); + my %columns = %{$headers->{columns}}; + my @treatment_names = @{$headers->{treatments}}; + + my %seen_accession_names; + for my $row ( 1 .. $row_max ) { + my $accession_name; + if ($worksheet->get_cell($row,$columns{accession_name}->{index})) { + $accession_name = $worksheet->get_cell($row,$columns{accession_name}->{index})->value(); + $accession_name =~ s/^\s+|\s+$//g; #trim whitespace from front and end... + $seen_accession_names{$accession_name}++; + } + } + my $accession_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'accession', 'stock_type')->cvterm_id(); + my $synonym_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'stock_synonym', 'stock_property')->cvterm_id(); + + my @accessions = keys %seen_accession_names; + my $acc_synonym_rs = $schema->resultset("Stock::Stock")->search({ + 'me.is_obsolete' => { '!=' => 't' }, + 'stockprops.value' => { -in => \@accessions}, + 'me.type_id' => $accession_cvterm_id, + 'stockprops.type_id' => $synonym_cvterm_id + },{join => 'stockprops', '+select'=>['stockprops.value'], '+as'=>['synonym']}); + my %acc_synonyms_lookup; + while (my $r=$acc_synonym_rs->next){ + $acc_synonyms_lookup{$r->get_column('synonym')}->{$r->uniquename} = $r->stock_id; + } + + my %all_designs; + my %single_design; + my %design_details; + my $trial_name = ''; + my $breeding_program; + my $location; + my $year; + my $design_type; + my $description; + my @valid_trial_types = CXGN::Trial::get_all_project_types($schema); + my %trial_type_map = map { @{$_}[1] => @{$_}[0] } @valid_trial_types; + my $trial_type; + my $plot_width; + my $plot_length; + my $field_size; + my $planting_date; + my $harvest_date; + my %seen_entry_numbers; + + for my $row ( 1 .. $row_max ) { + + my $current_trial_name; + my $plot_name; + my $accession_name; + my $plot_number; + my $block_number; + my $is_a_control; + my $rep_number; + my $range_number; + my $row_number; + my $col_number; + my $seedlot_name; + my $num_seed_per_plot = 0; + my $weight_gram_seed_per_plot = 0; + my $entry_number; + + if ($worksheet->get_cell($row,$columns{trial_name}->{index})) { + $current_trial_name = $worksheet->get_cell($row,$columns{trial_name}->{index})->value(); + } + + if ($current_trial_name && $current_trial_name ne $trial_name) { + + if ($trial_name) { + ## Save old single trial hash in all trials hash; reinitialize temp hashes + my %final_design_details = (); + my $previous_design_data = $all_designs{$trial_name}{'design_details'}; + if ($previous_design_data) { + %final_design_details = (%design_details, %{$all_designs{$trial_name}{'design_details'}}); + } else { + %final_design_details = %design_details; + } + %design_details = (); + $single_design{'design_details'} = \%final_design_details; + $single_design{'entry_numbers'} = $seen_entry_numbers{$trial_name}; + my %final_single_design = %single_design; + %single_design = (); + $all_designs{$trial_name} = \%final_single_design; + } + + # Get location and replace codes with names + my $location = $worksheet->get_cell($row,$columns{location}->{index})->value(); + if ( $self->_has_location_code_map() ) { + my $location_code_map = $self->_get_location_code_map(); + if ( exists $location_code_map->{$location} ) { + $location = $location_code_map->{$location}; + } + } + + $single_design{'breeding_program'} = $worksheet->get_cell($row,$columns{breeding_program}->{index})->value(); + $single_design{'location'} = $location; + $single_design{'year'} = $worksheet->get_cell($row,$columns{year}->{index})->value(); + # $single_design{'transplanting_date'} = $worksheet->get_cell($row,$columns{transplanting_date}->{index})->value(); + $single_design{'design_type'} = $worksheet->get_cell($row,$columns{design_type}->{index})->value(); + $single_design{'description'} = $worksheet->get_cell($row,$columns{description}->{index})->value(); + + + # for a moment transplanting_date is moves as not required but whole design of that features must be redone + # including use cases + if ($worksheet->get_cell($row,$columns{transplanting_date}->{index})) { + $single_design{'transplanting_date'} = $worksheet->get_cell($row,$columns{transplanting_date}->{index})->value(); + } + + if ($worksheet->get_cell($row,$columns{trial_type}->{index})) { # get and save trial type cvterm_id using trial type name + my $trial_type_id = $trial_type_map{$worksheet->get_cell($row,$columns{trial_type}->{index})->value()}; + $single_design{'trial_type'} = $trial_type_id; + } + if ($worksheet->get_cell($row,$columns{plot_width}->{index})) { + $single_design{'plot_width'} = $worksheet->get_cell($row,$columns{plot_width}->{index})->value(); + } + if ($worksheet->get_cell($row,$columns{plot_length}->{index})) { + $single_design{'plot_length'} = $worksheet->get_cell($row,$columns{plot_length}->{index})->value(); + } + if ($worksheet->get_cell($row,$columns{field_size}->{index})) { + $single_design{'field_size'} = $worksheet->get_cell($row,$columns{field_size}->{index})->value(); + } + if ($worksheet->get_cell($row,$columns{planting_date}->{index})) { + $single_design{'planting_date'} = $worksheet->get_cell($row,$columns{planting_date}->{index})->value(); + } + if ($worksheet->get_cell($row,$columns{harvest_date}->{index})) { + $single_design{'harvest_date'} = $worksheet->get_cell($row,$columns{harvest_date}->{index})->value(); + } + ## Update trial name + $trial_name = $current_trial_name; + } + + #skip blank rows + my $has_row_value; + foreach (keys %columns) { + if ( $worksheet->get_cell($row,$columns{$_}->{index})) { + if ( $worksheet->get_cell($row,$columns{$_}->{index})->value() ) { + $has_row_value = 1; + } + } + } + if ( !$has_row_value ) { + next; + } + + if ($worksheet->get_cell($row,$columns{plot_number}->{index})) { + $plot_number = $worksheet->get_cell($row,$columns{plot_number}->{index})->value(); + } + if ($worksheet->get_cell($row,$columns{plot_name}->{index})) { + $plot_name = $worksheet->get_cell($row,$columns{plot_name}->{index})->value(); + } + if (!$plot_name || $plot_name eq '') { + $plot_name = _create_plot_name($current_trial_name, $plot_number); + } + $plot_name =~ s/^\s+|\s+$//g; #trim whitespace from front and end... + if ($worksheet->get_cell($row,$columns{accession_name}->{index})) { + $accession_name = $worksheet->get_cell($row,$columns{accession_name}->{index})->value(); + } + $accession_name =~ s/^\s+|\s+$//g; #trim whitespace from front and end... + if ($worksheet->get_cell($row,$columns{block_number}->{index})) { + $block_number = $worksheet->get_cell($row,$columns{block_number}->{index})->value(); + } + if ($worksheet->get_cell($row,$columns{is_a_control}->{index})) { + $is_a_control = $worksheet->get_cell($row,$columns{is_a_control}->{index})->value(); + } + if ($worksheet->get_cell($row,$columns{rep_number}->{index})) { + $rep_number = $worksheet->get_cell($row,$columns{rep_number}->{index})->value(); + } + if ($worksheet->get_cell($row,$columns{range_number}->{index})) { + $range_number = $worksheet->get_cell($row,$columns{range_number}->{index})->value(); + } + if ($worksheet->get_cell($row,$columns{row_number}->{index})) { + $row_number = $worksheet->get_cell($row, $columns{row_number}->{index})->value(); + } + if ($worksheet->get_cell($row,$columns{col_number}->{index})) { + $col_number = $worksheet->get_cell($row, $columns{col_number}->{index})->value(); + } + if ($worksheet->get_cell($row,$columns{seedlot_name}->{index})) { + $seedlot_name = $worksheet->get_cell($row, $columns{seedlot_name}->{index})->value(); + } + if ($worksheet->get_cell($row,$columns{entry_number}->{index})) { + $entry_number = $worksheet->get_cell($row, $columns{entry_number}->{index})->value(); + } + + if ($seedlot_name){ + $seedlot_name =~ s/^\s+|\s+$//g; #trim whitespace from front and end... + } + if ($worksheet->get_cell($row,$columns{num_seed_per_plot}->{index})) { + $num_seed_per_plot = $worksheet->get_cell($row, $columns{num_seed_per_plot}->{index})->value(); + } + if ($worksheet->get_cell($row,$columns{weight_gram_seed_per_plot}->{index})) { + $weight_gram_seed_per_plot = $worksheet->get_cell($row, $columns{weight_gram_seed_per_plot}->{index})->value(); + } + + if ($entry_number) { + $seen_entry_numbers{$current_trial_name}->{$accession_name} = $entry_number; + } + + foreach my $treatment_name (@treatment_names){ + my $treatment_col = $columns{$treatment_name}->{index}; + if($worksheet->get_cell($row,$treatment_col)){ + if($worksheet->get_cell($row,$treatment_col)->value()){ + push @{$design_details{treatments}->{$treatment_name}{new_treatment_stocks}}, $plot_name; + } + } + } + + if ($acc_synonyms_lookup{$accession_name}){ + my @accession_names = keys %{$acc_synonyms_lookup{$accession_name}}; + if (scalar(@accession_names)>1){ + print STDERR "There is more than one uniquename for this synonym $accession_name. this should not happen!\n"; + } + $accession_name = $accession_names[0]; + } + + my $key = $row; + $design_details{$key}->{plot_name} = $plot_name; + $design_details{$key}->{stock_name} = $accession_name; + $design_details{$key}->{plot_number} = $plot_number; + $design_details{$key}->{block_number} = $block_number; + if ($is_a_control) { + $design_details{$key}->{is_a_control} = 1; + } else { + $design_details{$key}->{is_a_control} = 0; + } + if ($rep_number) { + $design_details{$key}->{rep_number} = $rep_number; + } + if ($range_number) { + $design_details{$key}->{range_number} = $range_number; + } + if ($row_number) { + $design_details{$key}->{row_number} = $row_number; + } + if ($col_number) { + $design_details{$key}->{col_number} = $col_number; + } + if ($seedlot_name){ + $design_details{$key}->{seedlot_name} = $seedlot_name; + $design_details{$key}->{num_seed_per_plot} = $num_seed_per_plot; + $design_details{$key}->{weight_gram_seed_per_plot} = $weight_gram_seed_per_plot; + } + + } + + # add last trial design to all_designs and save parsed data, then return + my %final_design_details = (); + my $previous_design_data = $all_designs{$trial_name}{'design_details'}; + if ($previous_design_data) { + %final_design_details = (%design_details, %{$all_designs{$trial_name}{'design_details'}}); + } else { + %final_design_details = %design_details; + } + $single_design{'design_details'} = \%final_design_details; + $single_design{'entry_numbers'} = $seen_entry_numbers{$trial_name}; + $all_designs{$trial_name} = \%single_design; + + $self->_set_parsed_data(\%all_designs); + + return 1; + +} + +sub _parse_headers { + my $worksheet = shift; + my ( $col_min, $col_max ) = $worksheet->col_range(); + my %columns; + my @treatments; + my @errors; + + for ( $col_min .. $col_max ) { + if ( $worksheet->get_cell(0,$_) ) { + my $header = $worksheet->get_cell(0,$_)->value(); + $header =~ s/^\s+|\s+$//g; + my $is_required = !!grep( /^$header$/, @REQUIRED_COLUMNS ); + my $is_optional = !!grep( /^$header$/, @OPTIONAL_COLUMNS ); + my $is_treatment = !grep( /^$header$/, @REQUIRED_COLUMNS ) && !grep( /^$header$/, @OPTIONAL_COLUMNS ); + $columns{$header} = { + header => $header, + index => $_, + is_required => $is_required, + is_optional => $is_optional, + is_treatment => $is_treatment + }; + if ( $is_treatment ) { + push(@treatments, $header); + } + } + } + + foreach (@REQUIRED_COLUMNS) { + if ( !exists $columns{$_} ) { + push(@errors, "Required column $_ is missing from the file!"); + } + } + + return { + columns => \%columns, + treatments => \@treatments, + errors => \@errors + } +} + +sub _create_plot_name { + my $trial_name = shift; + my $plot_number = shift; + return $trial_name . "-PLOT_" . $plot_number; +} + +1; diff --git a/lib/SGN/Controller/AJAX/Trial.pm b/lib/SGN/Controller/AJAX/Trial.pm index 3e9e3e3210..4def815a02 100644 --- a/lib/SGN/Controller/AJAX/Trial.pm +++ b/lib/SGN/Controller/AJAX/Trial.pm @@ -1241,9 +1241,20 @@ sub upload_multiple_trial_designs_file_POST : Args(0) { #parse uploaded file with appropriate plugin $parser = CXGN::Trial::ParseUpload->new(chado_schema => $chado_schema, filename => $archived_filename_with_path); - $parser->load_plugin('MultipleTrialDesignExcelFormat'); + $parser->load_plugin('MultipleTrialDesignGeneric'); $parsed_data = $parser->parse(); + print STDERR "\n\n\n\n===> PARSED TRIAL DATA:\n"; + print STDERR Dumper $parsed_data; + print STDERR "\nErrors:\n"; + print STDERR Dumper $parser->get_parse_errors(); + print STDERR "\nWarnings:\n"; + print STDERR Dumper $parser->get_parse_warnings(); + print STDERR "\n\n\n\n"; + + $c->stash->{rest} = {errors => "Upload not implemented"}; + return; + # print STDERR "check the parsed data : \n" . Dumper($parsed_data); if (!$parsed_data) { my $return_error = ''; From 174c4b161a49a77aebeef5e4af181b26c6d9a565 Mon Sep 17 00:00:00 2001 From: David Waring Date: Wed, 30 Oct 2024 16:29:21 -0400 Subject: [PATCH 02/19] [WIP] Multi-Trial Generic Upload --- .../Plugin/MultipleTrialDesignGeneric.pm | 326 ++++++++++++++++-- 1 file changed, 288 insertions(+), 38 deletions(-) diff --git a/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm b/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm index 5af4f36c72..0c6ae1db4a 100644 --- a/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm +++ b/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm @@ -14,19 +14,44 @@ my @REQUIRED_COLUMNS = qw|trial_name breeding_program location year design_type my @OPTIONAL_COLUMNS = qw|plot_name trial_type plot_width plot_length field_size planting_date transplanting_date harvest_date is_a_control rep_number range_number row_number col_number seedlot_name num_seed_per_plot weight_gram_seed_per_plot entry_number|; # Any additional columns that are not required or optional will be used as a treatment +# VALID DESIGN TYPES +my %valid_design_types = ( + "CRD" => 1, + "RCBD" => 1, + "RRC" => 1, + "DRRC" => 1, + "ARC" => 1, + "Alpha" => 1, + "Lattice" => 1, + "Augmented" => 1, + "MAD" => 1, + "genotyping_plate" => 1, + "greenhouse" => 1, + "p-rep" => 1, + "splitplot" => 1, + "stripplot" => 1, + "Westcott" => 1, + "Analysis" => 1 +); + sub _validate_with_plugin { my $self = shift; my $filename = $self->get_filename(); my $schema = $self->get_chado_schema(); + # Date and List validators + my $calendar_funcs = CXGN::Calendar->new({}); + my $validator = CXGN::List::Validate->new(); + + # Valid Trial Types + my @valid_trial_types = CXGN::Trial::get_all_project_types($schema); + my %valid_trial_types = map { @{$_}[1] => 1 } @valid_trial_types; + + # Encountered Error and Warning Messages my %errors; my @error_messages; my %warnings; my @warning_messages; - my %missing_accessions; - - my $calendar_funcs = CXGN::Calendar->new({}); - my $validator = CXGN::List::Validate->new(); # Read and parse the upload file my $parser = CXGN::File::Parse->new( @@ -35,10 +60,10 @@ sub _validate_with_plugin { optional_columns => \@OPTIONAL_COLUMNS ); my $parsed = $parser->parse(); - my $parsed_errors = $parsed->{errors}; - my $parsed_data = $parsed->{data}; - my $parsed_values = $parsed->{values}; - my $treatments = $parsed->{additional_columns}; + my $parsed_errors = $parsed->{'errors'}; + my $parsed_data = $parsed->{'data'}; + my $parsed_values = $parsed->{'values'}; + my $treatments = $parsed->{'additional_columns'}; # Return file parsing errors if ( $parsed_errors && scalar(@$parsed_errors) > 0 ) { @@ -47,12 +72,217 @@ sub _validate_with_plugin { return; } + # Maps of plot-level data to use in overall validation + my %seen_plot_numbers; # check for a plot numbers: used only once per trial + my %seen_plot_names; # check for plot names: used only once per trial + my %seen_plot_positions; # check for plot row / col positions: each position only used once per trial + my %seen_entry_numbers; # check for entry numbers: used only once per trial + my @seedlot_pairs; # 2D array of [seedlot_name, accession_name] + + ## + ## ROW BY ROW VALIDATION + ## These are checks on the individual plot-level data + ## + foreach (@$parsed_data) { + my $row = $_->{'_row'}; + my $trial_name = $_->{'trial_name'}; + my $breeding_program = $_->{'breeding_program'}; + my $location = $_->{'location'}; + my $year = $_->{'year'}; + my $design_type = $_->{'design_type'}; + my $description = $_->{'description'}; + my $accession_name = $_->{'accession_name'}; + my $plot_number = $_->{'plot_number'}; + my $block_number = $_->{'block_number'}; + my $plot_name = $_->{'plot_name'}; + my $trial_type = $_->{'trial_type'}; + my $plot_width = $_->{'plot_width'}; + my $plot_length = $_->{'plot_length'}; + my $field_size = $_->{'field_size'}; + my $planting_date = $_->{'planting_date'}; + my $transplanting_date = $_->{'transplanting_date'}; + my $harvest_date = $_->{'harvest_date'}; + my $is_a_control = $_->{'is_a_control'}; + my $rep_number = $_->{'rep_number'}; + my $range_number = $_->{'range_number'}; + my $row_number = $_->{'row_number'}; + my $col_number = $_->{'col_number'}; + my $seedlot_name = $_->{'seedlot_name'}; + my $num_seed_per_plot = $_->{'num_seed_per_plot'}; + my $weight_gram_seed_per_plot = $_->{'weight_gram_seed_per_plot'}; + my $entry_number = $_->{'entry_number'}; + + # TODO: Remove + print STDERR "ROW: $row = $trial_name / $plot_name / $accession_name / $plot_number\n"; + + # Plot Number: must be a positive number + if (!($plot_number =~ /^\d+?$/)) { + push @error_messages, "Row $row: plot number $plot_number must be a positive integer."; + } + + # Block Number: must be a positive integer + if (!($block_number =~ /^\d+?$/)) { + push @error_messages, "Row $row: block number $block_number must be a positive integer."; + } + + # Rep Number: must be a positive integer, if provided + if ($rep_number && !($rep_number =~ /^\d+?$/)){ + push @error_messages, "Row $row: rep_number $rep_number must be a positive integer."; + } + + # Plot Name: cannot contain spaces, should not contain slashes + if ($plot_name =~ /\s/ ) { + push @error_messages, "Row $row: plot name $plot_name must not contain spaces."; + } + if ($plot_name =~ /\// || $plot_name =~ /\\/) { + push @warning_messages, "Row $row: plot name $plot_name contains slashes. Note that slashes can cause problems for third-party applications; however, plot names can be saved with slashes if you ignore warnings."; + } + + # Plot Width / Plot Length / Field Size: must be a positive number, if provided + if ($plot_width && !($plot_width =~ /^([\d]*)([\.]?)([\d]+)$/)) { + push @error_messages, "Row $row: plot_width $plot_width must be a positive number."; + } + if ($plot_length && !($plot_length =~ /^([\d]*)([\.]?)([\d]+)$/)) { + push @error_messages, "Row $row: plot_length $plot_length must be a positive number."; + } + if ($field_size && !($field_size =~ /^([\d]*)([\.]?)([\d]+)$/)) { + push @error_messages, "Row $row: plot_width $field_size must be a positive number."; + } + + # Transplanting / Planting / Harvest Dates: must be YYYY-MM-DD format, if provided + if ($transplanting_date && !$calendar_funcs->check_value_format($transplanting_date)) { + push @error_messages, "Row $row: transplanting_date $transplanting_date must be in the format YYYY-MM-DD."; + } + if ($planting_date && !$calendar_funcs->check_value_format($planting_date)) { + push @error_messages, "Row $row: planting_date $planting_date must be in the format YYYY-MM-DD."; + } + if ($harvest_date && !$calendar_funcs->check_value_format($harvest_date)) { + push @error_messages, "Row $row: harvest_date $harvest_date must be in the format YYYY-MM-DD."; + } + + # Is A Control: must be blank, 0, or 1, if provided + if ( $is_a_control && $is_a_control ne '' && $is_a_control ne '0' && $is_a_control ne '1' ) { + push @error_messages, "Row $row: is_a_control value of $is_a_control is invalid. It must be blank (not a control), 0 (not a control), or 1 (is a control)."; + } + + # Range Number / Row Number / Col Number: must be a positive integer, if provided + if ($range_number && !($range_number =~ /^\d+?$/)) { + push @error_messages, "Row $row: range_number $range_number must be a positive integer."; + } + if ($row_number && !($row_number =~ /^\d+?$/)) { + push @error_messages, "Row $row: row_number $row_number must be a positive integer."; + } + if ($col_number && !($col_number =~ /^\d+?$/)) { + push @error_messages, "Row $row: col_number $col_number must be a positive integer."; + } + + # Seedlots: add seedlot_name / accession_name to seedlot_pairs + # count and weight must be a positive integer + # return a warning if both count and weight are not provided + if ( $seedlot_name ) { + push @seedlot_pairs, [$seedlot_name, $accession_name]; + if ( $num_seed_per_plot && $num_seed_per_plot ne '' && !($num_seed_per_plot =~ /^\d+?$/) ) { + push @error_messages, "Row $row: num_seed_per_plot $num_seed_per_plot must be a positive integer."; + } + if ( $weight_gram_seed_per_plot && $weight_gram_seed_per_plot ne '' && !($weight_gram_seed_per_plot =~ /^\d+?$/) ) { + push @error_messages, "Row $row: weight_gram_seed_per_plot $weight_gram_seed_per_plot must be a positive integer."; + } + if ( !$num_seed_per_plot && !$weight_gram_seed_per_plot ) { + push @warning_messages, "Row $row: this plot does not have a count or weight of seed to be used from seedlot $seedlot_name." + } + } + + # Entry Number: must be a positive integer, if provided + if ($entry_number && !($entry_number =~ /^\d+?$/)) { + push @error_messages, "Row $row: entry_number $entry_number must be a positive integer."; + } + + # Treatment Values: must be either blank, 0, or 1 + foreach my $treatment (@$treatments) { + my $treatment_value = $row->{$treatment}; + print STDERR "Row $row: $treatment = $treatment_value\n"; + if ( $treatment_value && $treatment_value ne '' && $treatment_value ne '0' && $treatment_value ne '1' ) { + push @error_messages, "Row $row: Treatment value for treatment $treatment should be either 1 (applied) or empty (not applied)."; + } + } + + + # Create maps to check for overall validation within individual trials + my $tk = $trial_name; + + # Map to check for duplicated plot numbers + if ( $plot_number ) { + my $pk = $plot_number; + if ( !exists $seen_plot_numbers{$tk} ) { + $seen_plot_numbers{$tk} = {}; + } + if ( !exists $seen_plot_numbers{$tk}{$pk} ) { + $seen_plot_numbers{$tk}{$pk} = 1; + } + else { + $seen_plot_numbers{$tk}{$pk}++; + } + } + + # Map to check for duplicated plot names + if ( $plot_name ) { + my $pk = $plot_name; + if ( !exists $seen_plot_names{$tk} ) { + $seen_plot_names{$tk} = {}; + } + if ( !exists $seen_plot_names{$tk}{$pk} ) { + $seen_plot_names{$tk}{$pk} = [$plot_number]; + } + else { + push @{$seen_plot_names{$tk}{$pk}}, $plot_number; + } + } + + # Map to check for overlapping plots + if ( $row_number && $col_number ) { + my $pk = "$row_number-$col_number"; + if ( !exists $seen_plot_positions{$tk} ) { + $seen_plot_positions{$tk} = {}; + } + if ( !exists $seen_plot_positions{$tk}{$pk} ) { + $seen_plot_positions{$tk}{$pk} = [$plot_number]; + } + else { + push @{$seen_plot_positions{$tk}{$pk}}, $plot_number; + } + } + + # Map to check the entry number <-> accession associations + # For each trial: each entry number should only be associated with one accession + # and each accession should only be associated with one entry number + if ( $entry_number ) { + if ( !exists $seen_entry_numbers{$tk} ) { + $seen_entry_numbers{$tk}->{'by_num'} = {}; + $seen_entry_numbers{$tk}->{'by_acc'} = {}; + } + + if ( !exists $seen_entry_numbers{$tk}->{'by_num'}->{$entry_number} ) { + $seen_entry_numbers{$tk}->{'by_num'}->{$entry_number} = [$accession_name]; + } + else { + push @{$seen_entry_numbers{$tk}->{'by_num'}->{$entry_number}}, $accession_name; + } + + if ( !exists $seen_entry_numbers{$tk}->{'by_acc'}->{$accession_name} ) { + $seen_entry_numbers{$tk}->{'by_acc'}->{$accession_name} = [$entry_number]; + } + else { + push @{$seen_entry_numbers{$tk}->{'by_acc'}->{$accession_name}}, $entry_number; + } + } + } + ## ## OVERALL VALIDATION ## These are checks on the unique values of different columns ## - # Trial Name: cannot contain spaces or slashes or already exist in the database + # Trial Name: cannot already exist in the database, cannot contain spaces, should not contain slashes my @already_used_trial_names; my @missing_trial_names = @{$validator->validate($schema,'trials',$parsed_values->{'trial_name'})->{'missing'}}; my %unused_trial_names = map { $missing_trial_names[$_] => $_ } 0..$#missing_trial_names; @@ -61,8 +291,8 @@ sub _validate_with_plugin { if ($_ =~ /\s/) { push @error_messages, "trial_name $_ must not contain spaces."; } - elsif ($_ =~ /\// || $_ =~ /\\/) { - push @warning_messages, "trial_name $_ contains slashes. Note that slashes can cause problems for third-party applications; however, plotnames can be saved with slashes."; + if ($_ =~ /\// || $_ =~ /\\/) { + push @warning_messages, "trial_name $_ contains slashes. Note that slashes can cause problems for third-party applications; however, trial names can be saved with slashes if you ignore warnings."; } } if (scalar(@already_used_trial_names) > 0) { @@ -76,11 +306,8 @@ sub _validate_with_plugin { push @error_messages, "Breeding program(s) ".join(',',@breeding_programs_missing)." are not in the database."; } - # Location: must already exist in the database - # Transform location abbreviations to full names + # Location: Transform location abbreviations/codes to full names my $locations_hashref = $validator->validate($schema,'locations',$parsed_values->{'location'}); - - # Find valid location codes my @codes = @{$locations_hashref->{'codes'}}; my %location_code_map; foreach my $code (@codes) { @@ -91,57 +318,80 @@ sub _validate_with_plugin { } $self->_set_location_code_map(\%location_code_map); - # Check the missing locations, ignoring matched codes + # Location: must already exist in the database my @locations_missing = @{$locations_hashref->{'missing'}}; my @locations_missing_no_codes = grep { !exists $location_code_map{$_} } @locations_missing; if (scalar(@locations_missing_no_codes) > 0) { push @error_messages, "Location(s) ".join(',',@locations_missing_no_codes)." are not in the database."; } - - - # Year: must be a 4 digit number + # Year: must be a 4 digit integer foreach (@{$parsed_values->{'year'}}) { if (!($_ =~ /^\d{4}$/)) { push @error_messages, "year $_ is not a valid year, must be a 4 digit positive integer."; } } + # Design Type: must be a valid / supported design type + foreach (@{$parsed_values->{'design_type'}}) { + if ( !exists $valid_design_types{$_} ) { + push @error_messages, "design_type $_ is not supported. Supported design types: " . join(', ', keys(%valid_design_types)) . "."; + } + } + # Trial Type: must be a valid / supported trial type + foreach (@{$parsed_values->{'trial_type'}}) { + if ( !exists $valid_trial_types{$_} ) { + push @error_messages, "trial_type $_ is not supported. Supported trial types: " . join(', ', keys(%valid_trial_types)) . "."; + } + } + # Accession Names: must exist in the database + my @accessions = @{$parsed_values->{'accession_name'}}; + my $accessions_hashref = $validator->validate($schema,'accessions',\@accessions); + #find unique synonyms. Sometimes trial uploads use synonym names instead of the unique accession name. We allow this if the synonym is unique and matches one accession in the database + my @synonyms = @{$accessions_hashref->{'synonyms'}}; + foreach my $synonym (@synonyms) { + my $found_acc_name_from_synonym = $synonym->{'uniquename'}; + my $matched_synonym = $synonym->{'synonym'}; + push @warning_messages, "File Accession $matched_synonym is a synonym of database accession $found_acc_name_from_synonym "; + @accessions = grep !/\Q$matched_synonym/, @accessions; + push @accessions, $found_acc_name_from_synonym; + } + #now validate again the accession names + $accessions_hashref = $validator->validate($schema,'accessions',\@accessions); - # Dates: must be YYYY-MM-DD format - foreach (@{$parsed_values->{'transplanting_date'}}) { - unless ($calendar_funcs->check_value_format($_)) { - push @error_messages, "transplanting_date $_ must be in the format YYYY-MM-DD."; - } - } - foreach (@{$parsed_values->{'planting_date'}}) { - unless ($calendar_funcs->check_value_format($_)) { - push @error_messages, "planting_date $_ must be in the format YYYY-MM-DD."; - } + my @accessions_missing = @{$accessions_hashref->{'missing'}}; + my @multiple_synonyms = @{$accessions_hashref->{'multiple_synonyms'}}; + + if (scalar(@accessions_missing) > 0) { + push @error_messages, "Accession(s) ".join(',',@accessions_missing)." are not in the database as uniquenames or synonyms."; } - foreach (@{$parsed_values->{'harvest_date'}}) { - unless ($calendar_funcs->check_value_format($_)) { - push @error_messages, "harvest_date $_ must be in the format YYYY-MM-DD."; + if (scalar(@multiple_synonyms) > 0) { + my @msgs; + foreach my $m (@multiple_synonyms) { + push(@msgs, 'Name: ' . @$m[0] . ' = Synonym: ' . @$m[1]); } + push @error_messages, "Accession(s) ".join(',',@msgs)." appear in the database as synonyms of more than one unique accession. Please change to the unique accession name or delete the multiple synonyms"; } + # Seedlots... + # Check trial-level maps - ## - ## ROW BY ROW VALIDATION - ## These are checks on the individual plot-level data - ## + # NOT FINISHED! + push @error_messages, "Generic Trial Upload not fully implemented!"; - # NOT FINISHED! - push(@error_messages, "Generic Trial Upload not fully implemented!"); + print STDERR "\n\n\n\n=====> WARNINGS:\n"; + print STDERR Dumper \@warning_messages; + print STDERR "\n=====> ERRORS:\n"; + print STDERR Dumper \@error_messages; if (scalar(@warning_messages) >= 1) { From 27a427edb99d1b615f845057c34f2b46921866f5 Mon Sep 17 00:00:00 2001 From: David Waring Date: Fri, 1 Nov 2024 09:00:42 -0400 Subject: [PATCH 03/19] Trial Upload: remove references to top left corner as position 1,1 --- mason/breeders_toolbox/trial/trial_upload_dialogs.mas | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mason/breeders_toolbox/trial/trial_upload_dialogs.mas b/mason/breeders_toolbox/trial/trial_upload_dialogs.mas index 340d8c86f3..f0518889e0 100644 --- a/mason/breeders_toolbox/trial/trial_upload_dialogs.mas +++ b/mason/breeders_toolbox/trial/trial_upload_dialogs.mas @@ -685,8 +685,8 @@ $design_types => ()
  • is_a_control (type 1 in this field if the plot is a control, otherwise leave blank. generally you will have accessions/cross unique ids/family names that are controls, so you should indicate the plots of those accessions/cross unique ids/family names as a control.)
  • rep_number (replicate number, numeric)
  • range_number (range number. often synonymous with col_number, numeric)
    • -
  • row_number (row number. If the field is a grid, this represents the y coordinate, numeric, required for field map generation. the top left plot should be row 1, column 1)
    • -
  • col_number (column number. If the field is a grid, this represents the x coordinate. Sometimes called range_number, numeric, required for field map generation. the top left plot should be row 1, column 1)
    • +
  • row_number (row number. If the field is a grid, this represents the y coordinate, numeric, required for field map generation.)
    • +
  • col_number (column number. If the field is a grid, this represents the x coordinate. Sometimes called range_number, numeric, required for field map generation.)
  • seedlot_name (the seedlot from where the planted seed originated. Must exist in the database)
  • num_seed_per_plot (number seeds per plot. Seed is transferred from seedlot mentioned in seedlot_name. Numeric)
  • weight_gram_seed_per_plot (weight in gram of seeds in plot. seed is transferred from seedlot mentioned in seedlot name. Numeric)
    • @@ -792,8 +792,8 @@ $design_types => ()
  • is_a_control (type 1 in this field if the plot is a control, otherwise 0 or leave blank. generally you will have accessions that are controls, so you should indicate the plots that that accession is in as a control.)
  • rep_number (replicate number, must be numeric)
  • range_number (range number. often synonymous with col_number, must be numeric)
    • -
  • row_number (row number. if the field is a grid, this represents the y coordinate, numeric, required for field map generation. the top left plot shuold be row 1, column 1)
    • -
  • col_number (column number. if the field is a grid, this represents the x coordinate. sometimes called range_number, numeric, required for field map generation. the top left plot shuold be row 1, column 1)
    • +
  • row_number (row number. if the field is a grid, this represents the y coordinate, numeric, required for field map generation.)
    • +
  • col_number (column number. if the field is a grid, this represents the x coordinate. sometimes called range_number, numeric, required for field map generation.)
  • seedlot_name (the seedlot from where the planted seed originated. must exist in the database)
  • num_seed_per_plot (number seeds per plot. seed is transferred from seedlot mentioned in seedlot_name. numeric)
  • weight_gram_seed_per_plot (weight in gram of seeds in plot. seed is transferred from seedlot mentioned in seedlot name. numeric)
    • From d134defafe8b6190a6dc5da3c9b17e53d72f28a4 Mon Sep 17 00:00:00 2001 From: David Waring Date: Fri, 1 Nov 2024 15:04:10 -0400 Subject: [PATCH 04/19] CXGN::Trial::Folder remove debugging output --- lib/CXGN/Trial/Folder.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/CXGN/Trial/Folder.pm b/lib/CXGN/Trial/Folder.pm index 7c0b85f5ca..9a8365b3e6 100644 --- a/lib/CXGN/Trial/Folder.pm +++ b/lib/CXGN/Trial/Folder.pm @@ -678,7 +678,7 @@ sub _jstree_li_html { my $type = shift; my $id = shift; my $name = shift; - print STDERR "TYPE =".Dumper($type)."\n"; + # print STDERR "TYPE =".Dumper($type)."\n"; my $url = '#'; if ($type eq 'trial' || $type eq 'genotyping_trial' || $type eq 'sampling_trial') { From 2cc1739ea8a18ca2fa16fd0f3cb6b225aaa8223a Mon Sep 17 00:00:00 2001 From: David Waring Date: Fri, 1 Nov 2024 15:05:11 -0400 Subject: [PATCH 05/19] Multi-Trial Generic Upload - validation --- .../Plugin/MultipleTrialDesignGeneric.pm | 96 +++++++++++++++---- 1 file changed, 80 insertions(+), 16 deletions(-) diff --git a/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm b/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm index 0c6ae1db4a..b2d3e0899f 100644 --- a/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm +++ b/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm @@ -1,6 +1,7 @@ package CXGN::Trial::ParseUpload::Plugin::MultipleTrialDesignGeneric; use Moose::Role; +use List::MoreUtils qw(uniq); use CXGN::File::Parse; use CXGN::Stock::StockLookup; use SGN::Model::Cvterm; @@ -224,17 +225,14 @@ sub _validate_with_plugin { } } - # Map to check for duplicated plot names + # Map to check for duplicated plot names (unique across all trials) if ( $plot_name ) { my $pk = $plot_name; - if ( !exists $seen_plot_names{$tk} ) { - $seen_plot_names{$tk} = {}; - } - if ( !exists $seen_plot_names{$tk}{$pk} ) { - $seen_plot_names{$tk}{$pk} = [$plot_number]; + if ( !exists $seen_plot_names{$pk} ) { + $seen_plot_names{$pk} = 1; } else { - push @{$seen_plot_names{$tk}{$pk}}, $plot_number; + $seen_plot_names{$pk}++; } } @@ -314,7 +312,7 @@ sub _validate_with_plugin { my $location_code = $code->[0]; my $found_location_name = $code->[1]; $location_code_map{$location_code} = $found_location_name; - push @warning_messages, "File location '$location_code' matches the code for the location named '$found_location_name' and will be substituted if you ignore warnings."; + push @warning_messages, "File location $location_code matches the code for the location named $found_location_name and will be substituted if you ignore warnings."; } $self->_set_location_code_map(\%location_code_map); @@ -379,21 +377,85 @@ sub _validate_with_plugin { push @error_messages, "Accession(s) ".join(',',@msgs)." appear in the database as synonyms of more than one unique accession. Please change to the unique accession name or delete the multiple synonyms"; } - # Seedlots... + # Plot Names: should nnot exist (as any stock) + my $rs = $schema->resultset("Stock::Stock")->search({ + 'is_obsolete' => { '!=' => 't' }, + 'uniquename' => { -in => $parsed_values->{'plot_name'} } + }); + while (my $r=$rs->next) { + push @error_messages, "Plot name ".$r->uniquename." already exists in the database. Each plot must have a new unique name."; + } + + # Seedlots: names must exist in the database + my @seedlots_missing = @{$validator->validate($schema,'seedlots',$parsed_values->{'seedlot_name'})->{'missing'}}; + if (scalar(@seedlots_missing) > 0) { + push @error_messages, "Seedlot(s) ".join(',',@seedlots_missing)." are not in the database. To use a seedlot as a seed source for a plot, the seedlot must already exist in the database."; + } + + # Verify seedlot pairs: accession name of plot must match seedlot contents + if ( scalar(@seedlot_pairs) > 0 ) { + my $return = CXGN::Stock::Seedlot->verify_seedlot_accessions_crosses($schema, \@seedlot_pairs); + if (exists($return->{error})){ + push @error_messages, $return->{error}; + } + } + + # Check for duplicated plot numbers + foreach my $tk (keys %seen_plot_numbers) { + foreach my $pk (keys %{$seen_plot_numbers{$tk}}) { + my $count = $seen_plot_numbers{$tk}{$pk}; + if ( $count > 1 ) { + push @error_messages, "Plot number $pk in trial $tk is used $count times. Each plot should have a unique plot number in an individual trial."; + } + } + } - # Check trial-level maps + # Check for duplicated plot names + foreach my $pk (keys %seen_plot_names) { + my $count = $seen_plot_names{$pk}; + if ( $count > 1 ) { + push @error_messages, "Plot name $pk is used $count times. Each plot should have a unique plot name across all trials."; + } + } + # Check for overlapping plot positions (more than one plot assigned the same row/col positions) + foreach my $tk (keys %seen_plot_positions) { + foreach my $pk (keys %{$seen_plot_positions{$tk}}) { + my $plots = $seen_plot_positions{$tk}{$pk}; + my $count = scalar(@$plots); + if ( $count > 1 ) { + my @pos = split('-', $pk); + push @warning_messages, "More than 1 plot is assigned to the position row=" . $pos[0] . " col=" . $pos[1] . " trial=" . $tk . " plots=" . join(',', @$plots); + } + } + } - # NOT FINISHED! - push @error_messages, "Generic Trial Upload not fully implemented!"; + # Check for entry number errors: + # the same accession assigned different entry numbers + # the same entry number assigned to different accessions + foreach my $tk (keys %seen_entry_numbers) { + # check assignments by accessions + foreach my $an (keys %{$seen_entry_numbers{$tk}{'by_acc'}} ) { + my @ens = uniq @{$seen_entry_numbers{$tk}{'by_acc'}{$an}}; + my $count = scalar(@ens); + if ( scalar(@ens) > 1 ) { + push @error_messages, "Entry Number mismatch: Accession $an has multiple entry numbers (" . join(', ', @ens) . ") assigned to it in trial $tk."; + } + } - print STDERR "\n\n\n\n=====> WARNINGS:\n"; - print STDERR Dumper \@warning_messages; - print STDERR "\n=====> ERRORS:\n"; - print STDERR Dumper \@error_messages; + # check assigments by entry number + foreach my $en (keys %{$seen_entry_numbers{$tk}{'by_num'}} ) { + my @ans = uniq @{$seen_entry_numbers{$tk}{'by_num'}{$en}}; + my $count = scalar(@ans); + if ( scalar(@ans) > 1 ) { + push @error_messages, "Entry Number mismatch: Entry number $en has multiple accessions (" . join(', ', @ans) . ") assigned to it in trial $tk."; + } + } + } + # Return warnings and error messages if (scalar(@warning_messages) >= 1) { $warnings{'warning_messages'} = \@warning_messages; $self->_set_parse_warnings(\%warnings); @@ -413,6 +475,8 @@ sub _parse_with_plugin { my $filename = $self->get_filename(); my $schema = $self->get_chado_schema(); + print STDERR "\n\n\n\n===> PARSE WITH PLUGIN:\n"; + # Match a dot, extension .xls / .xlsx my ($extension) = $filename =~ /(\.[^.]+)$/; my $parser; From be10658e255bc0153b08daa7c730b7322981e49e Mon Sep 17 00:00:00 2001 From: David Waring Date: Fri, 1 Nov 2024 16:06:14 -0400 Subject: [PATCH 06/19] Multi-Trial Generic Upload --- lib/CXGN/Trial/ParseUpload.pm | 9 + .../Plugin/MultipleTrialDesignGeneric.pm | 472 ++++++------------ 2 files changed, 163 insertions(+), 318 deletions(-) diff --git a/lib/CXGN/Trial/ParseUpload.pm b/lib/CXGN/Trial/ParseUpload.pm index abb5842ff5..b57d85a9c3 100644 --- a/lib/CXGN/Trial/ParseUpload.pm +++ b/lib/CXGN/Trial/ParseUpload.pm @@ -49,6 +49,15 @@ has '_parsed_data' => ( predicate => '_has_parsed_data' ); +has '_validated_data' => ( + is => 'ro', + isa => 'HashRef', + writer => '_set_validated_data', + reader => '_get_validated_data', + predicate => '_has_validated_data', + required => 0 +); + has '_location_code_map' => ( is => 'ro', isa => 'HashRef', diff --git a/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm b/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm index b2d3e0899f..d14eac957d 100644 --- a/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm +++ b/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm @@ -113,9 +113,6 @@ sub _validate_with_plugin { my $weight_gram_seed_per_plot = $_->{'weight_gram_seed_per_plot'}; my $entry_number = $_->{'entry_number'}; - # TODO: Remove - print STDERR "ROW: $row = $trial_name / $plot_name / $accession_name / $plot_number\n"; - # Plot Number: must be a positive number if (!($plot_number =~ /^\d+?$/)) { push @error_messages, "Row $row: plot number $plot_number must be a positive integer."; @@ -362,7 +359,6 @@ sub _validate_with_plugin { #now validate again the accession names $accessions_hashref = $validator->validate($schema,'accessions',\@accessions); - my @accessions_missing = @{$accessions_hashref->{'missing'}}; my @multiple_synonyms = @{$accessions_hashref->{'multiple_synonyms'}}; @@ -377,7 +373,7 @@ sub _validate_with_plugin { push @error_messages, "Accession(s) ".join(',',@msgs)." appear in the database as synonyms of more than one unique accession. Please change to the unique accession name or delete the multiple synonyms"; } - # Plot Names: should nnot exist (as any stock) + # Plot Names: should not exist (as any stock) my $rs = $schema->resultset("Stock::Stock")->search({ 'is_obsolete' => { '!=' => 't' }, 'uniquename' => { -in => $parsed_values->{'plot_name'} } @@ -466,341 +462,181 @@ sub _validate_with_plugin { return; } + $self->_set_validated_data($parsed); return 1; #returns true if validation is passed } sub _parse_with_plugin { - my $self = shift; - my $filename = $self->get_filename(); - my $schema = $self->get_chado_schema(); - - print STDERR "\n\n\n\n===> PARSE WITH PLUGIN:\n"; - - # Match a dot, extension .xls / .xlsx - my ($extension) = $filename =~ /(\.[^.]+)$/; - my $parser; - - if ($extension eq '.xlsx') { - $parser = Spreadsheet::ParseXLSX->new(); - } - else { - $parser = Spreadsheet::ParseExcel->new(); - } - - my $excel_obj; - my $worksheet; - - $excel_obj = $parser->parse($filename); - if ( !$excel_obj ) { - return; - } - - $worksheet = ( $excel_obj->worksheets() )[0]; - my ( $row_min, $row_max ) = $worksheet->row_range(); - - my $headers = _parse_headers($worksheet); - my %columns = %{$headers->{columns}}; - my @treatment_names = @{$headers->{treatments}}; - - my %seen_accession_names; - for my $row ( 1 .. $row_max ) { - my $accession_name; - if ($worksheet->get_cell($row,$columns{accession_name}->{index})) { - $accession_name = $worksheet->get_cell($row,$columns{accession_name}->{index})->value(); - $accession_name =~ s/^\s+|\s+$//g; #trim whitespace from front and end... - $seen_accession_names{$accession_name}++; - } - } - my $accession_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'accession', 'stock_type')->cvterm_id(); - my $synonym_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'stock_synonym', 'stock_property')->cvterm_id(); - - my @accessions = keys %seen_accession_names; - my $acc_synonym_rs = $schema->resultset("Stock::Stock")->search({ - 'me.is_obsolete' => { '!=' => 't' }, - 'stockprops.value' => { -in => \@accessions}, - 'me.type_id' => $accession_cvterm_id, - 'stockprops.type_id' => $synonym_cvterm_id - },{join => 'stockprops', '+select'=>['stockprops.value'], '+as'=>['synonym']}); - my %acc_synonyms_lookup; - while (my $r=$acc_synonym_rs->next){ - $acc_synonyms_lookup{$r->get_column('synonym')}->{$r->uniquename} = $r->stock_id; - } - - my %all_designs; - my %single_design; - my %design_details; - my $trial_name = ''; - my $breeding_program; - my $location; - my $year; - my $design_type; - my $description; - my @valid_trial_types = CXGN::Trial::get_all_project_types($schema); - my %trial_type_map = map { @{$_}[1] => @{$_}[0] } @valid_trial_types; - my $trial_type; - my $plot_width; - my $plot_length; - my $field_size; - my $planting_date; - my $harvest_date; - my %seen_entry_numbers; - - for my $row ( 1 .. $row_max ) { - - my $current_trial_name; - my $plot_name; - my $accession_name; - my $plot_number; - my $block_number; - my $is_a_control; - my $rep_number; - my $range_number; - my $row_number; - my $col_number; - my $seedlot_name; - my $num_seed_per_plot = 0; - my $weight_gram_seed_per_plot = 0; - my $entry_number; - - if ($worksheet->get_cell($row,$columns{trial_name}->{index})) { - $current_trial_name = $worksheet->get_cell($row,$columns{trial_name}->{index})->value(); - } + my $self = shift; + my $schema = $self->get_chado_schema(); + my $parsed = $self->_get_validated_data(); + my $data = $parsed->{'data'}; + my $values = $parsed->{'values'}; + my $treatments = $parsed->{'additional_columns'}; - if ($current_trial_name && $current_trial_name ne $trial_name) { + # Get synonyms for accessions in data + my $accession_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'accession', 'stock_type')->cvterm_id(); + my $synonym_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'stock_synonym', 'stock_property')->cvterm_id(); + my @accessions = @{$values->{'accession_name'}}; + my $acc_synonym_rs = $schema->resultset("Stock::Stock")->search({ + 'me.is_obsolete' => { '!=' => 't' }, + 'stockprops.value' => { -in => \@accessions}, + 'me.type_id' => $accession_cvterm_id, + 'stockprops.type_id' => $synonym_cvterm_id + },{join => 'stockprops', '+select'=>['stockprops.value'], '+as'=>['synonym']}); - if ($trial_name) { - ## Save old single trial hash in all trials hash; reinitialize temp hashes - my %final_design_details = (); - my $previous_design_data = $all_designs{$trial_name}{'design_details'}; - if ($previous_design_data) { - %final_design_details = (%design_details, %{$all_designs{$trial_name}{'design_details'}}); - } else { - %final_design_details = %design_details; - } - %design_details = (); - $single_design{'design_details'} = \%final_design_details; - $single_design{'entry_numbers'} = $seen_entry_numbers{$trial_name}; - my %final_single_design = %single_design; - %single_design = (); - $all_designs{$trial_name} = \%final_single_design; - } - - # Get location and replace codes with names - my $location = $worksheet->get_cell($row,$columns{location}->{index})->value(); - if ( $self->_has_location_code_map() ) { - my $location_code_map = $self->_get_location_code_map(); - if ( exists $location_code_map->{$location} ) { - $location = $location_code_map->{$location}; - } - } - - $single_design{'breeding_program'} = $worksheet->get_cell($row,$columns{breeding_program}->{index})->value(); - $single_design{'location'} = $location; - $single_design{'year'} = $worksheet->get_cell($row,$columns{year}->{index})->value(); - # $single_design{'transplanting_date'} = $worksheet->get_cell($row,$columns{transplanting_date}->{index})->value(); - $single_design{'design_type'} = $worksheet->get_cell($row,$columns{design_type}->{index})->value(); - $single_design{'description'} = $worksheet->get_cell($row,$columns{description}->{index})->value(); - - - # for a moment transplanting_date is moves as not required but whole design of that features must be redone - # including use cases - if ($worksheet->get_cell($row,$columns{transplanting_date}->{index})) { - $single_design{'transplanting_date'} = $worksheet->get_cell($row,$columns{transplanting_date}->{index})->value(); - } - - if ($worksheet->get_cell($row,$columns{trial_type}->{index})) { # get and save trial type cvterm_id using trial type name - my $trial_type_id = $trial_type_map{$worksheet->get_cell($row,$columns{trial_type}->{index})->value()}; - $single_design{'trial_type'} = $trial_type_id; - } - if ($worksheet->get_cell($row,$columns{plot_width}->{index})) { - $single_design{'plot_width'} = $worksheet->get_cell($row,$columns{plot_width}->{index})->value(); - } - if ($worksheet->get_cell($row,$columns{plot_length}->{index})) { - $single_design{'plot_length'} = $worksheet->get_cell($row,$columns{plot_length}->{index})->value(); - } - if ($worksheet->get_cell($row,$columns{field_size}->{index})) { - $single_design{'field_size'} = $worksheet->get_cell($row,$columns{field_size}->{index})->value(); - } - if ($worksheet->get_cell($row,$columns{planting_date}->{index})) { - $single_design{'planting_date'} = $worksheet->get_cell($row,$columns{planting_date}->{index})->value(); - } - if ($worksheet->get_cell($row,$columns{harvest_date}->{index})) { - $single_design{'harvest_date'} = $worksheet->get_cell($row,$columns{harvest_date}->{index})->value(); - } - ## Update trial name - $trial_name = $current_trial_name; + # Create lookup hash for synonym -> uniquename -> stock id + my %acc_synonyms_lookup; + while (my $r=$acc_synonym_rs->next){ + $acc_synonyms_lookup{$r->get_column('synonym')}->{$r->uniquename} = $r->stock_id; } - #skip blank rows - my $has_row_value; - foreach (keys %columns) { - if ( $worksheet->get_cell($row,$columns{$_}->{index})) { - if ( $worksheet->get_cell($row,$columns{$_}->{index})->value() ) { - $has_row_value = 1; - } - } - } - if ( !$has_row_value ) { - next; - } + # Create map of trial type codes to trial type ids + my @valid_trial_types = CXGN::Trial::get_all_project_types($schema); + my %trial_type_map = map { @{$_}[1] => @{$_}[0] } @valid_trial_types; + + my %all_designs; + my %single_design; + my %design_details; + my %seen_entry_numbers; + my $trial_name = ''; + for my $row (@$data) { + print STDERR Dumper $row; + + my $current_trial_name = $row->{'trial_name'}; + my $accession_name = $row->{'accession_name'}; + my $plot_number = $row->{'plot_number'}; + my $plot_name = $row->{'plot_name'} || _create_plot_name($current_trial_name, $plot_number); + my $block_number = $row->{'block_number'}; + my $is_a_control = $row->{'is_a_control'}; + my $rep_number = $row->{'rep_number'}; + my $range_number = $row->{'range_number'}; + my $row_number = $row->{'row_number'}; + my $col_number = $row->{'col_number'}; + my $seedlot_name = $row->{'seedlot_name'}; + my $num_seed_per_plot = $row->{'num_seed_per_plot'} || 0; + my $weight_gram_seed_per_plot = $row->{'weight_gram_seed_per_plot'} || 0; + my $entry_number = $row->{'entry_number'}; + + if ($current_trial_name && $current_trial_name ne $trial_name) { + + ## Save old single trial hash in all trials hash; reinitialize temp hashes + if ($trial_name) { + my %final_design_details = (); + my $previous_design_data = $all_designs{$trial_name}{'design_details'}; + if ($previous_design_data) { + %final_design_details = (%design_details, %{$all_designs{$trial_name}{'design_details'}}); + } else { + %final_design_details = %design_details; + } + %design_details = (); + $single_design{'design_details'} = \%final_design_details; + $single_design{'entry_numbers'} = $seen_entry_numbers{$trial_name}; + my %final_single_design = %single_design; + %single_design = (); + $all_designs{$trial_name} = \%final_single_design; + } - if ($worksheet->get_cell($row,$columns{plot_number}->{index})) { - $plot_number = $worksheet->get_cell($row,$columns{plot_number}->{index})->value(); - } - if ($worksheet->get_cell($row,$columns{plot_name}->{index})) { - $plot_name = $worksheet->get_cell($row,$columns{plot_name}->{index})->value(); - } - if (!$plot_name || $plot_name eq '') { - $plot_name = _create_plot_name($current_trial_name, $plot_number); - } - $plot_name =~ s/^\s+|\s+$//g; #trim whitespace from front and end... - if ($worksheet->get_cell($row,$columns{accession_name}->{index})) { - $accession_name = $worksheet->get_cell($row,$columns{accession_name}->{index})->value(); - } - $accession_name =~ s/^\s+|\s+$//g; #trim whitespace from front and end... - if ($worksheet->get_cell($row,$columns{block_number}->{index})) { - $block_number = $worksheet->get_cell($row,$columns{block_number}->{index})->value(); - } - if ($worksheet->get_cell($row,$columns{is_a_control}->{index})) { - $is_a_control = $worksheet->get_cell($row,$columns{is_a_control}->{index})->value(); - } - if ($worksheet->get_cell($row,$columns{rep_number}->{index})) { - $rep_number = $worksheet->get_cell($row,$columns{rep_number}->{index})->value(); - } - if ($worksheet->get_cell($row,$columns{range_number}->{index})) { - $range_number = $worksheet->get_cell($row,$columns{range_number}->{index})->value(); - } - if ($worksheet->get_cell($row,$columns{row_number}->{index})) { - $row_number = $worksheet->get_cell($row, $columns{row_number}->{index})->value(); - } - if ($worksheet->get_cell($row,$columns{col_number}->{index})) { - $col_number = $worksheet->get_cell($row, $columns{col_number}->{index})->value(); - } - if ($worksheet->get_cell($row,$columns{seedlot_name}->{index})) { - $seedlot_name = $worksheet->get_cell($row, $columns{seedlot_name}->{index})->value(); - } - if ($worksheet->get_cell($row,$columns{entry_number}->{index})) { - $entry_number = $worksheet->get_cell($row, $columns{entry_number}->{index})->value(); - } + # Get location and replace codes with names + my $location = $row->{'location'}; + if ( $self->_has_location_code_map() ) { + my $location_code_map = $self->_get_location_code_map(); + if ( exists $location_code_map->{$location} ) { + $location = $location_code_map->{$location}; + } + } - if ($seedlot_name){ - $seedlot_name =~ s/^\s+|\s+$//g; #trim whitespace from front and end... - } - if ($worksheet->get_cell($row,$columns{num_seed_per_plot}->{index})) { - $num_seed_per_plot = $worksheet->get_cell($row, $columns{num_seed_per_plot}->{index})->value(); - } - if ($worksheet->get_cell($row,$columns{weight_gram_seed_per_plot}->{index})) { - $weight_gram_seed_per_plot = $worksheet->get_cell($row, $columns{weight_gram_seed_per_plot}->{index})->value(); - } + $single_design{'breeding_program'} = $row->{'breeding_program'}; + $single_design{'location'} = $location; + $single_design{'year'} = $row->{'year'}; + $single_design{'design_type'} = $row->{'design_type'}; + $single_design{'description'} = $row->{'description'}; + $single_design{'plot_width'} = $row->{'plot_width'}; + $single_design{'plot_length'} = $row->{'plot_length'}; + $single_design{'field_size'} = $row->{'field_size'}; + $single_design{'planting_date'} = $row->{'planting_date'}; + $single_design{'harvest_date'} = $row->{'harvest_date'}; + + # for a moment transplanting_date is moves as not required but whole design of that features must be redone + # including use cases + if ($row->{'transplanting_date'}) { + $single_design{'transplanting_date'} = $row->{'transplanting_date'}; + } - if ($entry_number) { - $seen_entry_numbers{$current_trial_name}->{$accession_name} = $entry_number; - } + # get and save trial type cvterm_id using trial type name + if ($row->{'trial_type'}) { + my $trial_type_id = $trial_type_map{$row->{'trial_type'}}; + $single_design{'trial_type'} = $trial_type_id; + } - foreach my $treatment_name (@treatment_names){ - my $treatment_col = $columns{$treatment_name}->{index}; - if($worksheet->get_cell($row,$treatment_col)){ - if($worksheet->get_cell($row,$treatment_col)->value()){ - push @{$design_details{treatments}->{$treatment_name}{new_treatment_stocks}}, $plot_name; + ## Update trial name + $trial_name = $current_trial_name; } - } - } - - if ($acc_synonyms_lookup{$accession_name}){ - my @accession_names = keys %{$acc_synonyms_lookup{$accession_name}}; - if (scalar(@accession_names)>1){ - print STDERR "There is more than one uniquename for this synonym $accession_name. this should not happen!\n"; - } - $accession_name = $accession_names[0]; - } - - my $key = $row; - $design_details{$key}->{plot_name} = $plot_name; - $design_details{$key}->{stock_name} = $accession_name; - $design_details{$key}->{plot_number} = $plot_number; - $design_details{$key}->{block_number} = $block_number; - if ($is_a_control) { - $design_details{$key}->{is_a_control} = 1; - } else { - $design_details{$key}->{is_a_control} = 0; - } - if ($rep_number) { - $design_details{$key}->{rep_number} = $rep_number; - } - if ($range_number) { - $design_details{$key}->{range_number} = $range_number; - } - if ($row_number) { - $design_details{$key}->{row_number} = $row_number; - } - if ($col_number) { - $design_details{$key}->{col_number} = $col_number; - } - if ($seedlot_name){ - $design_details{$key}->{seedlot_name} = $seedlot_name; - $design_details{$key}->{num_seed_per_plot} = $num_seed_per_plot; - $design_details{$key}->{weight_gram_seed_per_plot} = $weight_gram_seed_per_plot; - } - } - - # add last trial design to all_designs and save parsed data, then return - my %final_design_details = (); - my $previous_design_data = $all_designs{$trial_name}{'design_details'}; - if ($previous_design_data) { - %final_design_details = (%design_details, %{$all_designs{$trial_name}{'design_details'}}); - } else { - %final_design_details = %design_details; - } - $single_design{'design_details'} = \%final_design_details; - $single_design{'entry_numbers'} = $seen_entry_numbers{$trial_name}; - $all_designs{$trial_name} = \%single_design; - - $self->_set_parsed_data(\%all_designs); + if ($entry_number) { + $seen_entry_numbers{$current_trial_name}->{$accession_name} = $entry_number; + } - return 1; + foreach my $treatment_name (@$treatments){ + my $treatment_value = $row->{$treatment_name}; + if ( $treatment_value ) { + push @{$design_details{treatments}->{$treatment_name}{new_treatment_stocks}}, $plot_name; + } + } -} + if ($acc_synonyms_lookup{$accession_name}){ + my @accession_names = keys %{$acc_synonyms_lookup{$accession_name}}; + if (scalar(@accession_names)>1){ + print STDERR "There is more than one uniquename for this synonym $accession_name. this should not happen!\n"; + } + $accession_name = $accession_names[0]; + } -sub _parse_headers { - my $worksheet = shift; - my ( $col_min, $col_max ) = $worksheet->col_range(); - my %columns; - my @treatments; - my @errors; - - for ( $col_min .. $col_max ) { - if ( $worksheet->get_cell(0,$_) ) { - my $header = $worksheet->get_cell(0,$_)->value(); - $header =~ s/^\s+|\s+$//g; - my $is_required = !!grep( /^$header$/, @REQUIRED_COLUMNS ); - my $is_optional = !!grep( /^$header$/, @OPTIONAL_COLUMNS ); - my $is_treatment = !grep( /^$header$/, @REQUIRED_COLUMNS ) && !grep( /^$header$/, @OPTIONAL_COLUMNS ); - $columns{$header} = { - header => $header, - index => $_, - is_required => $is_required, - is_optional => $is_optional, - is_treatment => $is_treatment - }; - if ( $is_treatment ) { - push(@treatments, $header); - } + my $key = $row; + $design_details{$key}->{plot_name} = $plot_name; + $design_details{$key}->{stock_name} = $accession_name; + $design_details{$key}->{plot_number} = $plot_number; + $design_details{$key}->{block_number} = $block_number; + if ($is_a_control) { + $design_details{$key}->{is_a_control} = 1; + } else { + $design_details{$key}->{is_a_control} = 0; + } + if ($rep_number) { + $design_details{$key}->{rep_number} = $rep_number; + } + if ($range_number) { + $design_details{$key}->{range_number} = $range_number; + } + if ($row_number) { + $design_details{$key}->{row_number} = $row_number; + } + if ($col_number) { + $design_details{$key}->{col_number} = $col_number; + } + if ($seedlot_name){ + $design_details{$key}->{seedlot_name} = $seedlot_name; + $design_details{$key}->{num_seed_per_plot} = $num_seed_per_plot; + $design_details{$key}->{weight_gram_seed_per_plot} = $weight_gram_seed_per_plot; + } } - } - foreach (@REQUIRED_COLUMNS) { - if ( !exists $columns{$_} ) { - push(@errors, "Required column $_ is missing from the file!"); + # add last trial design to all_designs and save parsed data, then return + my %final_design_details = (); + my $previous_design_data = $all_designs{$trial_name}{'design_details'}; + if ($previous_design_data) { + %final_design_details = (%design_details, %{$all_designs{$trial_name}{'design_details'}}); + } else { + %final_design_details = %design_details; } - } + $single_design{'design_details'} = \%final_design_details; + $single_design{'entry_numbers'} = $seen_entry_numbers{$trial_name}; + $all_designs{$trial_name} = \%single_design; + + $self->_set_parsed_data(\%all_designs); - return { - columns => \%columns, - treatments => \@treatments, - errors => \@errors - } + return 1; } sub _create_plot_name { From 43af812036d03283e73f02c051a7574fdaa70adb Mon Sep 17 00:00:00 2001 From: David Waring Date: Fri, 1 Nov 2024 16:08:53 -0400 Subject: [PATCH 07/19] Multi-Trial Generic Upload: fix row key --- .../Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm b/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm index d14eac957d..657104c64d 100644 --- a/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm +++ b/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm @@ -502,8 +502,7 @@ sub _parse_with_plugin { my %seen_entry_numbers; my $trial_name = ''; for my $row (@$data) { - print STDERR Dumper $row; - + my $row_id = $row->{'_row'}; my $current_trial_name = $row->{'trial_name'}; my $accession_name = $row->{'accession_name'}; my $plot_number = $row->{'plot_number'}; @@ -593,7 +592,7 @@ sub _parse_with_plugin { $accession_name = $accession_names[0]; } - my $key = $row; + my $key = $row_id; $design_details{$key}->{plot_name} = $plot_name; $design_details{$key}->{stock_name} = $accession_name; $design_details{$key}->{plot_number} = $plot_number; From 62caad4d19e7a6bf1ab0337c7280dbcf1f3c0aae Mon Sep 17 00:00:00 2001 From: David Waring Date: Sun, 3 Nov 2024 16:21:14 -0500 Subject: [PATCH 08/19] Multi-Trial Generic Upload: group existing plot name error messages --- .../ParseUpload/Plugin/MultipleTrialDesignGeneric.pm | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm b/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm index 657104c64d..e67a93b9fd 100644 --- a/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm +++ b/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm @@ -291,7 +291,7 @@ sub _validate_with_plugin { } } if (scalar(@already_used_trial_names) > 0) { - push @error_messages, "Trial name(s) ".join(',',@already_used_trial_names)." are invalid because they are already used in the database."; + push @error_messages, "Trial name(s) ".join(', ',@already_used_trial_names)." are invalid because they are already used in the database."; } # Breeding Program: must already exist in the database @@ -374,12 +374,16 @@ sub _validate_with_plugin { } # Plot Names: should not exist (as any stock) + my @already_used_plot_names; my $rs = $schema->resultset("Stock::Stock")->search({ 'is_obsolete' => { '!=' => 't' }, 'uniquename' => { -in => $parsed_values->{'plot_name'} } }); while (my $r=$rs->next) { - push @error_messages, "Plot name ".$r->uniquename." already exists in the database. Each plot must have a new unique name."; + push @already_used_plot_names, $r->uniquename(); + } + if (scalar(@already_used_plot_names) > 0) { + push @error_messages, "Plot name(s) ".join(', ',@already_used_plot_names)." are invalid because they are already used in the database."; } # Seedlots: names must exist in the database From abf2da108161c72830e7d0bd3802cf9504d42ea6 Mon Sep 17 00:00:00 2001 From: David Waring Date: Mon, 4 Nov 2024 10:17:16 -0500 Subject: [PATCH 09/19] Multi-Trial Generic Upload: refactor backend script perform file validation and parsing in backend script remove unused code parse errors and warnings from backend script in AJAX endpoint --- bin/upload_multiple_trial_design.pl | 617 ++++++------------ .../CXGN/BreedersToolbox/UploadTrial.js | 20 +- .../Plugin/MultipleTrialDesignGeneric.pm | 2 - lib/SGN/Controller/AJAX/Trial.pm | 199 ++---- 4 files changed, 251 insertions(+), 587 deletions(-) diff --git a/bin/upload_multiple_trial_design.pl b/bin/upload_multiple_trial_design.pl index d3967cb4c6..be386bc8ee 100644 --- a/bin/upload_multiple_trial_design.pl +++ b/bin/upload_multiple_trial_design.pl @@ -6,51 +6,27 @@ =head1 SYNOPSIS - upload_multiple_trial_design.pl -H [dbhost] -D [dbname] -P [dbpass] -w [basepath] -U [dbuser] -b [breeding program name] -i infile -un [username] -e [email address] -eo [email_option_enabled] -r [temp_file_nd_experiment_id] +upload_multiple_trial_design.pl -H [dbhost] -D [dbname] -P [dbpass] -w [basepath] -U [dbuser] -i infile -un [username] -e [email address] =head1 COMMAND-LINE OPTIONS ARGUMENTS -H host name (required) Ex: "breedbase_db" -D database name (required) Ex: "breedbase" - -P database userpass (required) Ex: "postgres" + -U database username Ex: "postgres" + -P database userpass Ex: "postgres" -w basepath (required) Ex: /home/production/cxgn/sgn -i path to infile (required) - -U username (required) Ex: "postgres" - -b breeding program name (required) Ex: test - -t test run . Rolling back at the end + -un username of uploader (required) -e email address of the user - -l name of the user - -r temp_file_nd_experiment_id (required) Ex: /temp/delete_nd_experiment_ids.txt if loading trial data from metadata file, phenotypes + layout from infile =head2 DESCRIPTION -perl bin/upload_multiple_trial_design.pl -h breedbase_db -d breedbase -p postgres -w /home/cxgn/sgn/ -u postgres -i ~/Desktop/test_multi.xlsx -b test -n janedoe -e 'sk2783@cornell.edu' -l 'sri' -r /tmp/delete_nd_experiment_ids.txt +perl bin/upload_multiple_trial_design.pl -H breedbase_db -D breedbase -U postgres -P postgres -w /home/cxgn/sgn/ -un janedoe -i ~/Desktop/test_multi.xlsx -e 'sk2783@cornell.edu' -iw -This script will parse and validate the input file. If there are any warnings or errors during validation it will send a error message to the provided email. -If there are no warnings(or errors) during validation it will then store the data. -The input file should be either .xlsx or .xls format. - -CHECK cvterms for trial metadata!! - -################################################ -Minimal metadata requirements are - trial_name - trial_description (can also be built from the trial name, type, year, location) - trial_type (read from an input file) - trial_location geo_description ( must be in the database - nd_geolocation.description - can be read from metadata file) - year (can be read from the metadata file ) - design (defaults to 'RCBD' ) - breeding_program (provide with option -b ) - - -Other OPTIONAL trial metadata (projectprops) - -project planting date -project fertilizer date -project harvest date -project sown plants -project harvested plants +This script will parse and validate the input file. If there are any warnings or errors during validation it will send a error message to the provided email. It will print any errors and warnings to the console. +If there are no errors or warnings (or warnings are ignored) during validation it will then store the data. +The input file should be any file supported by the CXGN::File::Parse class. =head1 AUTHOR @@ -60,479 +36,252 @@ =head1 AUTHOR use strict; use Getopt::Long; -use File::Basename; -use CXGN::File::Parse; use Bio::Chado::Schema; use CXGN::DB::InsertDBH; -use Carp qw /croak/ ; use Try::Tiny; use DateTime; use Pod::Usage; -use List::Util qw(max); -use List::Util qw(first); -use List::Util qw(uniq); -use CXGN::Metadata::Schema; -use CXGN::Phenome::Schema; -use CXGN::People::Person; -use Data::Dumper; -use CXGN::Phenotypes::StorePhenotypes; -use CXGN::Trial; # add project metadata -#use CXGN::BreedersToolbox::Projects; # associating a breeding program +use CXGN::Trial; # add project metadata +use CXGN::Trial::ParseUpload; use CXGN::Trial::TrialCreate; -use CXGN::Tools::Run; use CXGN::Contact; -use CXGN::Trial::ParseUpload; use CXGN::TrialStatus; -use CXGN::Calendar; -use CXGN::UploadFile; -use File::Path qw(make_path); -use File::Spec; -my ( $help, $dbhost, $dbname, $basepath, $dbuser, $dbpass, $infile, $sites, $types, $username, $breeding_program_name, $email_address, $logged_in_name, $email_option_enabled, $temp_file_nd_experiment_id); +my ( $help, $dbhost, $dbname, $basepath, $dbuser, $dbpass, $infile, $username, $email_address, $ignore_warnings); GetOptions( - 'dbhost|H=s' => \$dbhost, - 'dbname|D=s' => \$dbname, - 'dbpass|P=s' => \$dbpass, - 'basepath|w=s' => \$basepath, - 'dbuser|U=s' => \$dbuser, - 'i=s' => \$infile, - 'b=s' => \$breeding_program_name, - 'user|un=s' => \$username, - 'help' => \$help, - 'email|e=s' => \$email_address, - # 'logged_in_user|l=s' => \$logged_in_name, - 'temp_file|r=s' => \$temp_file_nd_experiment_id, - 'email_option_enabled|eo=s' => \$email_option_enabled, + 'dbhost|H=s' => \$dbhost, + 'dbname|D=s' => \$dbname, + 'dbuser|U=s' => \$dbuser, + 'dbpass|P=s' => \$dbpass, + 'basepath|w=s' => \$basepath, + 'i=s' => \$infile, + 'user|un=s' => \$username, + 'email|e=s' => \$email_address, + 'ignore_warnings|iw!' => \$ignore_warnings, + 'help' => \$help, ); - -#Ensure the parent directory exists before creating the temporary file -my $parent_dir = File::Spec->catdir($basepath, 'static', 'documents', 'tempfiles', 'delete_nd_experiment_ids'); -unless (-d $parent_dir) { - make_path($parent_dir) or die "Failed to create directory $parent_dir: $!"; -} - -# Create the temporary file in the parent directory -my $temp_file_nd_experiment_id = File::Spec->catfile($parent_dir, 'fileXXXX'); - pod2usage(1) if $help; -if (!$infile || !$breeding_program_name || !$username || !$dbname || !$dbhost ) { - pod2usage( { -msg => 'Error. Missing options!' , -verbose => 1, -exitval => 1 } ) ; -} - -my $dbh; - -if ($dbpass) { - print STDERR "Logging in with password\n"; - $dbh = DBI->connect("dbi:Pg:database=$dbname;host=$dbhost", - $dbuser, - $dbpass, - {AutoCommit => 1, - RaiseError => 1}); -} else { - $dbh = CXGN::DB::InsertDBH->new( { - dbhost =>$dbhost, - dbname =>$dbname, - dbargs => {AutoCommit => 1, - RaiseError => 1} - }); +if (!$infile || !$username || !$basepath || !$dbname || !$dbhost ) { + pod2usage({ -msg => 'Error. Missing options!', -verbose => 1, -exitval => 1 }); } -print STDERR "Database connection ok!\n"; - -my $schema= Bio::Chado::Schema->connect( sub { $dbh } , { on_connect_do => ['SET search_path TO public, sgn, metadata, phenome;'] } ); - -# ################ -# getting the last database ids for resetting at the end in case of rolling back -# ############### - -# my $last_nd_experiment_id = $schema->resultset('NaturalDiversity::NdExperiment')->get_column('nd_experiment_id')->max; -# my $last_cvterm_id = $schema->resultset('Cv::Cvterm')->get_column('cvterm_id')->max; - -# my $last_nd_experiment_project_id = $schema->resultset('NaturalDiversity::NdExperimentProject')->get_column('nd_experiment_project_id')->max; -# my $last_nd_experiment_stock_id = $schema->resultset('NaturalDiversity::NdExperimentStock')->get_column('nd_experiment_stock_id')->max; -# my $last_nd_experiment_phenotype_id = $schema->resultset('NaturalDiversity::NdExperimentPhenotype')->get_column('nd_experiment_phenotype_id')->max; -# my $last_phenotype_id = $schema->resultset('Phenotype::Phenotype')->get_column('phenotype_id')->max; -# my $last_stock_id = $schema->resultset('Stock::Stock')->get_column('stock_id')->max; -# my $last_stock_relationship_id = $schema->resultset('Stock::StockRelationship')->get_column('stock_relationship_id')->max; -# my $last_project_id = $schema->resultset('Project::Project')->get_column('project_id')->max; -# my $last_nd_geolocation_id = $schema->resultset('NaturalDiversity::NdGeolocation')->get_column('nd_geolocation_id')->max; -# my $last_geoprop_id = $schema->resultset('NaturalDiversity::NdGeolocationprop')->get_column('nd_geolocationprop_id')->max; -# my $last_projectprop_id = $schema->resultset('Project::Projectprop')->get_column('projectprop_id')->max; - -# my %seq = ( -# 'nd_experiment_nd_experiment_id_seq' => $last_nd_experiment_id, -# 'cvterm_cvterm_id_seq' => $last_cvterm_id, -# 'nd_experiment_project_nd_experiment_project_id_seq' => $last_nd_experiment_project_id, -# 'nd_experiment_stock_nd_experiment_stock_id_seq' => $last_nd_experiment_stock_id, -# 'nd_experiment_phenotype_nd_experiment_phenotype_id_seq' => $last_nd_experiment_phenotype_id, -# 'phenotype_phenotype_id_seq' => $last_phenotype_id, -# 'stock_stock_id_seq' => $last_stock_id, -# 'stock_relationship_stock_relationship_id_seq' => $last_stock_relationship_id, -# 'project_project_id_seq' => $last_project_id, -# 'nd_geolocation_nd_geolocation_id_seq' => $last_nd_geolocation_id, -# 'nd_geolocationprop_nd_geolocationprop_id_seq' => $last_geoprop_id, -# 'projectprop_projectprop_id_seq' => $last_projectprop_id, -# ); - - -# ############## -# Breeding program for associating the trial/s ## -# ############## - -my $breeding_program = $schema->resultset("Project::Project")->find( - {'me.name' => $breeding_program_name, 'type.name' => 'breeding_program'}, - { join => { projectprops => 'type' }} -); - -if (!$breeding_program) { die "Breeding program $breeding_program_name does not exist in the database. Check your input \n"; } -# print STDERR "Found breeding program $breeding_program_name " . $breeding_program->project_id . "\n"; - -my $sp_person_id= CXGN::People::Person->get_person_by_username($dbh, $username); -die "Need to have a user pre-loaded in the database! " if !$sp_person_id; - -#Column headers for trial design/s -#plot_name accession_name plot_number block_number trial_name trial_description trial_location year trial_type is_a_control rep_number range_number row_number col_number entry_numbers - -#parse file using the generic file parser -my $parser = CXGN::File::Parse->new( - file => $infile, - required_columns => ['trial_name', 'accession_name', 'plot_number', 'block_number', 'location', 'year'], -); - -# warn "Starting to parse the file...\n"; -my $parsed = $parser->parse(); -# warn "Parsed data = " . Dumper($parsed); - -die "Error parsing file: " . join(',', @{$parsed->{errors}}) if scalar(@{$parsed->{errors}}) > 0; +# Lists of encountered errors and warnings +my @errors; +my @warnings; -if (exists $parsed->{warnings}) { - print "Warnings: " . join("\n", @{$parsed->{warnings}}) . "\n"; +# Connect to databases +my $dbh; +if ($dbpass && $dbuser) { + $dbh = DBI->connect( + "dbi:Pg:database=$dbname;host=$dbhost", + $dbuser, + $dbpass, + {AutoCommit => 1, RaiseError => 1} + ); } -# #parse the file -# my $parsed = $parser->parse(); - -# if (scalar(@{$parsed->{errors}}) > 0) { -# die "Error parsing file: ".join(',', @{$parsed->{errors}}); -# } - -my @traits; -my %multi_trial_data; -my %metadata_fields = map { $_ => 1 } qw(trial_name accession_name plot_number block_number location year design_type trial_description); - -foreach my $row (@{$parsed->{data}}) { - my $trial_name = $row->{trial_name}; - next unless $trial_name; # Skip rows with empty trial names - - # Check if the location exists in the database - my $trial_location = $row->{location}; - my $location_rs = $schema->resultset("NaturalDiversity::NdGeolocation")->search({ - description => { ilike => '%' . $trial_location . '%' }, +else { + $dbh = CXGN::DB::InsertDBH->new({ + dbhost => $dbhost, + dbname => $dbname, + dbargs => {AutoCommit => 1, RaiseError => 1} }); - if (scalar($location_rs) == 0) { - die "ERROR: location must be pre-loaded in the database. Location name = '" . $trial_location . "'\n"; - } - my $location_id = $location_rs->first->nd_geolocation_id; - ###################################################### - - # # Store all data for the current trial - $multi_trial_data{$trial_name} = { - trial_location => $row->{location}, - trial_year => $row->{year}, - design_type => $row->{design_type}, - trial_description => $row->{description}, - program => $breeding_program->name, - plot_name => $row->{plot_name}, - accession_name => $row->{accession_name}, - plots => [], - }; - - foreach my $col (@{$parsed->{columns}}) { - next if exists $metadata_fields{$col}; - } -} - -print STDERR "unique trial names:\n"; -foreach my $name(keys %multi_trial_data) { - print"$name\n"; -} - -print STDERR "Reading phenotyping file:\n"; -my %phen_params = map { if ($_ =~ m/^\w+\|(\w+:\d{7})$/ ) { $_ => $1 } } @traits; -delete $phen_params{''}; - -# my @traits = keys %phen_params; -print STDERR "Found traits: " . Dumper(\%phen_params) . "\n"; - - -foreach my $trial_name (keys %multi_trial_data) { - $multi_trial_data{$trial_name}->{design} = $multi_trial_data{$trial_name}; - -} - -my %trial_design_hash; -my %phen_data_by_trial; - -foreach my $row (@{$parsed->{data}}) { - my $trial_name = $row->{trial_name}; - next unless $trial_name; - - my $plot_number = $row->{plot_number}; - my $plot_name = $row->{plot_name}; - $trial_design_hash{$trial_name}{$plot_number} = { - trial_name => $trial_name, - trial_type => $row->{trial_type}, - planting_date => $row->{planting_date}, - harvest_date => $row->{harvest_date}, - entry_numbers => $row->{entry_numbers}, - is_a_control => $row->{is_a_control}, - rep_number => $row->{rep_number}, - range_number => $row->{range_number}, - row_number => $row->{row_number}, - col_number => $row->{col_number}, - seedlot_name => $row->{seedlot_name}, - num_seed_per_plot => $row->{num_seed_per_plot}, - weight_gram_seed_per_plot => $row->{weight_gram_seed_per_plot}, - }; - - foreach my $trait_string (keys %phen_params) { - my $phen_value = $row->{$trait_string}; - $phen_data_by_trial{$trial_name}{$plot_name}{$trait_string} = [$phen_value, DateTime->now->datetime]; - } -} - -print STDERR "multi trial hash:" . Dumper(\%multi_trial_data); -print STDERR "trial design " . Dumper(\%trial_design_hash); -print STDERR "Processed trials: " . scalar(keys %trial_design_hash) . "\n"; -print STDERR "Phen data by trial: " . Dumper(\%phen_data_by_trial) . "\n"; - -#####create the design hash##### -print Dumper(keys %trial_design_hash); -foreach my $trial_name (keys %trial_design_hash) { - $multi_trial_data{$trial_name}->{design} = $trial_design_hash{$trial_name} ; } - -my $date = localtime(); -my $parser; -my %parsed_data; -my $parse_errors; -my @errors; -my $parsed_data; -my $ignore_warnings; -my $time = DateTime->now(); -my $timestamp = $time->ymd()."_".$time->hms(); - -my %phenotype_metadata = { - 'archived_file' => $infile, - 'archived_file_type' => 'spreadsheet phenotype file', - 'operator' => $username, - 'date' => $date, -}; - -#parse uploaded file with appropriate plugin -$parser = CXGN::Trial::ParseUpload->new(chado_schema => $schema, filename => $infile); -$parser->load_plugin('MultipleTrialDesignExcelFormat'); -$parsed_data = $parser->parse(); - - -if (!$parsed_data) { - my $return_error = ''; - - if (! $parser->has_parse_errors() ){ - die "could not get parsing errors\n"; - }else { - $parse_errors = $parser->get_parse_errors(); - die $parse_errors->{'error_messages'}; +my $chado_schema = Bio::Chado::Schema->connect(sub { $dbh }, { on_connect_do => ['SET search_path TO public, sgn, metadata, phenome;'] }); +print STDOUT "Database connection ok!\n"; + +# Parse uploaded file with appropriate plugin +my $parser = CXGN::Trial::ParseUpload->new(chado_schema => $chado_schema, filename => $infile); +$parser->load_plugin('MultipleTrialDesignGeneric'); +my $parsed_data = $parser->parse(); + +# Parser has errors, print error messages and quit +if ($parser->has_parse_errors()) { + my $errors = $parser->get_parse_errors(); + foreach (@{$errors->{'error_messages'}}) { + push @errors, $_; } - - die $return_error; + finish(); } +# Parser has warnings, print warning messages and quit unless we're ignoring warnings if ($parser->has_parse_warnings()) { unless ($ignore_warnings) { my $warnings = $parser->get_parse_warnings(); - print "Warnings: " . join("\n", @{$warnings->{'warning_messages'}}) . "\n"; + foreach (@{$warnings->{'warning_messages'}}) { + push @warnings, $_; + } + finish(); } } +# Check for parsed data +finish("There is no parsed data!") if !$parsed_data; + +# Get User ID +my $sp_person_id = CXGN::People::Person->get_person_by_username($dbh, $username); +finish("User not found in database for username $username!") if !$sp_person_id; -my %all_desings = %{$parsed_data}; -my %save; -$save{'errors'} = []; -my $coderef= sub { - foreach my $trial_name (keys %multi_trial_data) { - my $trial_location = $multi_trial_data{$trial_name}->{trial_location}; - my $trial_design_info = $all_desings{$trial_name}; - +# Create and Save Trials +my %all_designs = %{$parsed_data}; +my %saved_trials; +my $coderef = sub { + for my $trial_name ( keys %all_designs ) { + my $trial_design = $all_designs{$trial_name}; my %trial_info_hash = ( - chado_schema => $schema, - dbh => $dbh, - trial_year => $trial_design_info->{'year'}, - trial_description => $trial_design_info->{'description'}, - trial_location => $trial_design_info->{'location'}, - trial_name => $trial_name, - design_type => $trial_design_info->{'design_type'}, - design => $trial_design_info->{'design_details'}, - program => $trial_design_info->{'breeding_program'}, - operator => $username, - owner_id => $sp_person_id, + chado_schema => $chado_schema, + dbh => $dbh, + owner_id => $sp_person_id, + trial_year => $trial_design->{'year'}, + trial_description => $trial_design->{'description'}, + trial_location => $trial_design->{'location'}, + trial_name => $trial_name, + design_type => $trial_design->{'design_type'}, + design => $trial_design->{'design_details'}, + program => $trial_design->{'breeding_program'}, + upload_trial_file => $infile, + operator => $username, + owner_id => $sp_person_id ); + my $entry_numbers = $trial_design->{'entry_numbers'}; - # my $entry_numbers = $trial_design->{'entry_numbers'}; - - if ($trial_design_info->{'entry_numbers'}){ - $trial_info_hash{trial_type} = $trial_design_info->{'entry_numbers'}; + if ($trial_design->{'trial_type'}){ + $trial_info_hash{trial_type} = $trial_design->{'trial_type'}; } - if ($trial_design_info->{'trial_type'}){ - $trial_info_hash{trial_type} = $trial_design_info->{'trial_type'}; + if ($trial_design->{'plot_width'}){ + $trial_info_hash{plot_width} = $trial_design->{'plot_width'}; } - if ($trial_design_info->{'plot_width'}){ - $trial_info_hash{plot_width} = $trial_design_info->{'plot_width'}; + if ($trial_design->{'plot_length'}){ + $trial_info_hash{plot_length} = $trial_design->{'plot_length'}; } - if ($trial_design_info->{'plot_length'}){ - $trial_info_hash{plot_length} = $trial_design_info->{'plot_length'}; + if ($trial_design->{'field_size'}){ + $trial_info_hash{field_size} = $trial_design->{'field_size'}; } - if ($trial_design_info->{'field_size'}){ - $trial_info_hash{field_size} = $trial_design_info->{'field_size'}; + if ($trial_design->{'planting_date'}){ + $trial_info_hash{planting_date} = $trial_design->{'planting_date'}; } - if ($trial_design_info->{'planting_date'}){ - $trial_info_hash{planting_date} = $trial_design_info->{'planting_date'}; + if ($trial_design->{'harvest_date'}){ + $trial_info_hash{harvest_date} = $trial_design->{'harvest_date'}; } - if ($trial_design_info->{'harvest_date'}){ - $trial_info_hash{harvest_date} = $trial_design_info->{'harvest_date'}; + if ($trial_design->{'transplanting_date'}){ + $trial_info_hash{transplanting_date} = $trial_design->{'transplanting_date'}; } - my $trial_create = CXGN::Trial::TrialCreate->new(%trial_info_hash); + my $trial_create = CXGN::Trial::TrialCreate->new(\%trial_info_hash); my $current_save = $trial_create->save_trial(); if ($current_save->{error}){ - # $schema->txn_rollback(); - push @{$save{'errors'}}, $current_save->{'error'}; + $chado_schema->txn_rollback(); + finish($current_save->{'error'}); } elsif ($current_save->{'trial_id'}) { my $trial_id = $current_save->{'trial_id'}; + my $time = DateTime->now(); my $timestamp = $time->ymd(); my $calendar_funcs = CXGN::Calendar->new({}); my $formatted_date = $calendar_funcs->check_value_format($timestamp); my $upload_date = $calendar_funcs->display_start_date($formatted_date); + $saved_trials{$trial_id} = $trial_name; my %trial_activity; $trial_activity{'Trial Uploaded'}{'user_id'} = $sp_person_id; $trial_activity{'Trial Uploaded'}{'activity_date'} = $upload_date; - my $trial_activity_obj = CXGN::TrialStatus->new({ bcs_schema => $schema }); + my $trial_activity_obj = CXGN::TrialStatus->new({ bcs_schema => $chado_schema }); $trial_activity_obj->trial_activities(\%trial_activity); $trial_activity_obj->parent_id($trial_id); my $activity_prop_id = $trial_activity_obj->store(); } # save entry numbers, if provided - my $entry_numbers; - if ($entry_numbers = $trial_design_info->{'entry_numbers'}) { - if (scalar(keys %$entry_numbers) > 0 && $current_save->{'trial_id'} ) { - my %entry_numbers_prop; - my @stock_names = keys %$entry_numbers; - - # Convert stock names from parsed trial template to stock ids for data storage - my $stocks = $schema->resultset('Stock::Stock')->search({ uniquename=>{-in=>\@stock_names} }); - while (my $s = $stocks->next()) { - $entry_numbers_prop{$s->stock_id} = $entry_numbers->{$s->uniquename}; - } + if ( $entry_numbers && scalar(keys %$entry_numbers) > 0 && $current_save->{'trial_id'} ) { + my %entry_numbers_prop; + my @stock_names = keys %$entry_numbers; + + # Convert stock names from parsed trial template to stock ids for data storage + my $stocks = $chado_schema->resultset('Stock::Stock')->search({ uniquename=>{-in=>\@stock_names} }); + while (my $s = $stocks->next()) { + $entry_numbers_prop{$s->stock_id} = $entry_numbers->{$s->uniquename}; + } - # Lookup synonyms of accession names - my $synonym_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'stock_synonym', 'stock_property')->cvterm_id(); - my $acc_synonym_rs = $schema->resultset("Stock::Stock")->search({ - 'me.is_obsolete' => { '!=' => 't' }, - 'stockprops.value' => { -in => \@stock_names}, - 'stockprops.type_id' => $synonym_cvterm_id - },{join => 'stockprops', '+select'=>['stockprops.value'], '+as'=>['synonym']}); - while (my $r=$acc_synonym_rs->next) { - if ( exists($entry_numbers->{$r->get_column('synonym')}) ) { - $entry_numbers_prop{$r->stock_id} = $entry_numbers->{$r->get_column('synonym')}; - } + # Lookup synonyms of accession names + my $synonym_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($chado_schema, 'stock_synonym', 'stock_property')->cvterm_id(); + my $acc_synonym_rs = $chado_schema->resultset("Stock::Stock")->search({ + 'me.is_obsolete' => { '!=' => 't' }, + 'stockprops.value' => { -in => \@stock_names}, + 'stockprops.type_id' => $synonym_cvterm_id + },{join => 'stockprops', '+select'=>['stockprops.value'], '+as'=>['synonym']}); + while (my $r=$acc_synonym_rs->next) { + if ( exists($entry_numbers->{$r->get_column('synonym')}) ) { + $entry_numbers_prop{$r->stock_id} = $entry_numbers->{$r->get_column('synonym')}; } - - # store entry numbers - my $trial = CXGN::Trial->new({ bcs_schema => $schema, trial_id => $current_save->{'trial_id'} }); - $trial->set_entry_numbers(\%entry_numbers_prop); } - } - print STDERR "TrialCreate object created for trial: $trial_name\n"; - - my @plots = @{$multi_trial_data{$trial_name}->{plots} // []}; - print STDERR "Trial Name = $trial_name\n"; - - my %parsed_data = $phen_data_by_trial{$trial_name}; - if (scalar(@traits) > 0) { - foreach my $pname (keys %parsed_data) { - print STDERR "PLOT = $pname\n"; - my %trait_string_hash = $parsed_data{$pname}; - - foreach my $trait_string (keys %trait_string_hash ) { - print STDERR "trait = $trait_string\n"; - print STDERR "value = " . $trait_string_hash{$trait_string}[0] . "\n"; - } - } - - # # after storing the trial desgin store the phenotypes - my $metadata_schema = CXGN::Metadata::Schema->connect( sub { $dbh->get_actual_dbh() }, {on_connect_do => ['SET search_path TO metadata;'] } ); - my $phenome_schema = CXGN::Phenome::Schema->connect( sub { $dbh->get_actual_dbh() } , {on_connect_do => ['SET search_path TO phenome, sgn, public;'] } ); - my $store_phenotypes = CXGN::Phenotypes::StorePhenotypes->new( - basepath => $basepath, - dbhost => $dbhost, - dbname => $dbname, - dbuser => $dbuser, - dbpass => $dbpass, - temp_file_nd_experiment_id => $temp_file_nd_experiment_id, - bcs_schema => $schema, - metadata_schema => $metadata_schema, - phenome_schema => $phenome_schema, - user_id => $sp_person_id, - stock_list => \@plots, - trait_list => \@traits, - values_hash => \%parsed_data, - has_timestamps => 0, - overwrite_values => 0, - metadata_hash => \%phenotype_metadata, - ); - - #store the phenotypes - my ($verified_warning, $verified_error) = $store_phenotypes->verify(); - # print STDERR "Verified phenotypes. warning = $verified_warning, error = $verified_error\n"; - my $stored_phenotype_error = $store_phenotypes->store(); - # print STDERR "Stored phenotypes Error:" . Dumper($stored_phenotype_error). "\n"; - } else { - print STDERR "No traits defined for these $trial_name\n"; + # store entry numbers + my $trial = CXGN::Trial->new({ bcs_schema => $chado_schema, trial_id => $current_save->{'trial_id'} }); + $trial->set_entry_numbers(\%entry_numbers_prop); } } -}; -my ($email_subject, $email_body); +}; try { - $schema->txn_do($coderef); - - if ($email_option_enabled == 1 && $email_address) { - print "Transaction succeeded! Committing project and its metadata \n\n"; - - $email_subject = "Multiple Trial Designs upload status"; - $email_body = "Dear $username,\n\nCongratulations, all the multiple trial designs have been successfully uploaded into the database\n\nThank you\nHave a nice day\n\n"; - - CXGN::Contact::send_email($email_subject, $email_body, $email_address); - } + $chado_schema->txn_do($coderef); } catch { - # Transaction failed - my $error_message = "An error occurred! Rolling back! $_\n"; - # push @errors, $error_message; - # push @{$save{'errors'}}, $error_message; - # print STDERR $error_message; + push @errors, $_; +}; - if ($email_option_enabled == 1 && $email_address) { - $email_subject = 'Error in Trial Upload'; - $email_body = "Dear $username,\n\n$error_message\n\nThank You\nHave a nice day\n"; +finish(); - # print STDERR $error_message; +sub finish { + my $error = shift; + push @errors, $error if $error; - CXGN::Contact::send_email($email_subject, $email_body, $email_address); + # Print errors and warnings to STDERR + foreach (@errors) { + print STDERR "ERROR: $_\n"; } -}; + foreach (@warnings) { + print STDERR "WARNING: $_\n"; + } + + # Send email message, if requested + # Exit the script: 0 = success, 1 = errors, 2 = warnings + if ( scalar(@errors) > 0 ) { + if ( $email_address ) { + my $email_subject = "Multiple Trial Designs upload failed"; + my $email_body = "Dear $username,\n\nThere were one or more errors in uploading your trials:\n\n"; + foreach my $error (@errors) { + $error =~ s/<[^>]*>//g; + $email_body .= "$error\n"; + } + $email_body .= "\nYou will need to fix the errors and upload the corrected file. Thank you\nHave a nice day\n\n"; + CXGN::Contact::send_email($email_subject, $email_body, $email_address); + } + exit(1); + } + if ( scalar(@warnings) > 0 ) { + if ( $email_address ) { + my $email_subject = "Multiple Trial Designs upload failed"; + my $email_body = "Dear $username,\n\nThere were one or more warnings in uploading your trials and the option to ignore warnings was not enabled. The warnings include:\n\n"; + foreach my $warning (@warnings) { + $warning =~ s/<[^>]*>//g; + $email_body .= "$warning\n"; + } + $email_body .= "\nYou will need to either fix the warnings and upload the corrected file or upload the same file with the option to ignore warnings enabled. Thank you\nHave a nice day\n\n"; + CXGN::Contact::send_email($email_subject, $email_body, $email_address); + } + exit(2); + } + else { + my $bs = CXGN::BreederSearch->new({ dbh=>$dbh, dbname=>$dbname }); + my $refresh = $bs->refresh_matviews($dbhost, $dbname, $dbuser, $dbpass, 'all_but_genoview', 'concurrent', $basepath); + + if ( $email_address ) { + my $email_subject = "Multiple Trial Designs upload successful"; + my $email_body = "Dear $username,\n\nCongratulations, all the multiple trial designs have been successfully uploaded into the database\n\nThank you\nHave a nice day\n\n"; + CXGN::Contact::send_email($email_subject, $email_body, $email_address); + } + + exit(0); + } +} 1; \ No newline at end of file diff --git a/js/source/legacy/CXGN/BreedersToolbox/UploadTrial.js b/js/source/legacy/CXGN/BreedersToolbox/UploadTrial.js index 379ccdfb16..5f82a96323 100644 --- a/js/source/legacy/CXGN/BreedersToolbox/UploadTrial.js +++ b/js/source/legacy/CXGN/BreedersToolbox/UploadTrial.js @@ -317,7 +317,6 @@ jQuery(document).ready(function ($) { json: true, post: function () { var uploadedTrialLayoutFile = jQuery("#multiple_trial_designs_upload_file").val(); - var email_option_enabled = jQuery('#email_option_to_recieve_trial_upload_status').is(':checked') ? 1 : 0; // Clear existing messages jQuery("#upload_multiple_trials_warning_messages").html(''); @@ -328,17 +327,7 @@ jQuery(document).ready(function ($) { alert("No file selected"); return; } - - if (email_option_enabled === 1) { - if (confirm('You will receive an email once the process is complete. Do you want to continue?')) { - jQuery("#upload_trial_dialog").modal('hide'); - } else { - return; - } - } - else { - jQuery('#working_modal').modal("show"); - } + jQuery('#working_modal').modal("show"); }, complete: function(response) { // console.log(response); @@ -370,6 +359,13 @@ jQuery(document).ready(function ($) { jQuery("#upload_multiple_trials_success_button").show(); return; } + if (response.background) { + jQuery("#upload_multiple_trials_success_messages").show(); + jQuery("#upload_multiple_trials_success_messages").html("Your file has been uploaded. You will receive an email once the process is complete."); + jQuery("#multiple_trial_designs_upload_submit").hide(); + jQuery("#upload_multiple_trials_success_button").show(); + return; + } }, error: function(response) { if (!jQuery('#email_option_to_recieve_trial_upload_status').is(':checked')) { diff --git a/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm b/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm index e67a93b9fd..bdf3a0320a 100644 --- a/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm +++ b/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm @@ -3,9 +3,7 @@ package CXGN::Trial::ParseUpload::Plugin::MultipleTrialDesignGeneric; use Moose::Role; use List::MoreUtils qw(uniq); use CXGN::File::Parse; -use CXGN::Stock::StockLookup; use SGN::Model::Cvterm; -use Data::Dumper; use CXGN::List::Validate; use CXGN::Stock::Seedlot; use CXGN::Calendar; diff --git a/lib/SGN/Controller/AJAX/Trial.pm b/lib/SGN/Controller/AJAX/Trial.pm index 5896d7640f..51a543ae6f 100644 --- a/lib/SGN/Controller/AJAX/Trial.pm +++ b/lib/SGN/Controller/AJAX/Trial.pm @@ -1148,71 +1148,42 @@ sub upload_multiple_trial_designs_file : Path('/ajax/trial/upload_multiple_trial sub upload_multiple_trial_designs_file_POST : Args(0) { my ($self, $c) = @_; - - # print STDERR "Check 1: ".localtime()."\n"; - - # print STDERR Dumper $c->req->params(); - my $chado_schema = $c->dbic_schema('Bio::Chado::Schema', 'sgn_chado'); - my $metadata_schema = $c->dbic_schema("CXGN::Metadata::Schema"); - my $phenome_schema = $c->dbic_schema("CXGN::Phenome::Schema"); - my $dbh = $c->dbc->dbh; my $upload = $c->req->upload('multiple_trial_designs_upload_file'); - my $ignore_warnings = $c->req->param('upload_multiple_trials_ignore_warnings'); - my $dir = $c->tempfiles_subdir('/delete_nd_experiment_ids'); + my $ignore_warnings = $c->req->param('upload_multiple_trials_ignore_warnings') eq 'on'; + my $email_address = $c->req->param('trial_email_address_upload'); + my $email_option_enabled = $c->req->param('email_option_to_recieve_trial_upload_status') eq 'on'; + my $dbhost = $c->config->{dbhost}; my $dbname = $c->config->{dbname}; my $dbpass = $c->config->{dbpass}; my $basepath = $c->config->{basepath}; my $dbuser = $c->config->{dbuser}; - my $temp_file_nd_experiment_id = $c->config->{basepath}."/".$c->tempfile( TEMPLATE => 'delete_nd_experiment_ids/fileXXXX'); my $time = DateTime->now(); my $timestamp = $time->ymd()."_".$time->hms(); my $upload_original_name = $upload->filename(); my $upload_tempfile = $upload->tempname; my $subdirectory = "trial_upload"; my $archive_filename = $timestamp . "_" . $upload_original_name; - my $archived_filename_with_path; - my $parser; - my $parsed_data; - my $md5; - my $validate_file; - my $parsed_file; - my $parse_errors; - my %parsed_data; - my %upload_metadata; - my $user_id; - my $username; - my $error; - my $email_address ; - my $email_option_enabled; - my $breeding_program_name; - - - # print STDERR "Check 2: ".localtime()."\n"; - print STDERR "Ignore warnings is $ignore_warnings\n"; - if ($upload_original_name =~ /\s/ || $upload_original_name =~ /\// || $upload_original_name =~ /\\/ ) { - print STDERR "File name must not have spaces or slashes.\n"; - $c->stash->{rest} = {errors => "Uploaded file name must not contain spaces or slashes." }; - return; - } + # Check if user is logged in and has curator or submitter privileges if (!$c->user()) { print STDERR "User not logged in... not uploading a trial.\n"; $c->stash->{rest} = {errors => "You need to be logged in to upload a trial." }; return; } - if (!any { $_ eq "curator" || $_ eq "submitter" } ($c->user()->roles) ) { + if (!any { $_ eq "curator" || $_ eq "submitter" } ($c->user()->roles)) { $c->stash->{rest} = {errors => "You have insufficient privileges to upload a trial." }; return; } + my $user_id = $c->user()->get_object()->get_sp_person_id(); + my $username = $c->user()->get_object()->get_username(); - $user_id = $c->user()->get_object()->get_sp_person_id(); - $username = $c->user()->get_object()->get_username(); - $email_address = $c->req->param('trial_email_address_upload'); - $email_option_enabled = $c->req->param('email_option_to_recieve_trial_upload_status') eq 'on'; - - print STDERR "email option enabled : $email_option_enabled\n"; - print STDERR "recieved address: $email_address\n"; + # Check filename for spaces and/or slashes + if ($upload_original_name =~ /\s/ || $upload_original_name =~ /\// || $upload_original_name =~ /\\/ ) { + print STDERR "File name must not have spaces or slashes.\n"; + $c->stash->{rest} = {errors => "Uploaded file name must not contain spaces or slashes." }; + return; + } ## Store uploaded temporary file in archive my $uploader = CXGN::UploadFile->new({ @@ -1224,121 +1195,71 @@ sub upload_multiple_trial_designs_file_POST : Args(0) { user_id => $user_id, user_role => $c->user->get_object->get_user_type() }); - $archived_filename_with_path = $uploader->archive(); - $md5 = $uploader->get_md5($archived_filename_with_path); + my $archived_filename_with_path = $uploader->archive(); if (!$archived_filename_with_path) { $c->stash->{rest} = {errors => "Could not save file $archive_filename in archive",}; return; } unlink $upload_tempfile; - my $infile = $archived_filename_with_path; - # print STDERR "Check 3: ".localtime()."\n"; - $upload_metadata{'archived_file'} = $archived_filename_with_path; - $upload_metadata{'archived_file_type'}="trial upload file"; - $upload_metadata{'user_id'}=$user_id; - $upload_metadata{'date'}="$timestamp"; - - #parse uploaded file with appropriate plugin - $parser = CXGN::Trial::ParseUpload->new(chado_schema => $chado_schema, filename => $archived_filename_with_path); - $parser->load_plugin('MultipleTrialDesignGeneric'); - $parsed_data = $parser->parse(); + # Build the backend script command to parse, validate, and upload the trials + my $cmd = "perl \"$basepath/bin/upload_multiple_trial_design.pl\" -H \"$dbhost\" -D \"$dbname\" -U \"$dbuser\" -P \"$dbpass\" -w \"$basepath\" -i \"$archived_filename_with_path\" -un \"$username\""; + $cmd .= " -e \"$email_address\"" if $email_option_enabled && $email_address; + $cmd .= " -iw" if $ignore_warnings; - print STDERR "\n\n\n\n===> PARSED TRIAL DATA:\n"; - print STDERR Dumper $parsed_data; - print STDERR "\nErrors:\n"; - print STDERR Dumper $parser->get_parse_errors(); - print STDERR "\nWarnings:\n"; - print STDERR Dumper $parser->get_parse_warnings(); - print STDERR "\n\n\n\n"; + # Run asynchronously if email option is enabled + my $runner = CXGN::Tools::Run->new(); + if ( $email_option_enabled && $email_address ) { + $runner->run_async($cmd); + my $err = $runner->err(); + my $out = $runner->out(); - $c->stash->{rest} = {errors => "Upload not implemented"}; - return; + print STDERR "Upload Trials Output (async):\n"; + print STDERR "$err\n"; + print STDERR "$out\n"; - # print STDERR "check the parsed data : \n" . Dumper($parsed_data); - if (!$parsed_data) { - my $return_error = ''; - my $email_subject = "Errors in multiple trial upload"; - my $email_body = "Dear $username, \n\nErrors found. Please fix the following errors and try re-uploading again: $upload_original_name\n\n"; + $c->stash->{rest} = {background => 1}; + return; + } - if (! $parser->has_parse_errors() ){ - # $c->stash->{rest} = {errors => "Could not get parsing errors"}; - # return; - $return_error = "Could not get parsing errors"; - } - else { - # print STDERR "Parse errors are:\n"; - # print STDERR Dumper $parse_errors; - $parse_errors = $parser->get_parse_errors(); - if (ref($parse_errors) eq 'HASH' && exists $parse_errors->{'error_messages'}) { - $return_error = join("\n", @{$parse_errors->{'error_messages'}}); + # Otherwise run synchronously + else { + $runner->run($cmd); + my $err = $runner->err(); + my $out = $runner->out(); + + print STDERR "Upload Trials Output (sync):\n"; + print STDERR "$err\n"; + print STDERR "$out\n"; + + # Collect errors and warnings from STDERR + my @errors; + my @warnings; + foreach (split(/\n/, $err)) { + if ($_ =~ /^ERROR/) { + $_ =~ s/ERROR:? ?//; + push @errors, $_; + } + elsif ($_ =~ /^WARNING/) { + $_ =~ s/WARNING:? ?//; + push @warnings, $_; } - # $c->stash->{rest} = {errors => $parse_errors->{'error_messages'}}; - # return; - } - - #to remove HTML tags in the email message content - $return_error =~ s/<[^>]*>//g; - - $email_body .= $return_error; - $email_body .= "\n\nThank you\nHave a nice day\n"; - - if ($email_option_enabled == 1 && $email_address) { - CXGN::Contact::send_email($email_subject, $email_body, $email_address); } - $c->stash->{rest} = {errors => $return_error}; - return; - } - - if ($parser->has_parse_warnings()) { - unless ($ignore_warnings) { - my $warnings = $parser->get_parse_warnings(); - $c->stash->{rest} = {warnings => $warnings->{'warning_messages'}}; + if ( scalar(@errors) > 0 ) { + $c->stash->{rest} = {errors => \@errors}; return; } - } - - print STDERR "Check 4: ".localtime()."\n"; - # extract the breeding_program_name from the trial - - for my $trial (values%$parsed_data) { - if (exists $trial->{'breeding_program'}) { - $breeding_program_name = $trial->{'breeding_program'}; + if ( scalar(@warnings) > 0 ) { + $c->stash->{rest} = {warnings => \@warnings}; + return; } } - unless ($breeding_program_name) { - $c->stash->{rest} = {errors => "Breeding program not found in the uploaded file."}; - return; - } - print STDERR "Breeding program name: $breeding_program_name\n"; - - # my $projects = CXGN::BreedersToolbox::Projects->new({ schema => $chado_schema }); - # my $breeding_program = $projects->get_breeding_program_by_name($breeding_program_name); - - my %all_designs = %{$parsed_data}; - my %save; - $save{'errors'} = []; - - # print STDERR "breeding_program_name: $breeding_program_name \n"; - # print STDERR "infile: $infile \n"; - my $async_upload = CXGN::Tools::Run->new(); - $async_upload->run_async("perl $basepath/bin/upload_multiple_trial_design.pl -H $dbhost -D $dbname -P \"$dbpass\" -w \"$basepath\" -U \"$dbuser\" -b \"$breeding_program_name\" -i \"$infile\" -un \"$username\" -e \"$email_address\" -eo $email_option_enabled -r $temp_file_nd_experiment_id"); - #print STDERR "Check 5: ".localtime()."\n"; - if (scalar @{$save{'errors'}} > 0) { - print STDERR "Errors saving trials: ".@{$save{'errors'}}; - $c->stash->{rest} = {errors => $save{'errors'}}; - return; - # } elsif ($save->{'trial_id'}) { - } else { - my $dbh = $c->dbc->dbh(); - my $bs = CXGN::BreederSearch->new( { dbh=>$dbh, dbname=>$c->config->{dbname}, } ); - my $refresh = $bs->refresh_matviews($c->config->{dbhost}, $c->config->{dbname}, $c->config->{dbuser}, $c->config->{dbpass}, 'all_but_genoview', 'concurrent', $c->config->{basepath}); - $c->stash->{rest} = {success => "1",}; - return; - } + # Return success + $c->stash->{rest} = {success => "1"}; + return; } From fd7e24db787df21da085b0c767bbe3469bc36f70 Mon Sep 17 00:00:00 2001 From: David Waring Date: Mon, 4 Nov 2024 10:27:20 -0500 Subject: [PATCH 10/19] Mark MultipleTrialDesignExcelFormat plugin as deprecated --- .../ParseUpload/Plugin/MultipleTrialDesignExcelFormat.pm | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignExcelFormat.pm b/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignExcelFormat.pm index 82f88d14b2..ec93b98027 100644 --- a/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignExcelFormat.pm +++ b/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignExcelFormat.pm @@ -11,6 +11,10 @@ use CXGN::Stock::Seedlot; use CXGN::Calendar; use CXGN::Trial; +# +# DEPRECATED: This plugin has been replaced by the MultipleTrialDesignGeneric plugin +# + my @REQUIRED_COLUMNS = qw|trial_name breeding_program location year design_type description accession_name plot_number block_number|; my @OPTIONAL_COLUMNS = qw|plot_name trial_type plot_width plot_length field_size planting_date transplanting_date harvest_date is_a_control rep_number range_number row_number col_number seedlot_name num_seed_per_plot weight_gram_seed_per_plot entry_number|; # Any additional columns that are not required or optional will be used as a treatment From b6e38aff8a51a6368f6f90e296908d6cdd737d93 Mon Sep 17 00:00:00 2001 From: David Waring Date: Mon, 4 Nov 2024 10:46:28 -0500 Subject: [PATCH 11/19] Multi-Trial Generic Upload: add notification for file format to mason template --- mason/breeders_toolbox/trial/trial_upload_dialogs.mas | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mason/breeders_toolbox/trial/trial_upload_dialogs.mas b/mason/breeders_toolbox/trial/trial_upload_dialogs.mas index 3c472fd42a..551a3872e3 100644 --- a/mason/breeders_toolbox/trial/trial_upload_dialogs.mas +++ b/mason/breeders_toolbox/trial/trial_upload_dialogs.mas @@ -117,7 +117,9 @@ $design_types => ()
    +

    + <& /help/file_upload_type.mas, type => "Trials", optional_column => 1 &> @@ -719,7 +721,6 @@ $design_types => ()
    - Trials may be uploaded in an Excel file (.xls or .xlsx)
    Multiple Trial Designs:
    @@ -806,7 +807,7 @@ $design_types => ()
    -

    Only the required fields are necessary to include in the upload template. You may add any additional optional fields. The fields can be in any order.

    + <& /help/file_upload_type.mas, type => "Trials", optional_column => 1 &>
    From bf692c3ee2d27b86dcd2f4e8c9ceee696f6167ac Mon Sep 17 00:00:00 2001 From: David Waring Date: Fri, 8 Nov 2024 11:02:04 -0500 Subject: [PATCH 12/19] Multi-Trial Generic Upload: fix treatment retrieval --- .../Plugin/MultipleTrialDesignGeneric.pm | 57 ++++++++++--------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm b/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm index bdf3a0320a..9ae5b3583c 100644 --- a/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm +++ b/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm @@ -83,33 +83,34 @@ sub _validate_with_plugin { ## These are checks on the individual plot-level data ## foreach (@$parsed_data) { - my $row = $_->{'_row'}; - my $trial_name = $_->{'trial_name'}; - my $breeding_program = $_->{'breeding_program'}; - my $location = $_->{'location'}; - my $year = $_->{'year'}; - my $design_type = $_->{'design_type'}; - my $description = $_->{'description'}; - my $accession_name = $_->{'accession_name'}; - my $plot_number = $_->{'plot_number'}; - my $block_number = $_->{'block_number'}; - my $plot_name = $_->{'plot_name'}; - my $trial_type = $_->{'trial_type'}; - my $plot_width = $_->{'plot_width'}; - my $plot_length = $_->{'plot_length'}; - my $field_size = $_->{'field_size'}; - my $planting_date = $_->{'planting_date'}; - my $transplanting_date = $_->{'transplanting_date'}; - my $harvest_date = $_->{'harvest_date'}; - my $is_a_control = $_->{'is_a_control'}; - my $rep_number = $_->{'rep_number'}; - my $range_number = $_->{'range_number'}; - my $row_number = $_->{'row_number'}; - my $col_number = $_->{'col_number'}; - my $seedlot_name = $_->{'seedlot_name'}; - my $num_seed_per_plot = $_->{'num_seed_per_plot'}; - my $weight_gram_seed_per_plot = $_->{'weight_gram_seed_per_plot'}; - my $entry_number = $_->{'entry_number'}; + my $data = $_; + my $row = $data->{'_row'}; + my $trial_name = $data->{'trial_name'}; + my $breeding_program = $data->{'breeding_program'}; + my $location = $data->{'location'}; + my $year = $data->{'year'}; + my $design_type = $data->{'design_type'}; + my $description = $data->{'description'}; + my $accession_name = $data->{'accession_name'}; + my $plot_number = $data->{'plot_number'}; + my $block_number = $data->{'block_number'}; + my $plot_name = $data->{'plot_name'}; + my $trial_type = $data->{'trial_type'}; + my $plot_width = $data->{'plot_width'}; + my $plot_length = $data->{'plot_length'}; + my $field_size = $data->{'field_size'}; + my $planting_date = $data->{'planting_date'}; + my $transplanting_date = $data->{'transplanting_date'}; + my $harvest_date = $data->{'harvest_date'}; + my $is_a_control = $data->{'is_a_control'}; + my $rep_number = $data->{'rep_number'}; + my $range_number = $data->{'range_number'}; + my $row_number = $data->{'row_number'}; + my $col_number = $data->{'col_number'}; + my $seedlot_name = $data->{'seedlot_name'}; + my $num_seed_per_plot = $data->{'num_seed_per_plot'}; + my $weight_gram_seed_per_plot = $data->{'weight_gram_seed_per_plot'}; + my $entry_number = $data->{'entry_number'}; # Plot Number: must be a positive number if (!($plot_number =~ /^\d+?$/)) { @@ -195,7 +196,7 @@ sub _validate_with_plugin { # Treatment Values: must be either blank, 0, or 1 foreach my $treatment (@$treatments) { - my $treatment_value = $row->{$treatment}; + my $treatment_value = $data->{$treatment}; print STDERR "Row $row: $treatment = $treatment_value\n"; if ( $treatment_value && $treatment_value ne '' && $treatment_value ne '0' && $treatment_value ne '1' ) { push @error_messages, "Row $row: Treatment value for treatment $treatment should be either 1 (applied) or empty (not applied)."; From 4170a71fa6aec88ae9a4c3e685cdb6436841a2c7 Mon Sep 17 00:00:00 2001 From: David Waring Date: Fri, 8 Nov 2024 11:02:31 -0500 Subject: [PATCH 13/19] Multi-Trial Generic Upload: switch tests to generic upload plugin --- t/selenium2/breeders/upload_multi_trial_file.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/selenium2/breeders/upload_multi_trial_file.t b/t/selenium2/breeders/upload_multi_trial_file.t index 3a1cd49bc7..d33b68a25d 100644 --- a/t/selenium2/breeders/upload_multi_trial_file.t +++ b/t/selenium2/breeders/upload_multi_trial_file.t @@ -76,7 +76,7 @@ for my $extension ("xlsx", "xls") { #parse uploaded file with appropriate plugin my $parser = CXGN::Trial::ParseUpload->new(chado_schema=> $chado_schema, filename => $archived_filename_with_path); - $parser->load_plugin('MultipleTrialDesignExcelFormat'); + $parser->load_plugin('MultipleTrialDesignGeneric'); my $parsed_data = $parser->parse(); print STDERR "Parsed data: " . Dumper($parsed_data); ok(!$parsed_data->{error}, 'check if parse validate igd file fails for excel file'); From 21df7ac9fdedb63d76d34e672047a885320060e5 Mon Sep 17 00:00:00 2001 From: David Waring Date: Fri, 8 Nov 2024 15:03:08 -0500 Subject: [PATCH 14/19] Multi Trial Test: fix row index keys --- .../breeders/upload_multi_trial_file.t | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/t/selenium2/breeders/upload_multi_trial_file.t b/t/selenium2/breeders/upload_multi_trial_file.t index d33b68a25d..653cafd147 100644 --- a/t/selenium2/breeders/upload_multi_trial_file.t +++ b/t/selenium2/breeders/upload_multi_trial_file.t @@ -92,7 +92,7 @@ for my $extension ("xlsx", "xls") { 'description' => 'EPR', 'entry_numbers' => undef, 'design_details'=> { - '15' => { + '16' => { 'plot_name' => '198667HBEPR_popa_rep1_BRA33_9', 'range_number' => '6', 'is_a_control' => 1, @@ -103,7 +103,7 @@ for my $extension ("xlsx", "xls") { 'row_number' => '9', 'col_number' => '3' }, - '14' => { + '15' => { 'is_a_control' => 1, 'plot_number' => '8', 'stock_name' => 'UG120021', @@ -118,7 +118,7 @@ for my $extension ("xlsx", "xls") { 'col_number' => '3', 'block_number' => '1' }, - '13' => { + '14' => { 'stock_name' => 'UG120019', 'rep_number' => '1', 'plot_number' => '7', @@ -129,7 +129,7 @@ for my $extension ("xlsx", "xls") { 'row_number' => '7', 'block_number' => '1' }, - '11' => { + '12' => { 'plot_name' => '198667HBEPR_popa_rep1_BRA128_5', 'range_number' => '6', 'is_a_control' => 1, @@ -140,7 +140,7 @@ for my $extension ("xlsx", "xls") { 'row_number' => '5', 'col_number' => '3' }, - '12' => { + '13' => { 'range_number' => '6', 'plot_name' => '198667HBEPR_popa_rep1_BRA28_6', 'rep_number' => '2', @@ -166,7 +166,7 @@ for my $extension ("xlsx", "xls") { 'entry_numbers' => undef, 'location' => 'test_location', 'design_details' => { - '19' => { + '20' => { 'row_number' => '33', 'col_number' => '2', 'block_number' => '1', @@ -177,7 +177,7 @@ for my $extension ("xlsx", "xls") { 'plot_name' => '199275HBEPR_stom_rep1_CG1420-1_33', 'range_number' => '6' }, - '17' => { + '18' => { 'range_number' => '6', 'plot_name' => '199275HBEPR_stom_rep1_SOLITA_31', 'rep_number' => '1', @@ -188,7 +188,7 @@ for my $extension ("xlsx", "xls") { 'col_number' => '2', 'row_number' => '31' }, - '20' => { + '21' => { 'stock_name' => 'XG120073', 'rep_number' => '1', 'is_a_control' => 1, @@ -199,7 +199,7 @@ for my $extension ("xlsx", "xls") { 'row_number' => '34', 'block_number' => '1' }, - '18' => { + '19' => { 'row_number' => '32', 'col_number' => '3', 'block_number' => '1', @@ -210,7 +210,7 @@ for my $extension ("xlsx", "xls") { 'plot_name' => '199275HBEPR_stom_rep1_CG917-5_32', 'range_number' => '6' }, - '16' => { + '17' => { 'block_number' => '1', 'col_number' => '2', 'row_number' => '30', @@ -234,7 +234,7 @@ for my $extension ("xlsx", "xls") { 'trial_type' => 76515, 'location' => 'test_location', 'design_details' => { - '3' => { + '4' => { 'col_number' => '3', 'row_number' => '30', 'block_number' => '1', @@ -245,7 +245,7 @@ for my $extension ("xlsx", "xls") { 'range_number' => '4', 'plot_name' => '199934HBEPR_cara_rep1_UG120006_3' }, - '4' => { + '5' => { 'block_number' => '2', 'row_number' => '40', 'col_number' => '4', @@ -256,7 +256,7 @@ for my $extension ("xlsx", "xls") { 'rep_number' => '2', 'stock_name' => 'UG120008' }, - '5' => { + '6' => { 'col_number' => '5', 'row_number' => '50', 'block_number' => '2', @@ -267,7 +267,7 @@ for my $extension ("xlsx", "xls") { 'range_number' => '5', 'plot_name' => '199934HBEPR_cara_rep1_UG120009_5' }, - '1' => { + '2' => { 'stock_name' => 'UG120002', 'rep_number' => '2', 'is_a_control' => 1, @@ -278,7 +278,7 @@ for my $extension ("xlsx", "xls") { 'row_number' => '10', 'block_number' => '1' }, - '2' => { + '3' => { 'rep_number' => '2', 'stock_name' => 'UG120004', 'is_a_control' => 1, @@ -311,7 +311,7 @@ for my $extension ("xlsx", "xls") { 'breeding_program' => 'test', 'entry_numbers' => undef, 'design_details'=> { - '7' => { + '8' => { 'col_number' => '6', 'row_number' => '60', 'block_number' => '1', @@ -322,7 +322,7 @@ for my $extension ("xlsx", "xls") { 'range_number' => '6', 'plot_name' => '199947HBEPR_mora_rep1_UG120158_2' }, - '9' => { + '10' => { 'block_number' => '1', 'row_number' => '80', 'col_number' => '8', @@ -333,7 +333,7 @@ for my $extension ("xlsx", "xls") { 'rep_number' => '2', 'stock_name' => 'UG120160' }, - '8' => { + '9' => { 'row_number' => '70', 'col_number' => '7', 'block_number' => '1', @@ -344,7 +344,7 @@ for my $extension ("xlsx", "xls") { 'plot_name' => '199947HBEPR_mora_rep1_UG120159_3', 'range_number' => '7' }, - '6' => { + '7' => { 'plot_name' => '199947HBEPR_mora_rep1_UG120157_1', 'range_number' => '6', 'plot_number' => '1', @@ -355,7 +355,7 @@ for my $extension ("xlsx", "xls") { 'row_number' => '50', 'col_number' => '5' }, - '10' => { + '11' => { 'range_number' => '8', 'plot_name' => '199947HBEPR_mora_rep1_UG120161_5', 'rep_number' => '2', From 72a2f87e2ec5d44c8a5a3feb9b10ca7be10743f7 Mon Sep 17 00:00:00 2001 From: David Waring Date: Fri, 8 Nov 2024 15:03:34 -0500 Subject: [PATCH 15/19] Multi Trial Test: add csv test --- t/data/trial/demo_multiple_trial_design.csv | 21 +++++++++++++++++++ .../breeders/upload_multi_trial_file.t | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 t/data/trial/demo_multiple_trial_design.csv diff --git a/t/data/trial/demo_multiple_trial_design.csv b/t/data/trial/demo_multiple_trial_design.csv new file mode 100644 index 0000000000..6bad116b98 --- /dev/null +++ b/t/data/trial/demo_multiple_trial_design.csv @@ -0,0 +1,21 @@ +trial_name,breeding_program,location,year,design_type,description,trial_type,plot_width,plot_length,field_size,planting_date,harvest_date,plot_name,accession_name,plot_number,block_number,is_a_control,rep_number,range_number,row_number,col_number,seedlot_name,num_seed_per_plot,weight_gram_seed_per_plot +199934HBEPR_cara,test,test_location,1999,Augmented,EPR,Preliminary Yield Trial,5,5,8,1999-06-04,2000-03-14,199934HBEPR_cara_rep1_UG120002_1,UG120002,1,1,1,2,4,10,1,,2,10,1 +199934HBEPR_cara,test,test_location,1999,Augmented,EPR,Preliminary Yield Trial,5,5,8,1999-06-04,2000-03-14,199934HBEPR_cara_rep1_UG120004_2,UG120004,2,1,1,2,4,20,2,,2,20,2 +199934HBEPR_cara,test,test_location,1999,Augmented,EPR,Preliminary Yield Trial,5,5,8,1999-06-04,2000-03-14,199934HBEPR_cara_rep1_UG120006_3,UG120006,3,1,1,2,4,30,3,,2,30,3 +199934HBEPR_cara,test,test_location,1999,Augmented,EPR,Preliminary Yield Trial,5,5,8,1999-06-04,2000-03-14,199934HBEPR_cara_rep1_UG120008_4,UG120008,4,2,1,2,5,40,4,,2,40,4 +199934HBEPR_cara,test,test_location,1999,Augmented,EPR,Preliminary Yield Trial,5,5,8,1999-06-04,2000-03-14,199934HBEPR_cara_rep1_UG120009_5,UG120009,5,2,1,2,5,50,5,,2,40,5 +199947HBEPR_mora,test,test_location,1999,Augmented,EPR,Preliminary Yield Trial,5,5,8,1999-06-23,2000-10-19,199947HBEPR_mora_rep1_UG120157_1,UG120157,1,1,1,2,6,50,5,,2,50,5 +199947HBEPR_mora,test,test_location,1999,Augmented,EPR,Preliminary Yield Trial,5,5,8,1999-06-23,2000-10-19,199947HBEPR_mora_rep1_UG120158_2,UG120158,2,1,1,2,6,60,6,,2,60,6 +199947HBEPR_mora,test,test_location,1999,Augmented,EPR,Preliminary Yield Trial,5,5,8,1999-06-23,2000-10-19,199947HBEPR_mora_rep1_UG120159_3,UG120159,3,1,1,2,7,70,7,,2,70,7 +199947HBEPR_mora,test,test_location,1999,Augmented,EPR,Preliminary Yield Trial,5,5,8,1999-06-23,2000-10-19,199947HBEPR_mora_rep1_UG120160_4,UG120160,4,1,1,2,8,80,8,,2,80,8 +199947HBEPR_mora,test,test_location,1999,Augmented,EPR,Preliminary Yield Trial,5,5,8,1999-06-23,2000-10-19,199947HBEPR_mora_rep1_UG120161_5,UG120161,5,1,1,2,8,90,9,,2,90,9 +198667HBEPR_popa,test,test_location,1999,Augmented,EPR,Preliminary Yield Trial,5,5,8,1984-09-17,1985-08-16,198667HBEPR_popa_rep1_BRA128_5,UG120010,5,1,1,2,6,5,3,,2,100,10 +198667HBEPR_popa,test,test_location,1999,Augmented,EPR,Preliminary Yield Trial,5,5,8,1985-10-02,1986-09-15,198667HBEPR_popa_rep1_BRA28_6,UG120017,6,1,1,2,6,6,3,,2,110,11 +198667HBEPR_popa,test,test_location,1999,Augmented,EPR,Preliminary Yield Trial,5,5,8,1985-10-02,1986-09-15,198667HBEPR_popa_rep1_BRA30_7,UG120019,7,1,1,1,6,7,3,,2,120,12 +198667HBEPR_popa,test,test_location,1999,Augmented,EPR,Preliminary Yield Trial,5,5,8,1985-10-02,1986-09-15,198667HBEPR_popa_rep1_BRA131_8,UG120021,8,1,1,1,6,8,3,,2,130,13 +198667HBEPR_popa,test,test_location,1999,Augmented,EPR,Preliminary Yield Trial,5,5,8,1985-10-02,1986-09-15,198667HBEPR_popa_rep1_BRA33_9,UG120022,9,1,1,1,6,9,3,,2,140,14 +199275HBEPR_stom,test,test_location,1999,Augmented,EPR,Preliminary Yield Trial,5,5,8,1992-09-19,1993-08-04,199275HBEPR_stom_rep1_CM3306-4_30,XG120030,30,1,1,1,6,30,2,,2,150,15 +199275HBEPR_stom,test,test_location,1999,Augmented,EPR,Preliminary Yield Trial,5,5,8,1992-09-19,1993-08-04,199275HBEPR_stom_rep1_SOLITA_31,XG120061,31,1,1,1,6,31,2,,2,160,16 +199275HBEPR_stom,test,test_location,1999,Augmented,EPR,Preliminary Yield Trial,5,5,8,1992-09-19,1993-08-04,199275HBEPR_stom_rep1_CG917-5_32,XG120068,32,1,1,1,6,32,3,,2,170,17 +199275HBEPR_stom,test,test_location,1999,Augmented,EPR,Preliminary Yield Trial,5,5,8,1992-09-19,1993-08-04,199275HBEPR_stom_rep1_CG1420-1_33,XG120071,33,1,1,1,6,33,2,,2,180,18 +199275HBEPR_stom,test,test_location,1999,Augmented,EPR,Preliminary Yield Trial,5,5,8,1992-09-19,1993-08-04,199275HBEPR_stom_rep1_CM1785-6_34,XG120073,34,1,1,1,6,34,3,,2,190,19 diff --git a/t/selenium2/breeders/upload_multi_trial_file.t b/t/selenium2/breeders/upload_multi_trial_file.t index 653cafd147..2709176930 100644 --- a/t/selenium2/breeders/upload_multi_trial_file.t +++ b/t/selenium2/breeders/upload_multi_trial_file.t @@ -25,7 +25,7 @@ use Text::CSV; my $f = SGN::Test::Fixture->new(); -for my $extension ("xlsx", "xls") { +for my $extension ("xlsx", "xls", "csv") { my $c = SimulateC->new({ dbh => $f->dbh(), From 08f7f40d6c6c2f8019752e9583b1b4a86890a5dc Mon Sep 17 00:00:00 2001 From: David Waring Date: Fri, 8 Nov 2024 16:08:42 -0500 Subject: [PATCH 16/19] Multi-Trial Upload: remove usage of iFramePostForm plugin it hasn't been updated in 10+ years and sometimes fails to JSON parse the server response --- .../CXGN/BreedersToolbox/UploadTrial.js | 123 ++++++++---------- 1 file changed, 56 insertions(+), 67 deletions(-) diff --git a/js/source/legacy/CXGN/BreedersToolbox/UploadTrial.js b/js/source/legacy/CXGN/BreedersToolbox/UploadTrial.js index 7eb4c0e7c9..36da17148c 100644 --- a/js/source/legacy/CXGN/BreedersToolbox/UploadTrial.js +++ b/js/source/legacy/CXGN/BreedersToolbox/UploadTrial.js @@ -132,8 +132,62 @@ jQuery(document).ready(function ($) { jQuery("#upload_multiple_trials_warning_messages").html(''); jQuery("#upload_multiple_trials_error_messages").html(''); jQuery("#upload_multiple_trials_success_messages").html(''); - jQuery('#upload_multiple_trial_designs_form').attr("action", "/ajax/trial/upload_multiple_trial_designs_file"); - jQuery("#upload_multiple_trial_designs_form").submit(); + + var uploadedTrialLayoutFile = jQuery("#multiple_trial_designs_upload_file").val(); + if ( !uploadedTrialLayoutFile || uploadedTrialLayoutFile === '' ) { + alert("No file selected"); + return; + } + + jQuery("#working_modal").modal("show"); + jQuery.ajax({ + url: '/ajax/trial/upload_multiple_trial_designs_file', + type: 'POST', + data: new FormData(jQuery("#upload_multiple_trial_designs_form")[0]), + processData: false, + contentType: false, + success : function(response) { + jQuery("#working_modal").modal("hide"); + if (response.warnings) { + warnings = response.warnings; + warning_html = "
  • "+warnings.join("
  • ")+"
    • " + jQuery("#upload_multiple_trials_warning_messages").show(); + jQuery("#upload_multiple_trials_warning_messages").html('Warnings. Fix or ignore the following warnings and try again.

    '+warning_html); + return; + } + if (response.errors) { + errors = response.errors; + if (Array.isArray(errors)) { + error_html = "
  • "+errors.join("
  • ")+"
    • "; + } else { + error_html = "
  • "+errors+"
    • "; + } + jQuery("#upload_multiple_trials_error_messages").show(); + jQuery("#upload_multiple_trials_error_messages").html('Errors found. Fix the following problems and try again.

    '+error_html); + return; + } + if (response.success) { + refreshTrailJsTree(0); + jQuery("#upload_multiple_trials_success_messages").show(); + jQuery("#upload_multiple_trials_success_messages").html("Success! All trials successfully loaded."); + jQuery("#multiple_trial_designs_upload_submit").hide(); + jQuery("#upload_multiple_trials_success_button").show(); + return; + } + if (response.background) { + jQuery("#upload_multiple_trials_success_messages").show(); + jQuery("#upload_multiple_trials_success_messages").html("Your file has been uploaded. You will receive an email once the process is complete."); + jQuery("#multiple_trial_designs_upload_submit").hide(); + jQuery("#upload_multiple_trials_success_button").show(); + return; + } + }, + error: function() { + jQuery("#working_modal").modal("hide"); + jQuery("#upload_multiple_trials_error_messages").html("An error occurred while trying to upload this file. Please check the formatting and try again"); + return; + } + }); } @@ -198,7 +252,6 @@ jQuery(document).ready(function ($) { }); jQuery('#multiple_trial_designs_upload_submit').click(function () { - console.log("Registered click on multiple_trial_designs_upload_submit button"); upload_multiple_trial_designs_file(); }); @@ -313,70 +366,6 @@ jQuery(document).ready(function ($) { // Call the function initially in case the checkbox is already checked toggleEmailField(); - jQuery('#upload_multiple_trial_designs_form').iframePostForm({ - json: true, - post: function () { - var uploadedTrialLayoutFile = jQuery("#multiple_trial_designs_upload_file").val(); - - // Clear existing messages - jQuery("#upload_multiple_trials_warning_messages").html(''); - jQuery("#upload_multiple_trials_error_messages").html(''); - jQuery("#upload_multiple_trials_success_messages").html(''); - - if (uploadedTrialLayoutFile === '') { - alert("No file selected"); - return; - } - jQuery('#working_modal').modal("show"); - }, - complete: function(response) { - // console.log(response); - jQuery('#working_modal').modal("hide"); - if (response.warnings) { - warnings = response.warnings; - warning_html = "
  • "+warnings.join("
  • ")+"
    • " - jQuery("#upload_multiple_trials_warning_messages").show(); - jQuery("#upload_multiple_trials_warning_messages").html('Warnings. Fix or ignore the following warnings and try again.

    '+warning_html); - return; - } - if (response.errors) { - errors = response.errors; - if (Array.isArray(errors)) { - error_html = "
  • "+errors.join("
  • ")+"
    • "; - } else { - error_html = "
  • "+errors+"
    • "; - } - jQuery("#upload_multiple_trials_error_messages").show(); - jQuery("#upload_multiple_trials_error_messages").html('Errors found. Fix the following problems and try again.

    '+error_html); - console.log("check the errors: ", response.errors); - return; - } - if (response.success) { - console.log("Success!!"); - refreshTrailJsTree(0); - jQuery("#upload_multiple_trials_success_messages").show(); - jQuery("#upload_multiple_trials_success_messages").html("Success! All trials successfully loaded."); - jQuery("#multiple_trial_designs_upload_submit").hide(); - jQuery("#upload_multiple_trials_success_button").show(); - return; - } - if (response.background) { - jQuery("#upload_multiple_trials_success_messages").show(); - jQuery("#upload_multiple_trials_success_messages").html("Your file has been uploaded. You will receive an email once the process is complete."); - jQuery("#multiple_trial_designs_upload_submit").hide(); - jQuery("#upload_multiple_trials_success_button").show(); - return; - } - }, - error: function(response) { - if (!jQuery('#email_option_to_recieve_trial_upload_status').is(':checked')) { - jQuery("#working_modal").modal("hide"); - jQuery("#upload_multiple_trials_error_messages").html("An error occurred while trying to upload this file. Please check the formatting and try again"); - return; - } - } - }); - jQuery('#upload_multiple_trials_success_button').on('click', function(){ //alert('Trial was saved in the database'); jQuery('#upload_trial_dialog').modal('hide'); From 478b6b6709778eb7fc84725c07f458e0b5e1a32d Mon Sep 17 00:00:00 2001 From: David Waring Date: Fri, 15 Nov 2024 13:46:40 -0500 Subject: [PATCH 17/19] Multi-Trial Upload: fix issues with auto-generated plot names --- .../Plugin/MultipleTrialDesignGeneric.pm | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm b/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm index 9ae5b3583c..4a75e9a5c1 100644 --- a/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm +++ b/lib/CXGN/Trial/ParseUpload/Plugin/MultipleTrialDesignGeneric.pm @@ -94,7 +94,7 @@ sub _validate_with_plugin { my $accession_name = $data->{'accession_name'}; my $plot_number = $data->{'plot_number'}; my $block_number = $data->{'block_number'}; - my $plot_name = $data->{'plot_name'}; + my $plot_name = $data->{'plot_name'} || _create_plot_name($trial_name, $plot_number); my $trial_type = $data->{'trial_type'}; my $plot_width = $data->{'plot_width'}; my $plot_length = $data->{'plot_length'}; @@ -197,7 +197,6 @@ sub _validate_with_plugin { # Treatment Values: must be either blank, 0, or 1 foreach my $treatment (@$treatments) { my $treatment_value = $data->{$treatment}; - print STDERR "Row $row: $treatment = $treatment_value\n"; if ( $treatment_value && $treatment_value ne '' && $treatment_value ne '0' && $treatment_value ne '1' ) { push @error_messages, "Row $row: Treatment value for treatment $treatment should be either 1 (applied) or empty (not applied)."; } @@ -373,12 +372,13 @@ sub _validate_with_plugin { } # Plot Names: should not exist (as any stock) + my @plot_names = keys %seen_plot_names; my @already_used_plot_names; my $rs = $schema->resultset("Stock::Stock")->search({ 'is_obsolete' => { '!=' => 't' }, - 'uniquename' => { -in => $parsed_values->{'plot_name'} } + 'uniquename' => { -in => \@plot_names } }); - while (my $r=$rs->next) { + foreach my $r ($rs->all()) { push @already_used_plot_names, $r->uniquename(); } if (scalar(@already_used_plot_names) > 0) { @@ -388,14 +388,14 @@ sub _validate_with_plugin { # Seedlots: names must exist in the database my @seedlots_missing = @{$validator->validate($schema,'seedlots',$parsed_values->{'seedlot_name'})->{'missing'}}; if (scalar(@seedlots_missing) > 0) { - push @error_messages, "Seedlot(s) ".join(',',@seedlots_missing)." are not in the database. To use a seedlot as a seed source for a plot, the seedlot must already exist in the database."; + push @error_messages, "Seedlot(s) ".join(',',@seedlots_missing)." are not in the database. To use a seedlot as a seed source for a plot, the seedlot must already exist in the database."; } # Verify seedlot pairs: accession name of plot must match seedlot contents if ( scalar(@seedlot_pairs) > 0 ) { my $return = CXGN::Stock::Seedlot->verify_seedlot_accessions_crosses($schema, \@seedlot_pairs); - if (exists($return->{error})){ - push @error_messages, $return->{error}; + if (exists($return->{error})) { + push @error_messages, $return->{error}; } } From 6cbc8a089441cd6081997a84ad5a2a18bb70803c Mon Sep 17 00:00:00 2001 From: David Waring Date: Fri, 15 Nov 2024 14:04:12 -0500 Subject: [PATCH 18/19] Multi-Trial Upload: catch errors from upload plugin in backend script --- bin/upload_multiple_trial_design.pl | 51 +++++++++++++++++------------ 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/bin/upload_multiple_trial_design.pl b/bin/upload_multiple_trial_design.pl index be386bc8ee..1f8ed19f24 100644 --- a/bin/upload_multiple_trial_design.pl +++ b/bin/upload_multiple_trial_design.pl @@ -89,33 +89,42 @@ =head1 AUTHOR my $chado_schema = Bio::Chado::Schema->connect(sub { $dbh }, { on_connect_do => ['SET search_path TO public, sgn, metadata, phenome;'] }); print STDOUT "Database connection ok!\n"; -# Parse uploaded file with appropriate plugin -my $parser = CXGN::Trial::ParseUpload->new(chado_schema => $chado_schema, filename => $infile); -$parser->load_plugin('MultipleTrialDesignGeneric'); -my $parsed_data = $parser->parse(); - -# Parser has errors, print error messages and quit -if ($parser->has_parse_errors()) { - my $errors = $parser->get_parse_errors(); - foreach (@{$errors->{'error_messages'}}) { - push @errors, $_; +my $parsed_data; +my $validation_coderef = sub { + # Parse uploaded file with appropriate plugin + my $parser = CXGN::Trial::ParseUpload->new(chado_schema => $chado_schema, filename => $infile); + $parser->load_plugin('MultipleTrialDesignGeneric'); + $parsed_data = $parser->parse(); + + # Parser has errors, print error messages and quit + if ($parser->has_parse_errors()) { + my $errors = $parser->get_parse_errors(); + foreach (@{$errors->{'error_messages'}}) { + push @errors, $_; + } + finish(); } - finish(); -} -# Parser has warnings, print warning messages and quit unless we're ignoring warnings -if ($parser->has_parse_warnings()) { - unless ($ignore_warnings) { - my $warnings = $parser->get_parse_warnings(); - foreach (@{$warnings->{'warning_messages'}}) { - push @warnings, $_; + # Parser has warnings, print warning messages and quit unless we're ignoring warnings + if ($parser->has_parse_warnings()) { + unless ($ignore_warnings) { + my $warnings = $parser->get_parse_warnings(); + foreach (@{$warnings->{'warning_messages'}}) { + push @warnings, $_; + } + finish(); } - finish(); } -} +}; + +try { + $chado_schema->txn_do($validation_coderef); +} catch { + push @errors, $_; +}; # Check for parsed data -finish("There is no parsed data!") if !$parsed_data; +finish("There is parsed data from the input file!") if !$parsed_data; # Get User ID my $sp_person_id = CXGN::People::Person->get_person_by_username($dbh, $username); From 90f904916dc172668701aa2ad274ffce0981658c Mon Sep 17 00:00:00 2001 From: David Waring Date: Fri, 15 Nov 2024 14:05:36 -0500 Subject: [PATCH 19/19] Fix error message --- bin/upload_multiple_trial_design.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/upload_multiple_trial_design.pl b/bin/upload_multiple_trial_design.pl index 1f8ed19f24..e1903b6f99 100644 --- a/bin/upload_multiple_trial_design.pl +++ b/bin/upload_multiple_trial_design.pl @@ -124,7 +124,7 @@ =head1 AUTHOR }; # Check for parsed data -finish("There is parsed data from the input file!") if !$parsed_data; +finish("There is no parsed data from the input file!") if !$parsed_data; # Get User ID my $sp_person_id = CXGN::People::Person->get_person_by_username($dbh, $username);