From 40b2433abeaeae9aac7b9e4cf3d19bf3e94f720d Mon Sep 17 00:00:00 2001 From: william-dawson Date: Thu, 21 May 2020 13:00:25 +0900 Subject: [PATCH] Optimizations related to matrix conversion (#148) * First work on in place. * Optimization of the code related to conversion. This also incldues some code reduction work. * Something wrong with the CI? * Noticed a linting error. --- .github/workflows/ci.yml | 4 +- Source/Fortran/MatrixConversionModule.F90 | 29 +++---- Source/Fortran/MatrixReduceModule.F90 | 76 +++++++++++++++++++ Source/Fortran/PSMatrixModule.F90 | 39 +++++----- .../comm_includes/ReduceAndComposeMatrix.f90 | 11 +++ .../comm_includes/ReduceAndSumMatrix.f90 | 11 +++ .../FillMatrixFromTripletList.f90 | 70 +++++++++-------- .../FillMatrixIdentity.f90 | 39 ++++------ .../FillMatrixPermutation.f90 | 53 +++++-------- .../GatherMatrixToAll.f90 | 30 +------- .../distributed_includes/RedistributeData.f90 | 1 - UnitTests/test_psmatrix.py | 2 +- 12 files changed, 207 insertions(+), 158 deletions(-) create mode 100644 Source/Fortran/comm_includes/ReduceAndComposeMatrix.f90 create mode 100644 Source/Fortran/comm_includes/ReduceAndSumMatrix.f90 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 019b3b9d..2e3a18eb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,6 +82,4 @@ jobs: TESTEXAMPLES: ${{ matrix.testexamples }} - name: lint run: | - cd UnitTests - bash lint.sh - cd ../ + bash UnitTests/lint.sh diff --git a/Source/Fortran/MatrixConversionModule.F90 b/Source/Fortran/MatrixConversionModule.F90 index c4e831cb..dfa95292 100644 --- a/Source/Fortran/MatrixConversionModule.F90 +++ b/Source/Fortran/MatrixConversionModule.F90 @@ -3,11 +3,11 @@ !> to data structures used in other programs. MODULE MatrixConversionModule USE DataTypesModule, ONLY : NTREAL - USE MatrixMapsModule, ONLY : MapMatrix_psr USE PSMatrixModule, ONLY : Matrix_ps, ConvertMatrixToReal, CopyMatrix, & - & DestructMatrix + & DestructMatrix, MergeMatrixLocalBlocks, SplitMatrixToLocalBlocks USE PSMatrixAlgebraModule, ONLY : PairwiseMultiplyMatrix, ScaleMatrix, & & IncrementMatrix + USE SMatrixModule, ONLY : Matrix_lsr, DestructMatrix IMPLICIT NONE PRIVATE !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -27,16 +27,17 @@ SUBROUTINE SnapMatrixToSparsityPattern(mat, pattern) TYPE(Matrix_ps) :: filtered TYPE(Matrix_ps) :: pattern_1s TYPE(Matrix_ps) :: pattern_0s - TYPE(Matrix_ps) :: pattern_real - INTEGER :: II + TYPE(Matrix_lsr) :: local_mat !! First we need to make sure that the sparsity pattern is all 1s. IF (pattern%is_complex) THEN - CALL ConvertMatrixToReal(pattern, pattern_real) + CALL ConvertMatrixToReal(pattern, pattern_1s) ELSE - CALL CopyMatrix(pattern, pattern_real) + CALL CopyMatrix(pattern, pattern_1s) END IF - CALL MapMatrix_psr(pattern_real, pattern_1s, SetMatrixToOne) + CALL MergeMatrixLocalBlocks(pattern_1s, local_mat) + local_mat%values = 1.0_NTREAL + CALL SplitMatrixToLocalBlocks(pattern_1s, local_mat) !! Then all zeros CALL CopyMatrix(pattern_1s, pattern_0s) @@ -54,19 +55,9 @@ SUBROUTINE SnapMatrixToSparsityPattern(mat, pattern) !! Cleanup CALL DestructMatrix(pattern_1s) CALL DestructMatrix(pattern_0s) - CALL DestructMatrix(pattern_real) CALL DestructMatrix(filtered) + CALL DestructMatrix(local_mat) END SUBROUTINE SnapMatrixToSparsityPattern !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - FUNCTION SetMatrixToOne(row, column, val) RESULT(valid) - INTEGER, INTENT(INOUT), OPTIONAL :: row - INTEGER, INTENT(INOUT), OPTIONAL :: column - REAL(NTREAL), INTENT(INOUT), OPTIONAL :: val - LOGICAL :: valid - - val = 1.0_NTREAL - valid = .TRUE. - END FUNCTION SetMatrixToOne -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -END MODULE \ No newline at end of file +END MODULE diff --git a/Source/Fortran/MatrixReduceModule.F90 b/Source/Fortran/MatrixReduceModule.F90 index eb495ba6..62c472a3 100644 --- a/Source/Fortran/MatrixReduceModule.F90 +++ b/Source/Fortran/MatrixReduceModule.F90 @@ -41,10 +41,12 @@ MODULE MatrixReduceModule PUBLIC :: ReduceAndComposeMatrixSizes PUBLIC :: ReduceAndComposeMatrixData PUBLIC :: ReduceAndComposeMatrixCleanup + PUBLIC :: ReduceAndComposeMatrix !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! PUBLIC :: ReduceAndSumMatrixSizes PUBLIC :: ReduceAndSumMatrixData PUBLIC :: ReduceAndSumMatrixCleanup + PUBLIC :: ReduceAndSumMatrix !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! PUBLIC :: TestReduceSizeRequest PUBLIC :: TestReduceInnerRequest @@ -62,6 +64,10 @@ MODULE MatrixReduceModule MODULE PROCEDURE ReduceAndComposeMatrixCleanup_lsr MODULE PROCEDURE ReduceAndComposeMatrixCleanup_lsc END INTERFACE + INTERFACE ReduceAndComposeMatrix + MODULE PROCEDURE ReduceAndComposeMatrix_lsr + MODULE PROCEDURE ReduceAndComposeMatrix_lsc + END INTERFACE INTERFACE ReduceAndSumMatrixSizes MODULE PROCEDURE ReduceAndSumMatrixSizes_lsr MODULE PROCEDURE ReduceAndSumMatrixSizes_lsc @@ -74,6 +80,10 @@ MODULE MatrixReduceModule MODULE PROCEDURE ReduceAndSumMatrixCleanup_lsr MODULE PROCEDURE ReduceAndSumMatrixCleanup_lsc END INTERFACE + INTERFACE ReduceAndSumMatrix + MODULE PROCEDURE ReduceAndSumMatrix_lsr + MODULE PROCEDURE ReduceAndSumMatrix_lsc + END INTERFACE CONTAINS!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !> The first routine to call, gathers the sizes of the data to be sent. SUBROUTINE ReduceAndComposeMatrixSizes_lsr(matrix, communicator, & @@ -249,6 +259,38 @@ PURE SUBROUTINE ReduceAndComposeMatrixCleanup_lsc(matrix, gathered_matrix, & #endif END SUBROUTINE ReduceAndComposeMatrixCleanup_lsc +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !> Reduce and sum the matrices in one step. If you use this method, you + !> lose the opportunity for overlapping communication. + SUBROUTINE ReduceAndComposeMatrix_lsr(matrix, gathered_matrix, comm) + !> The matrix to send. + TYPE(Matrix_lsr), INTENT(IN) :: matrix + !> The matrix we are gathering. + TYPE(Matrix_lsr), INTENT(INOUT) :: gathered_matrix + !> The communicator to send along. + INTEGER, INTENT(INOUT) :: comm + !! Local Variables + TYPE(ReduceHelper_t) :: helper + + INCLUDE "comm_includes/ReduceAndComposeMatrix.f90" + + END SUBROUTINE ReduceAndComposeMatrix_lsr +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !> Reduce and sum the matrices in one step. If you use this method, you + !> lose the opportunity for overlapping communication. + SUBROUTINE ReduceAndComposeMatrix_lsc(matrix, gathered_matrix, comm) + !> The matrix to send. + TYPE(Matrix_lsc), INTENT(IN) :: matrix + !> The matrix we are gathering. + TYPE(Matrix_lsc), INTENT(INOUT) :: gathered_matrix + !> The communicator to send along. + INTEGER, INTENT(INOUT) :: comm + !! Local Variables + TYPE(ReduceHelper_t) :: helper + + INCLUDE "comm_includes/ReduceAndComposeMatrix.f90" + + END SUBROUTINE ReduceAndComposeMatrix_lsc !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !> The first routine to call, gathers the sizes of the data to be sent. SUBROUTINE ReduceAndSumMatrixSizes_lsr(matrix, communicator, & @@ -425,6 +467,40 @@ PURE SUBROUTINE ReduceAndSumMatrixCleanup_lsc(matrix, gathered_matrix, & END IF #endif END SUBROUTINE ReduceAndSumMatrixCleanup_lsc +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !> Reduce and sum the matrices in one step. If you use this method, you + !> lose the opportunity for overlapping communication. + SUBROUTINE ReduceAndSumMatrix_lsr(matrix, gathered_matrix, threshold, comm) + !> The matrix to send. + TYPE(Matrix_lsr), INTENT(IN) :: matrix + !> The gathered_matrix the matrix being gathered. + TYPE(Matrix_lsr), INTENT(INOUT) :: gathered_matrix + !> The threshold the threshold for flushing values. + REAL(NTREAL), INTENT(IN) :: threshold + !> The communicator to send along. + INTEGER, INTENT(INOUT) :: comm + !! Local Data + TYPE(ReduceHelper_t) :: helper + + INCLUDE "comm_includes/ReduceAndSumMatrix.f90" + END SUBROUTINE ReduceAndSumMatrix_lsr +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !> Reduce and sum the matrices in one step. If you use this method, you + !> lose the opportunity for overlapping communication. + SUBROUTINE ReduceAndSumMatrix_lsc(matrix, gathered_matrix, threshold, comm) + !> The matrix to send. + TYPE(Matrix_lsc), INTENT(IN) :: matrix + !> The threshold the threshold for flushing values. + TYPE(Matrix_lsc), INTENT(INOUT) :: gathered_matrix + !> The threshold the threshold for flushing values. + REAL(NTREAL), INTENT(IN) :: threshold + !> The communicator to send along. + INTEGER, INTENT(INOUT) :: comm + !! Local Data + TYPE(ReduceHelper_t) :: helper + + INCLUDE "comm_includes/ReduceAndSumMatrix.f90" + END SUBROUTINE ReduceAndSumMatrix_lsc !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !> Test if a request for the size of the matrices is complete. FUNCTION TestReduceSizeRequest(helper) RESULT(request_completed) diff --git a/Source/Fortran/PSMatrixModule.F90 b/Source/Fortran/PSMatrixModule.F90 index 9500f263..2a084472 100644 --- a/Source/Fortran/PSMatrixModule.F90 +++ b/Source/Fortran/PSMatrixModule.F90 @@ -9,11 +9,8 @@ MODULE PSMatrixModule & WriteListElement, WriteHeader USE MatrixMarketModule, ONLY : ParseMMHeader, MM_COMPLEX, WriteMMSize, & & WriteMMLine, MAX_LINE_LENGTH - USE MatrixReduceModule, ONLY : ReduceHelper_t, ReduceAndComposeMatrixSizes, & - & ReduceAndComposeMatrixData, ReduceAndComposeMatrixCleanup, & - & ReduceANdSumMatrixSizes, ReduceAndSumMatrixData, & - & ReduceAndSumMatrixCleanup, TestReduceSizeRequest, & - & TestReduceInnerRequest, TestReduceDataRequest + USE MatrixReduceModule, ONLY : ReduceHelper_t, ReduceAndComposeMatrix, & + & ReduceAndSumMatrix USE PermutationModule, ONLY : Permutation_t, ConstructDefaultPermutation USE ProcessGridModule, ONLY : ProcessGrid_t, global_grid, IsRoot, & & SplitProcessGrid @@ -773,23 +770,29 @@ END SUBROUTINE WriteMatrixToMatrixMarket_psc !> This routine fills in a matrix based on local triplet lists. Each process !> should pass in triplet lists with global coordinates. It does not matter !> where each triplet is stored, as long as global coordinates are given. - SUBROUTINE FillMatrixFromTripletList_psr(this,triplet_list,preduplicated_in) + !> However, if you explicitly set prepartitioned_in to True, all data must be + !> on the correct process. In that case, there is no communication required. + SUBROUTINE FillMatrixFromTripletList_psr(this, triplet_list, & + & preduplicated_in, prepartitioned_in) !> The matrix to fill. TYPE(Matrix_ps) :: this !> The triplet list of values. TYPE(TripletList_r) :: triplet_list !> If lists are preduplicated across slices set this to true. LOGICAL, INTENT(IN), OPTIONAL :: preduplicated_in + !> If all lists only contain local matrix elements set this to true. + LOGICAL, INTENT(IN), OPTIONAL :: prepartitioned_in !! Local Data TYPE(Matrix_ps) :: temp_matrix + TYPE(TripletList_r) :: shifted TYPE(TripletList_r) :: sorted_triplet_list TYPE(Matrix_lsr) :: local_matrix TYPE(Matrix_lsr) :: gathered_matrix !! Local Data TYPE(Permutation_t) :: basic_permutation - TYPE(ReduceHelper_t) :: gather_helper REAL(NTREAL), PARAMETER :: threshold = 0.0_NTREAL LOGICAL :: preduplicated + LOGICAL :: prepartitioned IF (this%is_complex) THEN CALL ConvertMatrixToReal(this, temp_matrix) @@ -803,23 +806,29 @@ END SUBROUTINE FillMatrixFromTripletList_psr !> This routine fills in a matrix based on local triplet lists. Each process !> should pass in triplet lists with global coordinates. It does not matter !> where each triplet is stored, as long as global coordinates are given. - SUBROUTINE FillMatrixFromTripletList_psc(this,triplet_list,preduplicated_in) + !> However, if you explicitly set prepartitioned_in to True, all data must be + !> on the correct process. In that case, there is no communication required. + SUBROUTINE FillMatrixFromTripletList_psc(this, triplet_list, & + & preduplicated_in, prepartitioned_in) !> The matrix to fill. TYPE(Matrix_ps) :: this !> The triplet list of values. TYPE(TripletList_c) :: triplet_list !> If lists are preduplicated across slices set this to true. LOGICAL, INTENT(IN), OPTIONAL :: preduplicated_in + !> If all lists only contain local matrix elements set this to true. + LOGICAL, INTENT(IN), OPTIONAL :: prepartitioned_in !! Local Data + TYPE(TripletList_c) :: shifted TYPE(TripletList_c) :: sorted_triplet_list TYPE(Matrix_lsc) :: local_matrix TYPE(Matrix_lsc) :: gathered_matrix !! Local Data TYPE(Matrix_ps) :: temp_matrix TYPE(Permutation_t) :: basic_permutation - TYPE(ReduceHelper_t) :: gather_helper REAL(NTREAL), PARAMETER :: threshold = 0.0_NTREAL LOGICAL :: preduplicated + LOGICAL :: prepartitioned IF (.NOT. this%is_complex) THEN CALL ConvertMatrixToComplex(this, temp_matrix) @@ -849,9 +858,6 @@ SUBROUTINE FillMatrixIdentity_psr(this) TYPE(Matrix_ps), INTENT(INOUT) :: this !! Local Data TYPE(TripletList_r) :: triplet_list - TYPE(TripletList_r) :: unsorted_triplet_list - TYPE(TripletList_r) :: sorted_triplet_list - TYPE(Matrix_lsr) :: local_matrix INCLUDE "distributed_includes/FillMatrixIdentity.f90" @@ -863,9 +869,6 @@ SUBROUTINE FillMatrixIdentity_psc(this) TYPE(Matrix_ps), INTENT(INOUT) :: this !! Local Data TYPE(TripletList_c) :: triplet_list - TYPE(TripletList_c) :: unsorted_triplet_list - TYPE(TripletList_c) :: sorted_triplet_list - TYPE(Matrix_lsc) :: local_matrix INCLUDE "distributed_includes/FillMatrixIdentity.f90" @@ -908,9 +911,6 @@ SUBROUTINE FillMatrixPermutation_psr(this, permutation_vector, rows) LOGICAL, INTENT(IN) :: rows !! Local Data TYPE(TripletList_r) :: triplet_list - TYPE(TripletList_r) :: unsorted_triplet_list - TYPE(TripletList_r) :: sorted_triplet_list - TYPE(Matrix_lsr) :: local_matrix INCLUDE "distributed_includes/FillMatrixPermutation.f90" @@ -926,9 +926,6 @@ SUBROUTINE FillMatrixPermutation_psc(this, permutation_vector, rows) LOGICAL, INTENT(IN) :: rows !! Local Data TYPE(TripletList_c) :: triplet_list - TYPE(TripletList_c) :: unsorted_triplet_list - TYPE(TripletList_c) :: sorted_triplet_list - TYPE(Matrix_lsc) :: local_matrix INCLUDE "distributed_includes/FillMatrixPermutation.f90" diff --git a/Source/Fortran/comm_includes/ReduceAndComposeMatrix.f90 b/Source/Fortran/comm_includes/ReduceAndComposeMatrix.f90 new file mode 100644 index 00000000..a37132e4 --- /dev/null +++ b/Source/Fortran/comm_includes/ReduceAndComposeMatrix.f90 @@ -0,0 +1,11 @@ + CALL ReduceAndComposeMatrixSizes(matrix, comm, gathered_matrix, helper) + DO WHILE(.NOT. TestReduceSizeRequest(helper)) + END DO + + CALL ReduceAndComposeMatrixData(matrix, comm, gathered_matrix, helper) + DO WHILE(.NOT. TestReduceInnerRequest(helper)) + END DO + DO WHILE(.NOT. TestReduceDataRequest(helper)) + END DO + + CALL ReduceAndComposeMatrixCleanup(matrix, gathered_matrix, helper) \ No newline at end of file diff --git a/Source/Fortran/comm_includes/ReduceAndSumMatrix.f90 b/Source/Fortran/comm_includes/ReduceAndSumMatrix.f90 new file mode 100644 index 00000000..6bea1353 --- /dev/null +++ b/Source/Fortran/comm_includes/ReduceAndSumMatrix.f90 @@ -0,0 +1,11 @@ + CALL ReduceAndSumMatrixSizes(matrix, comm, gathered_matrix, helper) + DO WHILE(.NOT. TestReduceSizeRequest(helper)) + END DO + + CALL ReduceAndSumMatrixData(matrix, gathered_matrix, comm, helper) + DO WHILE(.NOT. TestReduceInnerRequest(helper)) + END DO + DO WHILE(.NOT. TestReduceDataRequest(helper)) + END DO + + CALL ReduceAndSumMatrixCleanup(matrix, gathered_matrix, threshold, helper) \ No newline at end of file diff --git a/Source/Fortran/distributed_includes/FillMatrixFromTripletList.f90 b/Source/Fortran/distributed_includes/FillMatrixFromTripletList.f90 index 6b1f0637..f2213b8f 100644 --- a/Source/Fortran/distributed_includes/FillMatrixFromTripletList.f90 +++ b/Source/Fortran/distributed_includes/FillMatrixFromTripletList.f90 @@ -1,43 +1,53 @@ + !! Optional Parameteres IF (.NOT. PRESENT(preduplicated_in)) THEN preduplicated = .FALSE. ELSE preduplicated = preduplicated_in END IF + IF (.NOT. PRESENT(prepartitioned_in)) THEN + prepartitioned = .FALSE. + ELSE + prepartitioned = prepartitioned_in + END IF + CALL StartTimer("FillFromTriplet") - !! First we redistribute the triplet list to get all the local data - !! on the correct process. - CALL ConstructDefaultPermutation(basic_permutation, & - & this%logical_matrix_dimension) - CALL RedistributeData(this,basic_permutation%index_lookup, & - & basic_permutation%reverse_index_lookup, triplet_list, & - & sorted_triplet_list) - !! Now we can just construct a local matrix. - CALL ConstructMatrixFromTripletList(local_matrix, sorted_triplet_list, & - & this%local_rows, this%local_columns) - !! And reduce over the Z dimension. This can be accomplished by - !! summing up. - IF (.NOT. preduplicated .AND. & - & .NOT. this%process_grid%num_process_slices .EQ. 1) THEN - CALL ReduceAndSumMatrixSizes(local_matrix, & - & this%process_grid%between_slice_comm, gathered_matrix, & - & gather_helper) - DO WHILE(.NOT. TestReduceSizeRequest(gather_helper)) - END DO - CALL ReduceAndSumMatrixData(local_matrix, gathered_matrix, & - & this%process_grid%between_slice_comm, gather_helper) - DO WHILE(.NOT. TestReduceInnerRequest(gather_helper)) - END DO - DO WHILE(.NOT. TestReduceDataRequest(gather_helper)) - END DO - CALL ReduceAndSumMatrixCleanup(local_matrix, gathered_matrix, threshold, & - & gather_helper) - CALL SplitMatrixToLocalBlocks(this, gathered_matrix) - ELSE + IF (prepartitioned) THEN + !! Shift and sort the local entries. + shifted = triplet_list + CALL ShiftTripletList(shifted, 1 - this%start_row, 1 - this%start_column) + CALL SortTripletList(shifted, this%local_columns, & + & this%local_rows, sorted_triplet_list) + !! Build + CALL ConstructMatrixFromTripletList(local_matrix, sorted_triplet_list, & + & this%local_rows, this%local_columns) CALL SplitMatrixToLocalBlocks(this, local_matrix) + ELSE + !! First we redistribute the triplet list to get all the local data + !! on the correct process. + CALL ConstructDefaultPermutation(basic_permutation, & + & this%logical_matrix_dimension) + CALL RedistributeData(this,basic_permutation%index_lookup, & + & basic_permutation%reverse_index_lookup, triplet_list, & + & sorted_triplet_list) + + !! Now we can just construct a local matrix. + CALL ConstructMatrixFromTripletList(local_matrix, sorted_triplet_list, & + & this%local_rows, this%local_columns) + + !! And reduce over the Z dimension. + IF (.NOT. preduplicated .AND. & + & .NOT. this%process_grid%num_process_slices .EQ. 1) THEN + CALL ReduceAndSumMatrix(local_matrix, gathered_matrix, threshold, & + & this%process_grid%between_slice_comm) + CALL SplitMatrixToLocalBlocks(this, gathered_matrix) + ELSE + CALL SplitMatrixToLocalBlocks(this, local_matrix) + END IF END IF - CALL StopTimer("FillFromTriplet") CALL DestructMatrix(local_matrix) CALL DestructTripletList(sorted_triplet_list) + + CALL StopTimer("FillFromTriplet") diff --git a/Source/Fortran/distributed_includes/FillMatrixIdentity.f90 b/Source/Fortran/distributed_includes/FillMatrixIdentity.f90 index 904a90f1..24d1b4bb 100644 --- a/Source/Fortran/distributed_includes/FillMatrixIdentity.f90 +++ b/Source/Fortran/distributed_includes/FillMatrixIdentity.f90 @@ -1,35 +1,26 @@ !! Local Data - INTEGER :: i, j - INTEGER :: total_values + INTEGER :: II, JJ + INTEGER :: total !! There can't be more than one entry per row CALL ConstructTripletList(triplet_list, this%local_rows) - total_values = 0 + total = 0 !! Find local identity values - row_iter: DO j = 1, this%local_rows - column_iter: DO i = 1, this%local_columns - IF (j + this%start_row - 1 .EQ. i + this%start_column - 1 .AND. & - & j+this%start_row-1 .LE. this%actual_matrix_dimension) THEN - total_values = total_values + 1 - triplet_list%data(total_values)%index_column = i - triplet_list%data(total_values)%index_row = j - triplet_list%data(total_values)%point_value = 1.0 + DO JJ = this%start_row, this%end_row - 1 + DO II = this%start_column, this%end_column - 1 + IF (JJ .EQ. II .AND. JJ .LE. this%actual_matrix_dimension) THEN + total = total + 1 + triplet_list%data(total)%index_column = II + triplet_list%data(total)%index_row = JJ + triplet_list%data(total)%point_value = 1.0 END IF - END DO column_iter - END DO row_iter + END DO + END DO + triplet_list%CurrentSize = total !! Finish constructing - CALL ConstructTripletList(unsorted_triplet_list, total_values) - unsorted_triplet_list%data = triplet_list%data(:total_values) - CALL SortTripletList(unsorted_triplet_list,this%local_columns,& - & this%local_rows, sorted_triplet_list) - CALL ConstructMatrixFromTripletList(local_matrix, sorted_triplet_list, & - & this%local_rows, this%local_columns) + CALL FillMatrixFromTripletList(this, triplet_list, prepartitioned_in=.TRUE.) - CALL SplitMatrixToLocalBlocks(this, local_matrix) - - CALL DestructMatrix(local_matrix) + !! Cleanup CALL DestructTripletList(triplet_list) - CALL DestructTripletList(unsorted_triplet_list) - CALL DestructTripletList(sorted_triplet_list) diff --git a/Source/Fortran/distributed_includes/FillMatrixPermutation.f90 b/Source/Fortran/distributed_includes/FillMatrixPermutation.f90 index 64b2185f..6faa1182 100644 --- a/Source/Fortran/distributed_includes/FillMatrixPermutation.f90 +++ b/Source/Fortran/distributed_includes/FillMatrixPermutation.f90 @@ -1,49 +1,36 @@ !! Local Data - INTEGER :: total_values - INTEGER :: counter - INTEGER :: local_row, local_column + INTEGER :: total + INTEGER :: II !! Build Local Triplet List !! There can't be more than one entry per row CALL ConstructTripletList(triplet_list, this%local_rows) - total_values = 0 + total = 0 IF (rows) THEN - DO counter=this%start_row,this%end_row-1 - IF (permutation_vector(counter) .GE. this%start_column .AND. & - & permutation_vector(counter) .LT. this%end_column) THEN - total_values = total_values + 1 - local_column = permutation_vector(counter) - this%start_column + 1 - local_row = counter - this%start_row + 1 - triplet_list%data(total_values)%index_column = local_column - triplet_list%data(total_values)%index_row = local_row - triplet_list%data(total_values)%point_value = 1.0 + DO II=this%start_row,this%end_row-1 + IF (permutation_vector(II) .GE. this%start_column .AND. & + & permutation_vector(II) .LT. this%end_column) THEN + total = total + 1 + triplet_list%data(total)%index_column = permutation_vector(II) + triplet_list%data(total)%index_row = II + triplet_list%data(total)%point_value = 1.0 END IF END DO ELSE - DO counter=this%start_column,this%end_column-1 - IF (permutation_vector(counter) .GE. this%start_row .AND. & - & permutation_vector(counter) .LT. this%end_row) THEN - total_values = total_values + 1 - local_column = counter - this%start_column + 1 - local_row = permutation_vector(counter) - this%start_row + 1 - triplet_list%data(total_values)%index_column = local_column - triplet_list%data(total_values)%index_row = local_row - triplet_list%data(total_values)%point_value = 1.0 + DO II=this%start_column,this%end_column-1 + IF (permutation_vector(II) .GE. this%start_row .AND. & + & permutation_vector(II) .LT. this%end_row) THEN + total = total + 1 + triplet_list%data(total)%index_column = II + triplet_list%data(total)%index_row = permutation_vector(II) + triplet_list%data(total)%point_value = 1.0 END IF END DO END IF + triplet_list%CurrentSize = total !! Finish constructing - CALL ConstructTripletList(unsorted_triplet_list, total_values) - unsorted_triplet_list%data = triplet_list%data(:total_values) - CALL SortTripletList(unsorted_triplet_list, this%local_columns, & - & this%local_rows, sorted_triplet_list) - CALL ConstructMatrixFromTripletList(local_matrix, sorted_triplet_list, & - & this%local_rows, this%local_columns) + CALL FillMatrixFromTripletList(this, triplet_list, prepartitioned_in=.TRUE.) - CALL SplitMatrixToLocalBlocks(this, local_matrix) - - CALL DestructMatrix(local_matrix) + !! Cleanup CALL DestructTripletList(triplet_list) - CALL DestructTripletList(unsorted_triplet_list) - CALL DestructTripletList(sorted_triplet_list) diff --git a/Source/Fortran/distributed_includes/GatherMatrixToAll.f90 b/Source/Fortran/distributed_includes/GatherMatrixToAll.f90 index 46966f88..eb14cb47 100644 --- a/Source/Fortran/distributed_includes/GatherMatrixToAll.f90 +++ b/Source/Fortran/distributed_includes/GatherMatrixToAll.f90 @@ -1,37 +1,15 @@ - !! Local Data - TYPE(ReduceHelper_t) :: row_helper - TYPE(ReduceHelper_t) :: column_helper CALL MergeMatrixLocalBlocks(this, local) !! Merge Columns CALL TransposeMatrix(local, localT) - CALL ReduceAndComposeMatrixSizes(localT, this%process_grid%column_comm, & - & merged_columns, column_helper) - DO WHILE(.NOT. TestReduceSizeRequest(column_helper)) - END DO - CALL ReduceAndComposeMatrixData(localT, this%process_grid%column_comm, & - & merged_columns, column_helper) - DO WHILE(.NOT. TestReduceInnerRequest(column_helper)) - END DO - DO WHILE(.NOT. TestReduceDataRequest(column_helper)) - END DO - CALL ReduceAndComposeMatrixCleanup(localT, merged_columns, column_helper) + CALL ReduceAndComposeMatrix(localT, merged_columns, & + & this%process_grid%column_comm) !! Merge Rows CALL TransposeMatrix(merged_columns, merged_columnsT) - CALL ReduceAndComposeMatrixSizes(merged_columnsT, & - & this%process_grid%row_comm, gathered, row_helper) - DO WHILE(.NOT. TestReduceSizeRequest(row_helper)) - END DO - CALL ReduceAndComposeMatrixData(merged_columnsT, & - & this%process_grid%row_comm, gathered, row_helper) - DO WHILE(.NOT. TestReduceInnerRequest(row_helper)) - END DO - DO WHILE(.NOT. TestReduceDataRequest(row_helper)) - END DO - CALL ReduceAndComposeMatrixCleanup(merged_columnsT, gathered, & - & row_helper) + CALL ReduceAndComposeMatrix(merged_columnsT, gathered, & + & this%process_grid%row_comm) !! Remove the excess rows and columns that come from the logical size. CALL ConstructEmptyMatrix(local_mat, this%actual_matrix_dimension, & diff --git a/Source/Fortran/distributed_includes/RedistributeData.f90 b/Source/Fortran/distributed_includes/RedistributeData.f90 index 50780aac..fb90f200 100644 --- a/Source/Fortran/distributed_includes/RedistributeData.f90 +++ b/Source/Fortran/distributed_includes/RedistributeData.f90 @@ -40,7 +40,6 @@ CALL AppendToTripletList(send_triplet_lists(process_id+1), temp_triplet) END DO - !! Actual Send CALL RedistributeTripletLists(send_triplet_lists, & & this%process_grid%within_slice_comm, gathered_list) diff --git a/UnitTests/test_psmatrix.py b/UnitTests/test_psmatrix.py index 7c71188b..18621a3b 100644 --- a/UnitTests/test_psmatrix.py +++ b/UnitTests/test_psmatrix.py @@ -429,7 +429,7 @@ def __call__(self): def test_snap(self): '''Test the sparsity pattern setting routine.''' - from scipy.sparse import dok_matrix, csr_matrix + from scipy.sparse import dok_matrix for param in self.parameters: matrix1 = param.create_matrix(self.complex)