From 2c5a54d0d9fba8eca6591e20d730f95be500fe4b Mon Sep 17 00:00:00 2001 From: Matt Shin Date: Thu, 21 May 2015 14:42:19 +0100 Subject: [PATCH] fcm make: new --archive option If archive mode is specified, TAR-GZIP these items by default: * .fcm-make/cache/extract/ * build/include/ * build/o/ The categories for the sub-directories under build/ are configurable. --- doc/user_guide/annex_cfg.html | 14 ++++ doc/user_guide/command_ref.html | 11 ++++ doc/user_guide/make.html | 16 ++++- lib/FCM/CLI/Parser.pm | 13 ++-- lib/FCM/CLI/fcm-make.pod | 10 +++ lib/FCM/System/Make/Build.pm | 45 ++++++++++++- lib/FCM/System/Make/Extract.pm | 90 +++++++++++++++++++------- t/fcm-make/46-archive-mode.t | 111 ++++++++++++++++++++++++++++++++ 8 files changed, 277 insertions(+), 33 deletions(-) create mode 100755 t/fcm-make/46-archive-mode.t diff --git a/doc/user_guide/annex_cfg.html b/doc/user_guide/annex_cfg.html index 8e60f87b..a6c7afc1 100644 --- a/doc/user_guide/annex_cfg.html +++ b/doc/user_guide/annex_cfg.html @@ -972,8 +972,22 @@

FCM Make Configuration: Build

ar.flags
+
If a list of values is specified, the system will not inherit sources + with the name-spaces matching the specified values. If the value is a + *, the system will not inherit any sources.
The options used by the archive command. (default=rs)
+
archive-ok-target-category + *
+ +
If fcm make is invoked with the --archive + option, sub-directories in categories containing intermediate build files + will be put into TAR-GZIP files. E.g. with the default setting, + include/ and o/ will become + include.tar.gz and o.tar.gz respectively. + (default=include o)
+
cc
The C compiler and linker. (default=gcc)
diff --git a/doc/user_guide/command_ref.html b/doc/user_guide/command_ref.html index 73ff7bbc..185eb0fd 100644 --- a/doc/user_guide/command_ref.html +++ b/doc/user_guide/command_ref.html @@ -1334,6 +1334,17 @@

fcm make

+
--archive, -a
+ +
Switch on archive mode. In archive mode, intermediate files will be + put into TAR-GZIP archives on completion, e.g. extract system: + .fcm-make/cache/extract/, and build system: + build/include/ and build/o/. + +

The archive mode is not suitable for a make that will be inherited + or used by other makes.

+
+
--config-file-path=PATH, -F PATH
Specifies paths for searching configuration files specified in diff --git a/doc/user_guide/make.html b/doc/user_guide/make.html index 30a1f403..9d5fd4cd 100644 --- a/doc/user_guide/make.html +++ b/doc/user_guide/make.html @@ -167,8 +167,12 @@

FCM Make Concept: Directory Structure

.fcm-make/cache/
-
An area for caching non-local items. E.g. the extract system exports - source trees from the version control system into this cache.
+
An area for caching non-local items. E.g. the extract system exports + source trees from the version control system into this cache. If fcm + make is invoked with the --archive option, contents in + this directory will be compressed into TAR-GZIP files, e.g. + .fcm-make/cache/extract/ will become + .fcm-make/cache/extract.tar.gz.
fcm-make-as-parsed.cfg -> .fcm-make/config-as-parsed.cfg
@@ -940,6 +944,14 @@

Build: Basic

Sub-directories are only created as necessary, so you may not find all of the above in your destination tree.

+

If fcm make is invoked with the --archive option, + sub-directories in categories containing intermediate build files (or any + category specified in the archive-ok-target-category + property) will be put into TAR-GZIP files. E.g. with the default setting, + include/ and o/ will become + include.tar.gz and o.tar.gz respectively.

+

To use a different compiler and/or compiler options for Fortran/C/C++, you use the build.prop declaration to redefine the build properties. E.g.:

diff --git a/lib/FCM/CLI/Parser.pm b/lib/FCM/CLI/Parser.pm index ec5be367..afab456c 100644 --- a/lib/FCM/CLI/Parser.pm +++ b/lib/FCM/CLI/Parser.pm @@ -158,15 +158,16 @@ our %OPTIONS_FOR = ( $HELP_APP => [@OPTION_OF{qw{quiet verbose}}], 'keyword-print' => [@OPTION_OF{qw{verbose}}], 'loc-layout' => [@OPTION_OF{qw{verbose}}], - 'merge' => [@OPTION_OF{ - qw{auto-log custom dry-run non-interactive quiet reverse revision verbose} - }], - 'mkpatch' => [@OPTION_OF{qw{exclude organisation revision}}], 'make' => [@OPTION_OF{ - qw{ directory ignore-lock jobs config-file config-file-path name new - quiet verbose + qw{ archive directory ignore-lock jobs config-file config-file-path name + new quiet verbose } }], + 'merge' => [@OPTION_OF{ + qw{ auto-log custom dry-run non-interactive quiet reverse revision + verbose} + }], + 'mkpatch' => [@OPTION_OF{qw{exclude organisation revision}}], 'project-create'=> [@OPTION_OF{ qw{non-interactive password svn-non-interactive} }], diff --git a/lib/FCM/CLI/fcm-make.pod b/lib/FCM/CLI/fcm-make.pod index 01c2800e..be1ea003 100644 --- a/lib/FCM/CLI/fcm-make.pod +++ b/lib/FCM/CLI/fcm-make.pod @@ -17,6 +17,16 @@ configuration file. =over 4 +=item --archive, -a + +Switch on archive mode. In archive mode, intermediate files will be put into +TAR-GZIP archives on completion, e.g. extract system: +C<.fcm-make/cache/extract/>, and build system: C and +C. + +The archive mode is not suitable for a make that will be inherited or used by +other makes. + =item --config-file-path=PATH, -F PATH Specify paths for searching configuration files specified in relative paths. diff --git a/lib/FCM/System/Make/Build.pm b/lib/FCM/System/Make/Build.pm index 62633a1c..cbff538f 100644 --- a/lib/FCM/System/Make/Build.pm +++ b/lib/FCM/System/Make/Build.pm @@ -36,9 +36,9 @@ use FCM::System::Make::Build::FileType::H; use FCM::System::Make::Build::FileType::NS; use FCM::System::Make::Build::FileType::Script; use FCM::System::Make::Share::Subsystem; -use File::Basename qw{basename dirname}; +use File::Basename qw{basename dirname fileparse}; use File::Find qw{find}; -use File::Path qw{mkpath}; +use File::Path qw{mkpath rmtree}; use File::Spec::Functions qw{abs2rel catfile rel2abs splitdir splitpath}; use Storable qw{dclone}; use Text::ParseWords qw{shellwords}; @@ -74,6 +74,7 @@ our %CONFIG_PARSER_OF = ( # Default properties our %PROP_OF = ( # [default , ns-ok] + 'archive-ok-target-category' => [q{include o} , undef], 'ignore-missing-dep-ns' => [q{} , undef], 'no-step-source' => [q{} , undef], 'no-inherit-source' => [q{} , undef], @@ -656,6 +657,17 @@ sub _targets_update { } my $old_cwd = cwd(); chdir($ctx->get_dest()) || die(sprintf("%s: %s\n", $ctx->get_dest(), $!)); + # Extract any target category directories that are in .tar.gz + opendir(my $handle, '.'); + while (my $name = readdir($handle)) { + if ((fileparse($name, '.tar.gz'))[2] eq '.tar.gz') { + my %value_of = %{$UTIL->shell_simple([qw{tar -x -z}, '-f', $name])}; + if ($value_of{'rc'} == 0) { + unlink($name); + } + } + } + closedir($handle); # Determines the destination search path my $id = $ctx->get_id(); @{$ctx->get_dests()} = ( @@ -688,6 +700,7 @@ sub _targets_update { # Back to original working directory chdir($old_cwd) || die(sprintf("%s: %s\n", $old_cwd, $!)); if ($e) { + _finally($attrib_ref, $m_ctx, $ctx); die($e); } # Finally @@ -719,8 +732,10 @@ sub _targets_update { FCM::Context::Event->MAKE_BUILD_TARGETS_FAIL, \@failed_targets ); + _finally($attrib_ref, $m_ctx, $ctx); die("\n"); } + _finally($attrib_ref, $m_ctx, $ctx); } # Updates a target. @@ -1562,6 +1577,32 @@ sub _prev_hash_item_getter { sub {exists($p_item_of{$_[0]}) ? $p_item_of{$_[0]} : undef}; } +# Perform final actions. +# Archive intermediate target directories if necessary. +sub _finally { + my ($attrib_ref, $m_ctx, $ctx) = @_; + if (!$m_ctx->get_option_of('archive')) { + return; + } + my %can_archive = map {($_ => 1)} _props( + $attrib_ref, 'archive-ok-target-category', $ctx); + opendir(my $handle, $ctx->get_dest()); + while (my $name = readdir($handle)) { + if ($can_archive{$name}) { + my @command = ( + qw{tar -c -z}, '-C', $ctx->get_dest(), + '-f', catfile($ctx->get_dest(), $name . '.tar.gz'), + $name, + ); + my %value_of = %{$UTIL->shell_simple(\@command)}; + if ($value_of{'rc'} == 0) { + rmtree(catfile($ctx->get_dest(), $name)); + } + } + } + closedir($handle); +} + # ------------------------------------------------------------------------------ package FCM::System::Make::Build::State; use base qw{FCM::Class::HASH}; diff --git a/lib/FCM/System/Make/Extract.pm b/lib/FCM/System/Make/Extract.pm index 7d809cfa..f081c9fb 100644 --- a/lib/FCM/System/Make/Extract.pm +++ b/lib/FCM/System/Make/Extract.pm @@ -831,6 +831,20 @@ sub _extract_incremental { # Updates the project tree caches. sub _project_tree_caches_update { my ($attrib_ref, $m_ctx, $ctx) = @_; + # If previous cache in .tar.gz, extract it + my $cache_tar_gz = $attrib_ref->{shared_util_of}{dest}->path( + $m_ctx, 'sys-cache', $ctx->get_id() . '.tar.gz', + ); + if (-f $cache_tar_gz) { + my @command = ( + qw{tar -x -z}, '-C', dirname($cache_tar_gz), '-f', $cache_tar_gz, + ); + my %value_of = %{$UTIL->shell_simple(\@command)}; + if ($value_of{'rc'} == 0) { + unlink($cache_tar_gz); + } + } + # Start the parallel task runner to do project tree caches update my $timer = $UTIL->timer(); my $n_jobs = $m_ctx->get_option_of('jobs'); my $n_trees = scalar( @@ -859,6 +873,7 @@ sub _project_tree_caches_update { my $e = $@; $runner->destroy(); if ($e) { + _finally($attrib_ref, $m_ctx, $ctx); die($e); } $UTIL->event( @@ -1000,34 +1015,41 @@ sub _symlink_handle { sub _targets_update { my ($attrib_ref, $m_ctx, $ctx) = @_; my %basket_of = (status => {}, status_of_source => {}); - while (my ($ns, $target) = each(%{$ctx->get_target_of()})) { - if ($target->get_status() eq $target->ST_UNKNOWN) { - my %source_of = %{$target->get_source_of()}; - my $handler - = keys(%source_of) ? \&_target_update - : \&_target_delete - ; - $handler->($attrib_ref, $m_ctx, $ctx, $target); - my $base = delete($source_of{0}); - my @diffs = grep {!$_->is_unchanged()} values(%source_of); - $target->set_status_of_source( - !keys(%{$target->get_source_of()}) ? $target->ST_UNKNOWN - : $base->is_missing() ? $target->ST_ADDED - : (grep {$_->is_missing()} @diffs) ? $target->ST_DELETED - : scalar(@diffs) > 1 ? $target->ST_MERGED - : scalar(@diffs) ? $target->ST_MODIFIED - : $target->ST_UNCHANGED - ); - $UTIL->event( - FCM::Context::Event->MAKE_EXTRACT_TARGET, $target, - ); + eval { + while (my ($ns, $target) = each(%{$ctx->get_target_of()})) { + if ($target->get_status() eq $target->ST_UNKNOWN) { + my %source_of = %{$target->get_source_of()}; + my $handler + = keys(%source_of) ? \&_target_update + : \&_target_delete + ; + $handler->($attrib_ref, $m_ctx, $ctx, $target); + my $base = delete($source_of{0}); + my @diffs = grep {!$_->is_unchanged()} values(%source_of); + $target->set_status_of_source( + !keys(%{$target->get_source_of()}) ? $target->ST_UNKNOWN + : $base->is_missing() ? $target->ST_ADDED + : (grep {$_->is_missing()} @diffs) ? $target->ST_DELETED + : scalar(@diffs) > 1 ? $target->ST_MERGED + : scalar(@diffs) ? $target->ST_MODIFIED + : $target->ST_UNCHANGED + ); + $UTIL->event( + FCM::Context::Event->MAKE_EXTRACT_TARGET, $target, + ); + } + $basket_of{status}{$target->get_status()}++; + $basket_of{status_of_source}{$target->get_status_of_source()}++; } - $basket_of{status}{$target->get_status()}++; - $basket_of{status_of_source}{$target->get_status_of_source()}++; + }; + if (my $e = $@) { + _finally($attrib_ref, $m_ctx, $ctx); + die($e); } $UTIL->event( FCM::Context::Event->MAKE_EXTRACT_TARGET_SUMMARY, \%basket_of, ); + _finally($attrib_ref, $m_ctx, $ctx); } # Updates a deleted target. @@ -1178,6 +1200,28 @@ sub _target_update_source_merge { return $path_of_mine; } +# Perform final actions. +# Archive cache directory if necessary. +sub _finally { + my ($attrib_ref, $m_ctx, $ctx) = @_; + if (!$m_ctx->get_option_of('archive')) { + return; + } + my $cache = $attrib_ref->{shared_util_of}{dest}->path( + $m_ctx, 'sys-cache', $ctx->get_id(), + ); + if (-d $cache) { + my @command = ( + qw{tar -c -z}, '-C', dirname($cache), '-f', $cache . '.tar.gz', + $ctx->get_id(), + ); + my %value_of = %{$UTIL->shell_simple(\@command)}; + if ($value_of{'rc'} == 0) { + rmtree($cache); + } + } +} + # In scalar context, returns true if the contents or permissions of 2 paths # differ. In array context, returns ($is_diff_in_perms, $is_diff_in_content). sub _compare { diff --git a/t/fcm-make/46-archive-mode.t b/t/fcm-make/46-archive-mode.t new file mode 100755 index 00000000..7e0de299 --- /dev/null +++ b/t/fcm-make/46-archive-mode.t @@ -0,0 +1,111 @@ +#!/bin/bash +#------------------------------------------------------------------------------- +# (C) British Crown Copyright 2006-15 Met Office. +# +# This file is part of FCM, tools for managing and building source code. +# +# FCM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FCM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FCM. If not, see . +#------------------------------------------------------------------------------- +# Tests for "fcm make --archive" +#------------------------------------------------------------------------------- +. "$(dirname "$0")/test_header" +tests 11 +#------------------------------------------------------------------------------- +# Create a repository to extract +svnadmin create 'repos' +T_REPOS="file://${PWD}/repos" +mkdir 't' +cat >'t/hello.f90' <<'__FORTRAN__' +program hello +use world_mod, only: world +write(*, '(a,1x,a)') 'Hello', world +end program hello +__FORTRAN__ +cat >'t/world_mod.f90' <<'__FORTRAN__' +module world_mod +character(*), parameter :: world = 'Earth' +end module world_mod +__FORTRAN__ +svn import --no-auth-cache -q -m'Test' t "${T_REPOS}/hello/trunk" +rm -r 't' + +# Create a fcm-make.cfg to do some extract and build +cat >'fcm-make.cfg' <<__CFG__ +steps = extract build +extract.ns = hello +extract.location{primary}[hello] = ${T_REPOS}/hello +build.target{task} = link +build.prop{file-ext.bin} = +__CFG__ +#------------------------------------------------------------------------------- +TEST_KEY="${TEST_KEY_BASE}-on-new" +run_pass "${TEST_KEY}" fcm make -a +find '.fcm-make/cache' 'build' -type f | sort >"${TEST_KEY}.find" +file_cmp "${TEST_KEY}.find" "${TEST_KEY}.find" <<'__FIND__' +.fcm-make/cache/extract.tar.gz +build/bin/hello +build/include.tar.gz +build/o.tar.gz +__FIND__ + +touch 'new' +sleep 1 + +TEST_KEY="${TEST_KEY_BASE}-on-incr" +run_pass "${TEST_KEY}" fcm make -a +find '.fcm-make/cache' 'build' -type f -newer 'new' \ + | sort >"${TEST_KEY}.find.new" +file_cmp "${TEST_KEY}.find.new" "${TEST_KEY}.find.new" <<'__FIND__' +.fcm-make/cache/extract.tar.gz +build/include.tar.gz +build/o.tar.gz +__FIND__ +find '.fcm-make/cache' 'build' -type f '!' -newer 'new' \ + | sort >"${TEST_KEY}.find.old" +file_cmp "${TEST_KEY}.find.old" "${TEST_KEY}.find.old" <<'__FIND__' +build/bin/hello +__FIND__ + +TEST_KEY="${TEST_KEY_BASE}-on-incr-build-o" +run_pass "${TEST_KEY}" \ + fcm make -a 'build.prop{archive-ok-target-category}=o' +find '.fcm-make/cache' 'build' -type f -newer 'new' \ + | sort >"${TEST_KEY}.find.new" +file_cmp "${TEST_KEY}.find.new" "${TEST_KEY}.find.new" <<'__FIND__' +.fcm-make/cache/extract.tar.gz +build/o.tar.gz +__FIND__ +find '.fcm-make/cache' 'build' -type f '!' -newer 'new' \ + | sort >"${TEST_KEY}.find.old" +file_cmp "${TEST_KEY}.find.old" "${TEST_KEY}.find.old" <<'__FIND__' +build/bin/hello +build/include/world_mod.mod +__FIND__ + +run_pass "${TEST_KEY_BASE}-off" fcm make +find '.fcm-make/cache' 'build' -type f -newer 'new' \ + | sort >"${TEST_KEY}.find.new" +file_cmp "${TEST_KEY}.find.new" "${TEST_KEY}.find.new" <'/dev/null' +find '.fcm-make/cache' 'build' -type f '!' -newer 'new' \ + | sort >"${TEST_KEY}.find.old" +file_cmp "${TEST_KEY}.find.old" "${TEST_KEY}.find.old" <<'__FIND__' +.fcm-make/cache/extract/hello/0/hello.f90 +.fcm-make/cache/extract/hello/0/world_mod.f90 +build/bin/hello +build/include/world_mod.mod +build/o/hello.o +build/o/world_mod.o +__FIND__ +#------------------------------------------------------------------------------- +exit 0