Merge pull request #190 from matthewrmshin/fcm-make-archive

fcm make: new --archive option

doc/user_guide/annex_cfg.html

@@ -972,8 +972,22 @@ <h3 id="make.build">FCM Make Configuration: Build</h3>
 
     <dt id="make.build.prop.ar.flags">ar.flags</dt>
 
+    <dd>If a list of values is specified, the system will not inherit sources
+    with the name-spaces matching the specified values. If the value is a
+    <samp>*</samp>, the system will not inherit any sources.</dd>
     <dd>The options used by the archive command. (default=<samp>rs</samp>)</dd>
 
+    <dt id=
+    "make.build.prop.archive-ok-target-category">archive-ok-target-category
+    *</dt>
+
+    <dd>If <code>fcm make</code> is invoked with the <code>--archive</code>
+    option, sub-directories in categories containing intermediate build files
+    will be put into TAR-GZIP files. E.g. with the default setting,
+    <samp>include/</samp> and <samp>o/</samp> will become
+    <samp>include.tar.gz</samp> and <samp>o.tar.gz</samp> respectively.
+    (default=<samp>include o</samp>)</dd>
+
     <dt id="make.build.prop.cc">cc</dt>
 
     <dd>The C compiler and linker. (default=<samp>gcc</samp>)</dd>

doc/user_guide/command_ref.html

@@ -1334,6 +1334,17 @@ <h2 id="fcm-make">fcm make</h2>
 
     <dd>
       <dl>
+        <dt><code>--archive, -a</code></dt>
+
+        <dd>Switch on archive mode. In archive mode, intermediate files will be
+        put into TAR-GZIP archives on completion, e.g. extract system:
+        <samp>.fcm-make/cache/extract/</samp>, and build system:
+        <samp>build/include/</samp> and <samp>build/o/</samp>.
+
+          <p>The archive mode is not suitable for a make that will be inherited
+          or used by other makes.</p>
+        </dd>
+
         <dt><code>--config-file-path=PATH, -F PATH</code></dt>
 
         <dd>Specifies paths for searching configuration files specified in

doc/user_guide/make.html

@@ -167,8 +167,12 @@ <h3 id="concept.dest">FCM Make Concept: Directory Structure</h3>
   <dl>
     <dt><samp>.fcm-make/cache/</samp></dt>
 
-    <dd>An area for caching non-local items.  E.g.  the extract system exports
-    source trees from the version control system into this cache.</dd>
+    <dd>An area for caching non-local items. E.g. the extract system exports
+    source trees from the version control system into this cache. If <code>fcm
+    make</code> is invoked with the <code>--archive</code> option, contents in
+    this directory will be compressed into TAR-GZIP files, e.g.
+    <samp>.fcm-make/cache/extract/</samp> will become
+    <samp>.fcm-make/cache/extract.tar.gz</samp>.</dd>
 
     <dt><samp>fcm-make-as-parsed.cfg -&gt;
     .fcm-make/config-as-parsed.cfg</samp></dt>
@@ -940,6 +944,14 @@ <h3 id="build.basic">Build: Basic</h3>
   <p>Sub-directories are only created as necessary, so you may not find all of
   the above in your destination tree.</p>
 
+  <p>If <code>fcm make</code> is invoked with the <code>--archive</code> option,
+  sub-directories in categories containing intermediate build files (or any
+  category specified in the <a href=
+  "#make.build.prop.archive-ok-target-category">archive-ok-target-category</a>
+  property) will be put into TAR-GZIP files. E.g. with the default setting,
+  <samp>include/</samp> and <samp>o/</samp> will become
+  <samp>include.tar.gz</samp> and <samp>o.tar.gz</samp> respectively.</p>
+
   <p>To use a different compiler and/or compiler options for Fortran/C/C++, you
   use the <a href="annex_cfg.html#make.build.prop">build.prop</a> declaration to
   redefine the build properties. E.g.:</p>

lib/FCM/CLI/Parser.pm

@@ -158,15 +158,16 @@ our %OPTIONS_FOR = (
     $HELP_APP       => [@OPTION_OF{qw{quiet verbose}}],
     'keyword-print' => [@OPTION_OF{qw{verbose}}],
     'loc-layout'    => [@OPTION_OF{qw{verbose}}],
-    'merge'         => [@OPTION_OF{
-        qw{auto-log custom dry-run non-interactive quiet reverse revision verbose}
-    }],
-    'mkpatch'       => [@OPTION_OF{qw{exclude organisation revision}}],
     'make'          => [@OPTION_OF{
-        qw{ directory ignore-lock jobs config-file config-file-path name new
-            quiet verbose
+        qw{ archive directory ignore-lock jobs config-file config-file-path name
+            new quiet verbose
         }
     }],
+    'merge'         => [@OPTION_OF{
+        qw{ auto-log custom dry-run non-interactive quiet reverse revision
+            verbose}
+    }],
+    'mkpatch'       => [@OPTION_OF{qw{exclude organisation revision}}],
     'project-create'=> [@OPTION_OF{
         qw{non-interactive password svn-non-interactive}
     }],

lib/FCM/CLI/fcm-make.pod

@@ -17,6 +17,16 @@ configuration file.
 
 =over 4
 
+=item --archive, -a
+
+Switch on archive mode. In archive mode, intermediate files will be put into
+TAR-GZIP archives on completion, e.g. extract system:
+C<.fcm-make/cache/extract/>, and build system: C<build/include/> and
+C<build/o/>.
+
+The archive mode is not suitable for a make that will be inherited or used by
+other makes.
+
 =item --config-file-path=PATH, -F PATH
 
 Specify paths for searching configuration files specified in relative paths.

lib/FCM/System/Make/Build.pm

@@ -36,9 +36,9 @@ use FCM::System::Make::Build::FileType::H;
 use FCM::System::Make::Build::FileType::NS;
 use FCM::System::Make::Build::FileType::Script;
 use FCM::System::Make::Share::Subsystem;
-use File::Basename qw{basename dirname};
+use File::Basename qw{basename dirname fileparse};
 use File::Find qw{find};
-use File::Path qw{mkpath};
+use File::Path qw{mkpath rmtree};
 use File::Spec::Functions qw{abs2rel catfile rel2abs splitdir splitpath};
 use Storable qw{dclone};
 use Text::ParseWords qw{shellwords};
@@ -74,6 +74,7 @@ our %CONFIG_PARSER_OF = (
 # Default properties
 our %PROP_OF = (
     #                               [default       , ns-ok]
+    'archive-ok-target-category' => [q{include o}  , undef],
     'ignore-missing-dep-ns'      => [q{}           , undef],
     'no-step-source'             => [q{}           , undef],
     'no-inherit-source'          => [q{}           , undef],
@@ -656,6 +657,17 @@ sub _targets_update {
     }
     my $old_cwd = cwd();
     chdir($ctx->get_dest()) || die(sprintf("%s: %s\n", $ctx->get_dest(), $!));
+    # Extract any target category directories that are in .tar.gz
+    opendir(my $handle, '.');
+    while (my $name = readdir($handle)) {
+        if ((fileparse($name, '.tar.gz'))[2] eq '.tar.gz') {
+            my %value_of = %{$UTIL->shell_simple([qw{tar -x -z}, '-f', $name])};
+            if ($value_of{'rc'} == 0) {
+                unlink($name);
+            }
+        }
+    }
+    closedir($handle);
     # Determines the destination search path
     my $id = $ctx->get_id();
     @{$ctx->get_dests()} = (
@@ -688,6 +700,7 @@ sub _targets_update {
     # Back to original working directory
     chdir($old_cwd) || die(sprintf("%s: %s\n", $old_cwd, $!));
     if ($e) {
+        _finally($attrib_ref, $m_ctx, $ctx);
         die($e);
     }
     # Finally
@@ -719,8 +732,10 @@ sub _targets_update {
             FCM::Context::Event->MAKE_BUILD_TARGETS_FAIL,
             \@failed_targets
         );
+        _finally($attrib_ref, $m_ctx, $ctx);
         die("\n");
     }
+    _finally($attrib_ref, $m_ctx, $ctx);
 }
 
 # Updates a target.
@@ -1562,6 +1577,32 @@ sub _prev_hash_item_getter {
     sub {exists($p_item_of{$_[0]}) ? $p_item_of{$_[0]} : undef};
 }
 
+# Perform final actions.
+# Archive intermediate target directories if necessary.
+sub _finally {
+    my ($attrib_ref, $m_ctx, $ctx) = @_;
+    if (!$m_ctx->get_option_of('archive')) {
+        return;
+    }
+    my %can_archive = map {($_ => 1)} _props(
+        $attrib_ref, 'archive-ok-target-category', $ctx);
+    opendir(my $handle, $ctx->get_dest());
+    while (my $name = readdir($handle)) {
+        if ($can_archive{$name}) {
+            my @command = (
+                qw{tar -c -z}, '-C', $ctx->get_dest(),
+                '-f', catfile($ctx->get_dest(), $name . '.tar.gz'),
+                $name,
+            );
+            my %value_of = %{$UTIL->shell_simple(\@command)};
+            if ($value_of{'rc'} == 0) {
+                rmtree(catfile($ctx->get_dest(), $name));
+            }
+        }
+    }
+    closedir($handle);
+}
+
 # ------------------------------------------------------------------------------
 package FCM::System::Make::Build::State;
 use base qw{FCM::Class::HASH};

lib/FCM/System/Make/Extract.pm

@@ -831,6 +831,20 @@ sub _extract_incremental {
 # Updates the project tree caches.
 sub _project_tree_caches_update {
     my ($attrib_ref, $m_ctx, $ctx) = @_;
+    # If previous cache in .tar.gz, extract it
+    my $cache_tar_gz = $attrib_ref->{shared_util_of}{dest}->path(
+        $m_ctx, 'sys-cache', $ctx->get_id() . '.tar.gz',
+    );
+    if (-f $cache_tar_gz) {
+        my @command = (
+            qw{tar -x -z}, '-C', dirname($cache_tar_gz), '-f', $cache_tar_gz,
+        );
+        my %value_of = %{$UTIL->shell_simple(\@command)};
+        if ($value_of{'rc'} == 0) {
+            unlink($cache_tar_gz);
+        }
+    }
+    # Start the parallel task runner to do project tree caches update
     my $timer = $UTIL->timer();
     my $n_jobs = $m_ctx->get_option_of('jobs');
     my $n_trees = scalar(
@@ -859,6 +873,7 @@ sub _project_tree_caches_update {
     my $e = $@;
     $runner->destroy();
     if ($e) {
+        _finally($attrib_ref, $m_ctx, $ctx);
         die($e);
     }
     $UTIL->event(
@@ -1000,34 +1015,41 @@ sub _symlink_handle {
 sub _targets_update {
     my ($attrib_ref, $m_ctx, $ctx) = @_;
     my %basket_of = (status => {}, status_of_source => {});
-    while (my ($ns, $target) = each(%{$ctx->get_target_of()})) {
-        if ($target->get_status() eq $target->ST_UNKNOWN) {
-            my %source_of = %{$target->get_source_of()};
-            my $handler
-                = keys(%source_of) ? \&_target_update
-                :                    \&_target_delete
-                ;
-            $handler->($attrib_ref, $m_ctx, $ctx, $target);
-            my $base = delete($source_of{0});
-            my @diffs = grep {!$_->is_unchanged()} values(%source_of);
-            $target->set_status_of_source(
-                  !keys(%{$target->get_source_of()}) ? $target->ST_UNKNOWN
-                : $base->is_missing()                ? $target->ST_ADDED
-                : (grep {$_->is_missing()} @diffs)   ? $target->ST_DELETED
-                : scalar(@diffs) > 1                 ? $target->ST_MERGED
-                : scalar(@diffs)                     ? $target->ST_MODIFIED
-                :                                      $target->ST_UNCHANGED
-            );
-            $UTIL->event(
-                FCM::Context::Event->MAKE_EXTRACT_TARGET, $target,
-            );
+    eval {
+        while (my ($ns, $target) = each(%{$ctx->get_target_of()})) {
+            if ($target->get_status() eq $target->ST_UNKNOWN) {
+                my %source_of = %{$target->get_source_of()};
+                my $handler
+                    = keys(%source_of) ? \&_target_update
+                    :                    \&_target_delete
+                    ;
+                $handler->($attrib_ref, $m_ctx, $ctx, $target);
+                my $base = delete($source_of{0});
+                my @diffs = grep {!$_->is_unchanged()} values(%source_of);
+                $target->set_status_of_source(
+                      !keys(%{$target->get_source_of()}) ? $target->ST_UNKNOWN
+                    : $base->is_missing()                ? $target->ST_ADDED
+                    : (grep {$_->is_missing()} @diffs)   ? $target->ST_DELETED
+                    : scalar(@diffs) > 1                 ? $target->ST_MERGED
+                    : scalar(@diffs)                     ? $target->ST_MODIFIED
+                    :                                      $target->ST_UNCHANGED
+                );
+                $UTIL->event(
+                    FCM::Context::Event->MAKE_EXTRACT_TARGET, $target,
+                );
+            }
+            $basket_of{status}{$target->get_status()}++;
+            $basket_of{status_of_source}{$target->get_status_of_source()}++;
         }
-        $basket_of{status}{$target->get_status()}++;
-        $basket_of{status_of_source}{$target->get_status_of_source()}++;
+    };
+    if (my $e = $@) {
+        _finally($attrib_ref, $m_ctx, $ctx);
+        die($e);
     }
     $UTIL->event(
         FCM::Context::Event->MAKE_EXTRACT_TARGET_SUMMARY, \%basket_of,
     );
+    _finally($attrib_ref, $m_ctx, $ctx);
 }
 
 # Updates a deleted target.
@@ -1178,6 +1200,28 @@ sub _target_update_source_merge {
     return $path_of_mine;
 }
 
+# Perform final actions.
+# Archive cache directory if necessary.
+sub _finally {
+    my ($attrib_ref, $m_ctx, $ctx) = @_;
+    if (!$m_ctx->get_option_of('archive')) {
+        return;
+    }
+    my $cache = $attrib_ref->{shared_util_of}{dest}->path(
+        $m_ctx, 'sys-cache', $ctx->get_id(),
+    );
+    if (-d $cache) {
+        my @command = (
+            qw{tar -c -z}, '-C', dirname($cache), '-f', $cache . '.tar.gz',
+            $ctx->get_id(),
+        );
+        my %value_of = %{$UTIL->shell_simple(\@command)};
+        if ($value_of{'rc'} == 0) {
+            rmtree($cache);
+        }
+    }
+}
+
 # In scalar context, returns true if the contents or permissions of 2 paths
 # differ. In array context, returns ($is_diff_in_perms, $is_diff_in_content).
 sub _compare {