Merge pull request #1282 from metacpan/haarg/no-index-option

remove index option from scripts and reimplement it for backup
metacpan · Oct 4, 2024 · 13f4036 · 13f4036
2 parents 466f634 + 5d7bf0b
commit 13f4036
Show file tree

Hide file tree

Showing 6 changed files with 49 additions and 66 deletions.
diff --git a/bin/cron/author.sh b/bin/cron/author.sh
@@ -3,7 +3,4 @@
 # export ES_SCRIPT_INDEX=author_01
 # /home/metacpan/bin/metacpan-api-carton-exec bin/metacpan author --index author_01
 
-export ES_SCRIPT_INDEX=cpan_v1_01
-/home/metacpan/bin/metacpan-api-carton-exec bin/metacpan author --index cpan_v1_01
-
-unset ES_SCRIPT_INDEX
+/home/metacpan/bin/metacpan-api-carton-exec bin/metacpan author
diff --git a/bin/cron/backups.sh b/bin/cron/backups.sh
@@ -1,16 +1,6 @@
 #!/bin/sh
 
-#export ES_SCRIPT_INDEX=favorite_01
-#/home/metacpan/bin/metacpan-api-carton-exec bin/metacpan backup --index favorite_01 --type favorite
-
-#export ES_SCRIPT_INDEX=author_01
-#/home/metacpan/bin/metacpan-api-carton-exec bin/metacpan backup --index author_01 --type author
-
-export ES_SCRIPT_INDEX=cpan_v1_01
 /home/metacpan/bin/metacpan-api-carton-exec bin/metacpan backup --index cpan_v1_01 --type favorite
 /home/metacpan/bin/metacpan-api-carton-exec bin/metacpan backup --index cpan_v1_01 --type author
 
-export ES_SCRIPT_INDEX=user
 /home/metacpan/bin/metacpan-api-carton-exec bin/metacpan backup --index user
-
-unset ES_SCRIPT_INDEX
diff --git a/lib/MetaCPAN/Model.pm b/lib/MetaCPAN/Model.pm
@@ -44,7 +44,7 @@ analyzer edge => (
 
 index cpan => (
     namespace => 'MetaCPAN::Document',
-    alias_for => ( $ENV{'ES_SCRIPT_INDEX'} || 'cpan_v1_01' ),
+    alias_for => 'cpan_v1_01',
     shards    => 3
 );
 

diff --git a/lib/MetaCPAN/Role/Script.pm b/lib/MetaCPAN/Role/Script.pm
@@ -86,16 +86,6 @@ has model => (
     traits   => ['NoGetopt'],
 );
 
-has index => (
-    reader        => '_index',
-    is            => 'ro',
-    isa           => Str,
-    lazy          => 1,
-    default       => 'cpan',
-    documentation =>
-        'Index to use, defaults to "cpan" (when used: also export ES_SCRIPT_INDEX)',
-);
-
 has cluster_info => (
     isa     => HashRef,
     traits  => ['Hash'],
@@ -158,20 +148,6 @@ has queue => (
     documentation => 'add indexing jobs to the minion queue',
 );
 
-sub BUILDARGS {
-    my ( $self, @args ) = @_;
-    my %args = @args == 1 ? %{ $args[0] } : @args;
-
-    if ( exists $args{index} ) {
-        die
-            "when setting --index, please export ES_SCRIPT_INDEX to the same value\n"
-            unless $ENV{ES_SCRIPT_INDEX}
-            and $args{index} eq $ENV{ES_SCRIPT_INDEX};
-    }
-
-    return \%args;
-}
-
 sub handle_error {
     my ( $self, $error, $die_always ) = @_;
 
@@ -194,7 +170,7 @@ sub print_error {
 
 sub index {
     my $self = shift;
-    return $self->model->index( $self->_index );
+    return $self->model->index('cpan');
 }
 
 sub _build_model {

diff --git a/lib/MetaCPAN/Script/Backup.pm b/lib/MetaCPAN/Script/Backup.pm
@@ -8,7 +8,7 @@ use Cpanel::JSON::XS          qw( decode_json encode_json );
 use DateTime                  ();
 use IO::Zlib                  ();
 use Log::Contextual           qw( :log :dlog );
-use MetaCPAN::Types::TypeTiny qw( Bool Int Path Str );
+use MetaCPAN::Types::TypeTiny qw( Bool Int Path Str CommaSepOption );
 use Moose;
 use Try::Tiny qw( catch try );
 
@@ -22,6 +22,15 @@ has batch_size => (
         'Number of documents to restore in one batch, defaults to 100',
 );
 
+has index => (
+    reader        => '_index',
+    is            => 'ro',
+    isa           => CommaSepOption,
+    coerce        => 1,
+    default       => 'cpan',
+    documentation => 'ES indexes to backup, defaults to "cpan"',
+);
+
 has type => (
     is            => 'ro',
     isa           => Str,
@@ -61,34 +70,38 @@ sub run {
     return $self->run_restore if $self->restore;
 
     my $es = $self->es;
-    $self->index->refresh;
-
-    my $filename = join( '-',
-        DateTime->now->strftime('%F'),
-        grep {defined} $self->index->name,
-        $self->type );
-
-    my $file = $self->home->child( qw(var backup), "$filename.json.gz" );
-    $file->parent->mkpath unless ( -e $file->parent );
-    my $fh = IO::Zlib->new( "$file", 'wb4' );
-
-    my $scroll = $es->scroll_helper(
-        index => $self->index->name,
-        $self->type ? ( type => $self->type ) : (),
-        size   => $self->size,
-        fields => [qw(_parent _source)],
-        scroll => '1m',
-        body   => {
-            sort => '_doc',
-        },
-    );
 
-    log_info { 'Backing up ', $scroll->total, ' documents' };
+    for my $index ( @{ $self->_index } ) {
+
+        $self->es->indices->refresh( index => $index );
 
-    while ( my $result = $scroll->next ) {
-        print $fh encode_json($result), $/;
+        my $filename = join( '-',
+            DateTime->now->strftime('%F'),
+            grep {defined} $index,
+            $self->type );
+
+        my $file = $self->home->child( qw(var backup), "$filename.json.gz" );
+        $file->parent->mkpath unless ( -e $file->parent );
+        my $fh = IO::Zlib->new( "$file", 'wb4' );
+
+        my $scroll = $es->scroll_helper(
+            index => $index,
+            $self->type ? ( type => $self->type ) : (),
+            size   => $self->size,
+            fields => [qw(_parent _source)],
+            scroll => '1m',
+            body   => {
+                sort => '_doc',
+            },
+        );
+
+        log_info { 'Backing up ', $scroll->total, ' documents' };
+
+        while ( my $result = $scroll->next ) {
+            print $fh encode_json($result), $/;
+        }
+        close $fh;
     }
-    close $fh;
     log_info {'done'};
 }
 

diff --git a/lib/MetaCPAN/Types/TypeTiny.pm b/lib/MetaCPAN/Types/TypeTiny.pm
@@ -18,6 +18,8 @@ use Type::Library -base, -declare => ( qw(
 
     Logger
     HashRefCPANMeta
+
+    CommaSepOption
 ) );
 use Type::Utils qw( as coerce declare extends from via );
 
@@ -126,4 +128,9 @@ if ( eval { require MooseX::Getopt; 1 } ) {
     }
 }
 
+declare CommaSepOption, as ArrayRef [ StrMatch [qr{^[^, ]+$}] ];
+coerce CommaSepOption, from ArrayRef [Str], via {
+    return [ map split(/\s*,\s*/), @$_ ];
+};
+
 1;