new commandline utils.

what used to be mogtool, is now split up into multiple unix-y utils.
"bigfile" by default was very confusing to folks testing the utils.

new commands are added and more will be added around troubleshooting

MANIFEST

@@ -6,3 +6,9 @@ lib/MogileFS/Utils.pm
 mogtool
 mogadm
 mogstats
+mogupload
+mogdelete
+mogfetch
+mogfileinfo
+moglistfids
+moglistkeys

Makefile.PL

@@ -8,7 +8,10 @@ WriteMakefile(
     VERSION_FROM                => 'lib/MogileFS/Utils.pm',
     AUTHOR                      => 'Brad Fitzpatrick <[email protected]>',
     ABSTRACT                    => 'MogileFS utilities',
-    EXE_FILES                   => ['mogtool', 'mogadm', 'mogstats'],
+    EXE_FILES                   => ['mogtool', 'mogadm', 'mogstats',
+        'mogupload', 'mogfetch', 'mogdelete', 'mogfileinfo', 'moglistkeys',
+        'moglistfids',
+                                   ],
     PREREQ_PM                   => {
         'LWP::Simple'      => 0,
         'Compress::Zlib'   => 0,

lib/MogileFS/Utils.pm

@@ -3,6 +3,114 @@ package MogileFS::Utils;
 
 our $VERSION = '2.18';
 
+use Getopt::Long;
+use MogileFS::Client;
+
+use fields (
+            'config'
+           );
+
+# Helper object for the individual utilities.
+sub new {
+    my MogileFS::Utils $self = shift;
+    $self = fields::new($self) unless ref $self;
+    $self->_init(@_);
+
+    return $self;
+}
+
+# Predefine some options via configuration.
+sub _init {
+    my MogileFS::Utils $self = shift;
+
+    $self->{config} = {};
+}
+
+sub _readconf {
+    my MogileFS::Utils $self = shift;
+    my $args = shift;
+
+    # Liftedish from mogadm, but we can refactor mogadm to use this instead.
+    my @configs = ($args->{config}, $ENV{MOGUTILSCONF},
+        "$ENV{HOME}/.mogilefs.conf",
+        "/etc/mogilefs/mogilefs.conf");
+    my %opts = ();
+    for my $fn (reverse @configs) {
+        next unless $fn && -e $fn;
+        open my $file, "<$fn"
+            or die "unable to open $fn: $!";
+        while (<$file>) {
+            s/\#.*//;
+            next unless m/^\s*(\w+)\s*=\s*(.+?)\s*$/;
+            $opts{$1} = $2 unless ( defined $opts{$1} );
+        }
+        close $file;
+    }
+
+    return \%opts;
+}
+
+sub config {
+    my MogileFS::Utils $self = shift;
+    return $self->{config};
+}
+
+sub getopts {
+    my MogileFS::Utils $self = shift;
+    my $usage = shift;
+    my @want  = @_;
+
+    my %opts = ();
+    $self->abort_usage($usage) unless @ARGV;
+    GetOptions(\%opts, @want, qw/help trackers=s domain=s/)
+        or $self->abort_usage($usage);
+    my $config = $self->_readconf(\%opts);
+
+    $self->{config} = {%$config, %opts};
+    $self->_verify_config;
+    $self->abort_usage($usage) if $self->{config}->{help};
+
+    return $self->{config};
+}
+
+sub _verify_config {
+    my MogileFS::Utils $self = shift;
+    my $conf = $self->{config};
+
+    while (my ($k, $v) = each %$conf) {
+        if ($k =~ m/^trackers/) {
+            my @tr = split /,/, $v;
+            for (@tr) {
+                # Client is obnoxious about requiring a port.
+                if ($_ !~ m/:\d+/) {
+                    $_ = $_ . ':7001';
+                }
+            }
+            $conf->{$k} = \@tr;
+        } elsif ($k =~ m/class/) {
+            # "" means "default". Might have to remove this if people have
+            # been adding "default" classes, which I don't think is possible?
+            if ($v eq 'default') {
+                $conf->{$k} = '';
+            }
+        }
+    }
+}
+
+# Do we want to be fancier here?
+sub abort_usage {
+    my MogileFS::Utils $self = shift;
+    my $usage = shift;
+    print "Usage: $0 $usage\n";
+    exit;
+}
+
+sub client {
+    my MogileFS::Utils $self = shift;
+    my $c = $self->{config};
+    return MogileFS::Client->new(domain => $c->{domain},
+        hosts => $c->{trackers});
+}
 
 =head1 NAME
 
@@ -14,6 +122,18 @@ L<mogadm>
 
 L<mogtool>
 
+L<mogupload>
+
+L<mogfetch>
+
+L<mogdelete>
+
+L<mogfileinfo>
+
+L<moglistkeys>
+
+L<moglistfids>
+
 =head1 SUMMARY
 
 Please refer to the documentation for the tools included in this distribution.

mogdelete

@@ -0,0 +1,18 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+
+use lib './lib';
+use MogileFS::Utils;
+
+my $util = MogileFS::Utils->new;
+my $usage = "--trackers=host --domain=foo --key='/hello.jpg'";
+my $c = $util->getopts($usage, 'key=s');
+
+my $mogc = $util->client;
+
+$mogc->delete($c->{key});
+if ($mogc->errcode) {
+    print STDERR "Error deleting file: ", $mogc->errstr, "\n";
+}

mogfetch

@@ -0,0 +1,57 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+
+use lib './lib';
+use MogileFS::Utils;
+
+my $util = MogileFS::Utils->new;
+my $usage = "--trackers=host --domain=foo --key='/hello.jpg' --file='./output'";
+my $c = $util->getopts($usage, qw/key=s file=s/);
+
+my $mogc = $util->client;
+
+# Default to noverify, don't hang up the tracker. We'll try all paths.
+my @paths = $mogc->get_paths($c->{key}, { noverify => 1 });
+if ($mogc->errcode) {
+    die "Error fetching paths: " . $mogc->errstr;
+}
+
+die "No paths found or key does not exist" unless @paths;
+
+my $filename = $c->{file};
+my @resses;
+for my $path (@paths) {
+    next unless $path; # overparanoid?
+    my $ua = LWP::UserAgent->new;
+    $ua->timeout(10);
+
+    my $file;
+    if ($filename eq '-') {
+        $file = *STDOUT;
+    } else {
+        open($file, "> $filename") or die "Could not open " . $filename;
+    }
+
+    my $writeout = sub {
+        print $file $_[0];
+    };
+    my $res = $ua->get($path, ':content_cb' => $writeout,
+        ':read_size_hint' => 32768);
+
+    if ($res->is_success) {
+        last;
+    } else {
+        # print all the errors to be the most helpful
+        push(@resses, $res);
+        next;
+    }
+}
+
+if (@resses) {
+    for my $res (@resses) {
+        print STDERR "Got errors while trying to fetch:\n";
+        print STDERR $res->status_line, "\n";
+    }
+}

mogfileinfo

@@ -0,0 +1,35 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+
+use lib './lib';
+use MogileFS::Utils;
+
+my $util = MogileFS::Utils->new;
+my $usage = "--trackers=host --domain=foo --key='/hello.jpg'";
+my $c = $util->getopts($usage, 'key=s');
+
+my $mogc = $util->client;
+
+my $fid = $mogc->file_info($c->{key});
+if ($mogc->errcode) {
+    die "Error fetching file info: " . $mogc->errstr;
+}
+die "Key not found: " . $c->{key} unless $fid;
+
+# Might replace this with just fetching the devids from above...
+my @paths = $mogc->get_paths($c->{key}, { noverify => 1, pathcount => 99 });
+if ($mogc->errcode) {
+    die "Error fetching paths: " . $mogc->errstr;
+}
+die "No paths found or key does not exist" unless @paths;
+
+print "- file: ", $c->{key}, "\n";
+for my $item (sort keys %$fid) {
+    printf("  %8s: %20s\n", $item, $fid->{$item});
+}
+
+for my $path (@paths) {
+    print " - ", $path, "\n";
+}

moglistfids

@@ -0,0 +1,47 @@
+#!/usr/bin/perl
+# Example for building into a backup program.
+
+use strict;
+use warnings;
+
+use MogileFS::Admin;
+use lib './lib';
+use MogileFS::Utils;
+
+my $util = MogileFS::Utils->new;
+my $usage = "--trackers=host --fromfid=123 --count=5000";
+my $c = $util->getopts($usage, qw/fromfid=i count=i/);
+
+my $moga = MogileFS::Admin->new(hosts => $c->{trackers});
+
+my $fromfid = $c->{fromfid} || 0;
+my $count   = $c->{count} || 100;
+
+while ($count) {
+    # Try to fetch the max, but we will likely get less.
+    my $fids_chunk = $moga->list_fids($fromfid, $count);
+    if ($moga->errcode) {
+        die "Error listing fids: ", $moga->errstr, "\n";
+    }
+    my @fids = sort { $a <=> $b } keys %$fids_chunk;
+    last unless @fids;
+    $fromfid = $fids[-1];
+    $count  -= @fids;
+    for my $fid (@fids) {
+        my $file = $fids_chunk->{$fid};
+        print "fid ", $fid, "\n";
+        for my $key (sort keys %$file) {
+            my $val = $file->{$key};
+            $val = _escape_url_string($val) if $key eq 'dkey';
+            print $key, " ", $val, "\n";
+        }
+        print "\n";
+    }
+}
+
+sub _escape_url_string {
+    my $str = shift;
+    $str =~ s/([^a-zA-Z0-9_\,\-.\/\\\: ])/uc sprintf("%%%02x",ord($1))/eg;
+    $str =~ tr/ /+/;
+    return $str;
+}

moglistkeys

@@ -0,0 +1,23 @@
+#!/usr/bin/perl
+# TODO: Add ways to limit # of keys displayed
+
+use strict;
+use warnings;
+
+use lib './lib';
+use MogileFS::Utils;
+
+my $util = MogileFS::Utils->new;
+my $usage = "--trackers=host --domain=foo --key_prefix='bar/'";
+my $c = $util->getopts($usage, 'key_prefix=s');
+
+my $mogc = $util->client;
+
+$mogc->foreach_key(prefix => $c->{key_prefix}, sub {
+    my $key = shift;
+    print $key, "\n";
+});
+
+if ($mogc->errcode) {
+    print STDERR "Error listing files: ", $mogc->errstr, "\n";
+}

mogupload

@@ -0,0 +1,43 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+
+use lib './lib';
+use MogileFS::Utils;
+
+my $util = MogileFS::Utils->new;
+my $usage = "--trackers=host --domain=foo --key='/hello.jpg' --file='./hello.jpg'";
+my $c = $util->getopts($usage, qw/class=s key=s file=s/);
+
+my $mogc = $util->client;
+
+my $filename = $c->{file};
+
+my $fh;
+my $size;
+if ($filename eq '-') {
+    # Can't upload files without a known filesize?
+    die "STDIN mode not yet supported";
+    $fh = *STDIN;
+} else {
+    $size = -s $filename;
+    die "Could not stat " . $filename unless defined $size;
+    open($fh, "< $filename") or die "Could not open " . $filename;
+}
+
+my $mf = $mogc->new_file($c->{key}, $c->{class}, $size);
+if ($mogc->errcode) {
+    die "Error opening MogileFS file: " . $mogc->errstr;
+}
+
+my $buf;
+while (my $read = read($fh, $buf, 32768)) {
+    die "error reading file" unless defined $read;
+    last if $read == 0;
+    print $mf $buf;
+}
+
+unless ($mf->close) {
+    die "Error writing file: " . $mogc->errcode . ": " . $mogc->errstr;
+}