From b2606f09c537f5e6dc82ab1d7d0ce86d782fa67a Mon Sep 17 00:00:00 2001 From: dormando Date: Wed, 5 Jan 2011 13:56:25 -0800 Subject: [PATCH] documentation updates. for all new commands, mogstats, and MogileFS::Utils notes. --- lib/MogileFS/Utils.pm | 22 ++++++-- mogadm | 14 ----- mogdelete | 40 ++++++++++++++ mogfetch | 48 +++++++++++++++++ mogfileinfo | 40 ++++++++++++++ moglistfids | 46 +++++++++++++++- moglistkeys | 48 +++++++++++++++++ mogstats | 120 ++++++++++++++++++++++++++++++++++++++++++ mogtool | 5 +- mogupload | 54 +++++++++++++++++++ 10 files changed, 418 insertions(+), 19 deletions(-) diff --git a/lib/MogileFS/Utils.pm b/lib/MogileFS/Utils.pm index 44ba134..3ce12c8 100644 --- a/lib/MogileFS/Utils.pm +++ b/lib/MogileFS/Utils.pm @@ -31,7 +31,7 @@ sub _readconf { my $args = shift; # Liftedish from mogadm, but we can refactor mogadm to use this instead. - my @configs = ($args->{config}, $ENV{MOGUTILSCONF}, + my @configs = ($args->{conf}, $ENV{MOGUTILSCONF}, "$ENV{HOME}/.mogilefs.conf", "/etc/mogilefs/mogilefs.conf"); my %opts = (); @@ -62,7 +62,7 @@ sub getopts { my %opts = (); $self->abort_usage($usage) unless @ARGV; - GetOptions(\%opts, @want, qw/help trackers=s domain=s/) + GetOptions(\%opts, @want, qw/help trackers=s domain=s conf=s/) or $self->abort_usage($usage); my $config = $self->_readconf(\%opts); @@ -120,7 +120,7 @@ MogileFS::Utils - Command line utilities for the MogileFS distributed file syste L -L +L L @@ -134,14 +134,30 @@ L L +L (DEPRECATED: Do not use!) + =head1 SUMMARY Please refer to the documentation for the tools included in this distribution. +=head1 CONFIGURATION FILE + +Most of the utilities in this package support a configuration file. Common +options can be pushed into the config file, such as trackers, domain, or +class. The file is in B and B<~/.mogilefs.conf> +by default. You may also specify a configuration via B<--conf=filename> + +Example: + + trackers = 10.0.0.1:7001,10.0.0.3:7001 + domain = foo + =head1 AUTHOR Brad Fitzpatrick ELE +Dormando ELE + =head1 BUGS Please report any on the MogileFS mailing list: L. diff --git a/mogadm b/mogadm index e2edced..8bf4352 100755 --- a/mogadm +++ b/mogadm @@ -1437,16 +1437,6 @@ Check to ensure that all of the MogileFS system components are functioning and that we can contact everybody. The quickest way of ensuring that the entire MogileFS system is functional from the current machine's point of view. -=item B - -Attempt to get statistics for the MogileFS world. This should provide -statistics on: the number of files present on each device; the number of files -in each (domain,class) tuple; statistics about present replication devcounts -and pending replications. - -Due to design constraints, the stats command may presently fail if some -statistic takes too long to generate. - =item BhostE [host options]> =item BhostE [host options]> @@ -1662,10 +1652,6 @@ Check the status of your entire MogileFS system: $ mogadm check -Check MogileFS system statistics: - - $ mogadm stats - Check every file in the entire MogileFS system: $ mogadm fsck reset diff --git a/mogdelete b/mogdelete index 466ad44..8bdd898 100755 --- a/mogdelete +++ b/mogdelete @@ -1,5 +1,45 @@ #!/usr/bin/perl +=head1 NAME + +mogdelete -- Delete keys from a MogileFS installation + +=head1 SYNOPSIS + + $ mogdelete --trackers=host --domain=foo --key="/hello.jpg" + +=head1 OPTIONS + +=over + +=item --trackers=host1:7001,host2:7001 + +Use these MogileFS trackers to negotiate with. + +=item --domain= + +Set the MogileFS domain to use. + +=item --key="" + +A key to delete. Can be an arbitrary string. + +=back + +=head1 AUTHOR + +Dormando ELE + +=head1 BUGS + +Produces a slightly weird error if key does not exist. + +=head1 LICENSE + +Licensed for use and redistribution under the same terms as Perl itself. + +=cut + use strict; use warnings; diff --git a/mogfetch b/mogfetch index b8dfbab..e3cb006 100755 --- a/mogfetch +++ b/mogfetch @@ -1,5 +1,53 @@ #!/usr/bin/perl +=head1 NAME + +mogfetch -- Fetch data from a MogileFS installation + +=head1 SYNOPSIS + + $ mogfetch [options] + $ mogfetch [options] --file="-" > filename + + $ mogfetch --trackers=host --domain=foo \ + --key="/hello.jpg" --file="output.jpg" + +=head1 OPTIONS + +=over + +=item --trackers=host1:7001,host2:7001 + +Use these MogileFS trackers to negotiate with. + +=item --domain= + +Set the MogileFS domain to use. + +=item --key="" + +The key to locate the data with. Can be an arbitrary string. + +=item --file="" + +A local destination file. If '-', data is written to STDOUT instead. + +=back + +=head1 AUTHOR + +Dormando ELE + +=head1 BUGS + +None known. + +=head1 LICENSE + +Licensed for use and redistribution under the same terms as Perl itself. + +=cut + use strict; use warnings; diff --git a/mogfileinfo b/mogfileinfo index f7d9f4a..2fd5961 100755 --- a/mogfileinfo +++ b/mogfileinfo @@ -1,5 +1,45 @@ #!/usr/bin/perl +=head1 NAME + +mogfileinfo -- Fetch key metadata from a MogileFS installation + +=head1 SYNOPSIS + + $ mogfileinfo --trackers=host --domain=foo --key="/hello.jpg" + +=head1 OPTIONS + +=over + +=item --trackers=host1:7001,host2:7001 + +Use these MogileFS trackers to negotiate with. + +=item --domain= + +Set the MogileFS domain to use. + +=item --key="" + +The key to inspect. Can be an arbitrary string. + +=back + +=head1 AUTHOR + +Dormando ELE + +=head1 BUGS + +None known, but output might change in the future. + +=head1 LICENSE + +Licensed for use and redistribution under the same terms as Perl itself. + +=cut + use strict; use warnings; diff --git a/moglistfids b/moglistfids index 674893f..b11f2d1 100755 --- a/moglistfids +++ b/moglistfids @@ -1,5 +1,49 @@ #!/usr/bin/perl -# Example for building into a backup program. + +=head1 NAME + +moglistfids -- Iterate fid/key data from a MogileFS installation + +=head1 DESCRIPTION + +Example utility for pulling all file data out of a MogileFS installation. +Utilities like this can be built on for creating backup systems, +Mogile<->Mogile syncronization systems, or Mogile->S3 syncronization. + +This method is only a way of pulling new files which have existed since the +last time it was checked, as there's no logging of deleted files. + +=head1 OPTIONS + +=over + +=item --trackers=host1:7001,host2:7001 + +Use these MogileFS trackers to negotiate with. + +=item --fromfid= + +The highest numbered fid fetched the last time this utility was run. + +=item --count= + +Numer of fids to inspect and return. + +=back + +=head1 AUTHOR + +Dormando ELE + +=head1 BUGS + +None known. + +=head1 LICENSE + +Licensed for use and redistribution under the same terms as Perl itself. + +=cut use strict; use warnings; diff --git a/moglistkeys b/moglistkeys index fa8a5d4..6c6242b 100755 --- a/moglistkeys +++ b/moglistkeys @@ -1,4 +1,52 @@ #!/usr/bin/perl + +=head1 NAMe + +moglistkeys -- Lists keys out of a MogileFS domain + +=head1 SYNOPSIS + + $ moglistkeys --trackers=host --domain=foo --key_prefix="/foo/bar/" + +=head1 DESCRIPTION + +If you store your MogileFS keys in a logical "structure", you may use this +tool to view lists of subsets of keys. Note that this is not going to be +equivalent to "cd" and "ls" tools, as listing "foo/" will list everything +underneath, so it's more akin to "ls -R" + +=head1 OPTIONS + +=over + +=item --trackers=host1:7001,host2:7001 + +Use these MogileFS trackers to negotiate with. + +=item --domain= + +Set the MogileFS domain to use. + +=item --key_prefix="/foo/bar/" + +Search for keys starting with this prefix. Can be an arbitrary string. + +=back + +=head1 AUTHOR + +Dormando ELE + +=head1 BUGS + +None known. + +=head1 LICENSE + +Licensed for use and redistribution under the same terms as Perl itself. + +=cut + # TODO: Add ways to limit # of keys displayed use strict; diff --git a/mogstats b/mogstats index fbc6542..2566fc8 100755 --- a/mogstats +++ b/mogstats @@ -5,6 +5,125 @@ # timing output during fetch # support different db backends better (something mildly generic). +=head1 NAME + +mogstats -- Utility for calculating slow stats directly against a MogileFS DB + +=head1 SYNOPSIS + + $ mogstats --db_dsn="DBI:mysql:mfs:host=mfshost" --db_user="mfs" \ + --db_pass="mfs" --verbose --stats="devices,files" + $ mogstats --stats="all" + $ mogstats [all options in ~/.mogilefs.conf] + +=head1 DESCRIPTION + +Utility for inspecting queues and running general statistics against a +MogileFS database. Some of these stats can take a very long time to run +against a large instance, so the utility can be pointed at a read slave or +special account. + +=head1 OPTIONS + +=over + +=item --db_dsn= + +The DSN to use for connecting to the MogileFS database server. + +=item --db_user= + +A database user for connecting to the database. + +=item --db_pass= + +An optional password for the database user. + +=item --config= + +An explicit config file to use. By default /etc/mogilefs/mogilefs.conf and +~/.mogilefs.conf are checked. + +=item --verbose + +Print some extra text during processing. Mostly notes about what stats are +starting or finishing. + +=item --stats= + +A list of which statistics to calculate. Notes on some of them are listed +below, see --help for full list. A value of "all" fetches all possible stats. + +=item --help + +List usage info and supported statistics. + +=back + +=head1 AVAILABLE STATISTICS + +Contains notes on which stats may be fast or slow. + +=over + +=item devices + +Lists count of files and current database status per-device. Can be very slow. + +=item fids + +Lists the current highest file id. Should be fast. + +=item files + +Gives a breakdown of where files are by domain and class. Displays the size of +all stored files pre-replication, as well as post-replication size. The latter +being closer to the actual storage amount. Can be very slow. + +=item domains + +Shows a simple count of where files are by domain and class. Faster than using +"files" but displays less information. + +=item replication + +Displays a breakdown of devcount per domain/class combo. Shows number of files +in domain "foo" with class "bar" that have a current devcount of 0, 1, 2, 3, +etc. Useful for spotting broken files (devcount 0), replication lag, or over +replication bugs. Can be very slow. + +=item replication-queue + +Quick breakdown of how many fids are due for replication. Fids listed as +"manual" need manual intervention before they can be replicated, and could be +broken. Will be fast unless there are many files in queue. + +=item delete-queue + +Similar to replication-queue. + +=item general-queue + +Displays a breakdown of what's in the general queue. This includes FSCK, +Rebalance, and other temporary queueing systems MogileFS has. Should be fast +unless you have configured MogileFS to queue many fids at once. + +=back + +=head1 AUTHOR + +Dormando ELE + +=head1 BUGS + +None known. + +=head1 LICENSE + +Licensed for use and redistribution under the same terms as Perl itself. + +=cut + use strict; use warnings; use DBI; @@ -20,6 +139,7 @@ my $DB_TYPE = ''; my %opts; +# FIXME: Use MogileFS::Utils for configuration junk. Getopt::Long::Configure("require_order", "pass_through"); GetOptions( "config=s" => \$opts{config}, diff --git a/mogtool b/mogtool index f6936d8..0b6c333 100755 --- a/mogtool +++ b/mogtool @@ -8,6 +8,8 @@ eval 'exec /usr/bin/perl -S $0 ${1+"$@"}' mogtool -- Inject/extract data to/from a MogileFS installation +B: this utility is deprecated! See L + =head1 SYNOPSIS $ mogtool [general-opts] [command-opts] @@ -1417,7 +1419,8 @@ sub mdelete { abortWithUsage() if $opts{help}; sub abortWithUsage { - my $msg = join '', @_; + my $msg = "!!!mogtool is DEPRECATED and will be removed in the future!!!\n"; + $msg .= join '', @_; if ( $msg ) { pod2usage( -verbose => 1, -exitval => 1, -message => "\n$msg\n" ); diff --git a/mogupload b/mogupload index 15bc8fb..85015b4 100755 --- a/mogupload +++ b/mogupload @@ -1,5 +1,59 @@ #!/usr/bin/perl +=head1 NAME + +mogupload -- Upload data to a MogileFS installation + +=head1 SYNOPSIS + + $ mogupload [options] + $ mogupload [options] --file="-" < filename + + $ mogupload --trackers=host --domain=foo --class=bar \ + --key="/hello.jpg" --file="input.jpg" + + $ echo "why hello" | mogupload [opts] --key="world" --file="-" + +=head1 OPTIONS + +=over + +=item --trackers=host1:7001,host2:7001 + +Use these MogileFS trackers to negotiate with. + +=item --domain= + +Set the MogileFS domain to use. + +=item --class= + +Set the class to use. Will use default class if not specified + +=item --key="" + +A key to store the file under. Can be an arbitrary string. + +=item --file="" + +A local file to upload. If '-', read file from STDIN instead. + +=back + +=head1 AUTHOR + +Dormando ELE + +=head1 BUGS + +mogupload must buffer the upload in memory before transferring it. This makes it difficult to upload very large files. Future versions will lift this limitation. + +=head1 LICENSE + +Licensed for use and redistribution under the same terms as Perl itself. + +=cut + use strict; use warnings;