file_tools.py

"""common data management tasks"""
import shelve
import utils
import copy


def find_duplicates(tree_shelvename):
    mtree_shelve = shelve.open(mtree_shelvename, 'r')
    tree = mtree_shelve['tree']
    leaves = mtree_shelve['leaves']
    mtree_shelve.close()

    parent_tree = utils.make_parent_tree(tree)
    volhash = utils.make_hash_index(parent_tree, leaves)

    for key, filelist in volhash.iteritems():
        print key + "-" * 48
        for filename in filelist:
            print "%s" % filename


# TODO: output one file with duplicates, one with uniques
# TODO: output script to move uniques
def find_cross_duplicates(mtree_shelvename, cmtree_shelvename,
                          write_rm_list=None):
    """locate all of the checksums on one volume in a comparison volume
    write_rm_list optionally prints rm commands to delete anything that exists
    somewhere on the comparison volume. rm_list is only files, but to prune
    emtpy directories, issue:
    find <parent-dir> -depth -type d -empty -exec rmdir -v {} \;
    """
    mtree_shelve = shelve.open(mtree_shelvename, 'r')
    tree = mtree_shelve['tree']
    leaves = mtree_shelve['leaves']
    mtree_shelve.close()

    parent_tree = utils.make_parent_tree(tree)
    volhash = utils.make_hash_index(parent_tree, leaves)

    cmtree_shelve = shelve.open(cmtree_shelvename, 'r')
    ctree = cmtree_shelve['tree']
    cleaves = cmtree_shelve['leaves']
    cmtree_shelve.close()

    parent_ctree = utils.make_parent_tree(ctree)
    cvolhash = utils.make_hash_index(parent_ctree, cleaves)

    if write_rm_list:
        rmlistfile = open(write_rm_list, 'w')

    for key, filelist in volhash.iteritems():
        if key in cvolhash:
            cfilelist = cvolhash[key]
            print key + "-" * 48

            for filename in filelist:
                print "%s" % filename

            print "has duplicate file(s) in the comparison volume: "
            for filename in cfilelist:
                print "%s" % filename

            if write_rm_list:
                for filename in filelist:
                    rmlistfile.write("rm -fv %s\n" % filename)

    if write_rm_list:
        rmlistfile.close()


# TODO: break this into smaller component functions
# TODO: make more efficient
# TODO: make exclude list
def find_largest_common_directories(mtree_shelvename,
                                    print_size_only=False,
                                    exclude_list=[]):
    """find the largest directories that share the same checksum of all data
    under them
    """
    mtree_shelve = shelve.open(mtree_shelvename, 'r')
    tree = mtree_shelve['tree']
    leaves = mtree_shelve['leaves']
    mtree_shelve.close()

    parent_tree = utils.make_parent_tree(tree)

    md5dict = utils.make_hash_index(parent_tree, leaves,
                                    entry_type="dir")

    md5_size_dict = {}

    for md5_key in md5dict.keys():
        file_size = []
        for entry in md5dict[md5_key]:
            file_size.append(entry["tree_size"])

        file_size = list(set(file_size))
        if (len(file_size) > 1):
            print "ERROR: accounting for size of directories " +\
                  "with with hash %s failed" % md5_key

        file_size = file_size[0]
        md5_size_dict[md5_key] = file_size

    cutcount = 0

    reduced_md5_size_dict = copy.deepcopy(md5_size_dict)
    for md5_key in md5_size_dict:
        if (len(md5dict[md5_key]) > 1):
            md5cutlist = []
            for entry in md5dict[md5_key]:
                md5under_path = utils.hashes_under_tree(tree, leaves,
                                                        entry["leaf_number"])
                md5cutlist.append(set(md5under_path))

            combined_cutlist = md5cutlist[0]
            for md5list in md5cutlist:
                combined_cutlist.intersection(md5list)

            combined_cutlist = list(combined_cutlist)

            for cutmd5 in combined_cutlist:
                if cutmd5 in reduced_md5_size_dict:
                    cutcount += 1
                    del reduced_md5_size_dict[cutmd5]

    print "number of trees under a duplicate tree cut: %d" % (cutcount)
    total_duplicated_size = 0
    for key, value in sorted(reduced_md5_size_dict.iteritems(),
                             key=lambda (k, v): (v, k),
                             reverse=True):
        if (len(md5dict[key]) > 1):
            total_duplicated_size += (len(md5dict[key]) - 1) * value

            if print_size_only:
                print value
            else:
                print "-" * 80
                print "%s: %d" % (key, value)
                for entry in md5dict[key]:
                    full_pathname = utils.reconstruct_pathname(parent_tree,
                                                               leaves,
                                                 int(entry["leaf_number"]))
                    if not any(excluded in full_pathname
                               for excluded in exclude_list):
                        print full_pathname
                    #else:
                    #    print "#: " + full_pathname

    print "data volume in duplicated directories %d" % total_duplicated_size


# TODO: command-line utility
if __name__ == '__main__':
    #find_duplicates("mtree.shelve")

    #find_largest_common_directories("mtree.shelve",
    #                                print_size_only=False,
    #                                exclude_list=["iPhoto", "Documents"])
    find_cross_duplicates("mtree_maxtor_unix.shelve.db", "mtree_side.shelve.db",
                          write_rm_list="cleanunix_fromside.bash")

    find_cross_duplicates("mtree_maxtor_unix.shelve.db", "mtree_mac.shelve.db",
                          write_rm_list="cleanunix_frommac.bash")

    find_cross_duplicates("mtree_maxtor_mac.shelve.db", "mtree_side.shelve.db",
                          write_rm_list="cleanmac_fromside.bash")

    find_cross_duplicates("mtree_maxtor_mac.shelve.db", "mtree_mac.shelve.db",
                          write_rm_list="cleanmac_frommac.bash")