check-file-miscoded.pl

#!/usr/bin/env perl

use v5.22;
use strict;
use warnings;

# Turn on "file slurp" mode
undef $/;

# Double diamond automatically reads from @ARGV in order, but without involving
# the shell in interpreting file names.
while(<<>>) {
    # These specific characters are present throughout the source NAVADMIN text
    # and seem to be invariably in the Windows Code Point 1252 character set
    if (/[\x91-\x94\x96-\x97\xBC-\xBE]/) {
        say "$ARGV: cp1252";

        # From some manual work there never seems to be duplicate issues with
        # the source file so we bail immediately once we find one.
        next;
    }

    # 3 files in the source data contain these specific 3 bytes at the
    # beginning, which are the UTF-8 Byte Order Mark (BOM) which are improperly
    # generated by some Windows tools (BOM is only need on multi-byte character
    # sets but UTF-8 uses single bytes per character).
    if (/^\xEF\xBB\xBF/) {
        say "$ARGV: utf8-bom";
        next
    }

    if (/[\x80-\x90\x95\x98-\xBB\xBF-\xFF]/) {
        # This would be unexpected based on files we've seen so crash the
        # script so we take a look
        die "$ARGV: unknown-8bit sequence detected";
    }
}