From cd3b232e4a6416b8bb41d52d8acc40d3d4540ac1 Mon Sep 17 00:00:00 2001 From: Postmodern Date: Wed, 10 Apr 2024 19:57:18 -0700 Subject: [PATCH] Added the `ronin-masscan grep` command (closes #6). --- README.md | 1 + gemspec.yml | 1 + lib/ronin/masscan/cli/commands/grep.rb | 230 +++++++++++++++++++++++ man/ronin-masscan-grep.1.md | 49 +++++ spec/cli/commands/grep_spec.rb | 248 +++++++++++++++++++++++++ 5 files changed, 529 insertions(+) create mode 100644 lib/ronin/masscan/cli/commands/grep.rb create mode 100644 man/ronin-masscan-grep.1.md create mode 100644 spec/cli/commands/grep_spec.rb diff --git a/README.md b/README.md index fe5a28e..a74cdbe 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ Commands: completion convert dump + grep help import print diff --git a/gemspec.yml b/gemspec.yml index bae0340..74e8eb0 100644 --- a/gemspec.yml +++ b/gemspec.yml @@ -24,6 +24,7 @@ generated_files: - man/ronin-masscan-completion.1 - man/ronin-masscan-convert.1 - man/ronin-masscan-dump.1 + - man/ronin-masscan-grep.1 - man/ronin-masscan-import.1 - man/ronin-masscan-print.1 - man/ronin-masscan-scan.1 diff --git a/lib/ronin/masscan/cli/commands/grep.rb b/lib/ronin/masscan/cli/commands/grep.rb new file mode 100644 index 0000000..5bb6a76 --- /dev/null +++ b/lib/ronin/masscan/cli/commands/grep.rb @@ -0,0 +1,230 @@ +# frozen_string_literal: true +# +# ronin-masscan - A Ruby library and CLI for working with masscan. +# +# Copyright (c) 2023-2024 Hal Brodigan (postmodern.mod3@gmail.com) +# +# ronin-masscan is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ronin-masscan is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with ronin-masscan. If not, see . +# + +require 'ronin/masscan/cli/command' +require 'ronin/masscan/cli/filtering_options' + +require 'command_kit/colors' +require 'command_kit/printing/indent' +require 'masscan/output_file' + +module Ronin + module Masscan + class CLI + module Commands + # + # Greps the scanned services from masscan scan file(s) for the given + # pattern. + # + # ## Usage + # + # ronin-masscan grep [options] PATTERN MASSCAN_FILE [...] + # + # ## Options + # + # -P, --protocol tcp|udp Filters the targets by protocol + # --ip IP Filters the targets by IP + # --ip-range CIDR Filters the targets by IP range + # -p, --ports {PORT | PORT1-PORT2},... + # Filters targets by port number + # --with-app-protocol APP_PROTOCOL[,...] + # Filters targets with the app protocol + # --with-payload STRING Filters targets containing the payload + # --with-payload-regex /REGEX/ Filters targets with the matching payload + # -h, --help Print help information + # + # ## Arguments + # + # PATTERN The pattern to search for + # MASSCAN_FILE ... The masscan scan file(s) to parse + # + class Grep < Command + + usage '[options] PATTERN MASSCAN_FILE [...]' + + include CommandKit::Colors + include CommandKit::Printing::Indent + include FilteringOptions + + argument :pattern, required: true, + desc: 'The pattern to search for' + + argument :masscan_file, required: true, + repeats: true, + desc: 'The masscan scan file(s) to parse' + + description 'Greps the scanned services from masscan scan file(s)' + + man_page 'ronin-masscan-grep.1' + + # + # Runs the `ronin-masscan grep` command. + # + # @param [String] pattern + # The pattern to search for. + # + # @param [Array] masscan_files + # The nmap `.xml` files to parse. + # + def run(pattern,*masscan_files) + masscan_files.each do |masscan_file| + output_file = begin + ::Masscan::OutputFile.new(masscan_file) + rescue ArgumentError => error + print_error(error.message) + exit(1) + end + + records = grep_records(output_file,pattern) + + highlight_records(records,pattern) + end + end + + # + # Greps the masscan output file for the pattern. + # + # @param [::Masscan::OutputFile] output_file + # The masscan output file to search. + # + # @param [String] pattern + # The pattern to search for. + # + def grep_records(output_file,pattern) + records = filter_records(output_file) + + records.filter { |record| match_record(record,pattern) } + end + + # + # Determines if the masscan record includes the pattern. + # + # @param [::Masscan::Status, ::Masscan::Banner] record + # The masscan record to search. + # + # @param [String] pattern + # The pattern to search for. + # + # @return [Boolean] + # Indicates whether the masscan record contains the pattern. + # + def match_record(record,pattern) + case record + when ::Masscan::Banner + record.app_protocol.match(pattern) || + record.payload.match(pattern) + end + end + + # + # Prints the open ports for the IP. + # + # @param [Array<::Masscan::Status, ::Masscan::Banner>] records + # The masscan records to print. + # + # @param [String] pattern + # The pattern to highlight. + # + def highlight_records(records,pattern) + records.group_by(&:ip).each do |ip,records_for_ip| + puts "[ #{ip} ]" + puts + + records_for_ip.group_by { |record| + [record.port, record.protocol] + }.each do |(port,protocol),records_for_port| + indent do + puts "#{port}/#{protocol}" + + indent do + records_for_port.each do |record| + highlight_record(record,pattern) + end + end + end + end + + puts + end + end + + # + # Prints the masscan record with the pattern highlighted. + # + # @param [::Masscan:Status, ::Masscan::Banner] record + # The masscan record to print. + # + # @param [String] pattern + # The pattern to highlight. + # + def highlight_record(record,pattern) + case record + when ::Masscan::Banner + highlight_banner_record(record,pattern) + end + end + + # + # Prints the masscan banner record with the pattern highlighted. + # + # @param [::Masscan::Banner] banner + # The masscan banner record to print. + # + # @param [String] pattern + # The pattern to highlight. + # + def highlight_banner_record(banner,pattern) + payload = highlight(banner.payload,pattern) + app_protocol = highlight(banner.app_protocol,pattern) + + if payload.include?("\n") # multiline? + puts app_protocol + + indent do + payload.chomp.each_line(chomp: true) do |line| + puts line + end + end + else + puts "#{app_protocol}\t#{payload}" + end + end + + # + # Highlights the pattern in the text. + # + # @param [String] text + # The text to modify. + # + # @param [String] pattern + # The pattern to highlight. + # + # @return [String] + # The modified text. + # + def highlight(text,pattern) + text.to_s.gsub(pattern,colors.red(pattern)) + end + + end + end + end + end +end diff --git a/man/ronin-masscan-grep.1.md b/man/ronin-masscan-grep.1.md new file mode 100644 index 0000000..bac6877 --- /dev/null +++ b/man/ronin-masscan-grep.1.md @@ -0,0 +1,49 @@ +# ronin-masscan-grep 1 "2023-03-01" Ronin Masscan "User Manuals" + +## NAME + +ronin-masscan-grep - Greps the scanned services from masscan scan file(s) + +## SYNOPSIS + +`ronin-masscan grep` [options] *PATTERN* *MASSCAN_FILE* + +## DESCRIPTION + +## ARGUMENTS + +*MASSCAN_FILE* +: The masscan scan file to import. + +## OPTIONS + +`-P`, `--protocol` `tcp`|`udp` +: Filters the targets by the protocol of the open port. + +`--ip` *IP* +: Filters the targets by a specific IP address. + +`--ip-range` *CIDR* +: Filters the targets by a CIDR IP range (ex: `192.168.1.0/24`). + +`-p`, `--ports` {*PORT* | *PORT1-PORT2*},... +: Filter `IP:PORT` or `HOST:PORT` pairs who's ports are in the gvien port list. + The port list is a comma separated list of port numbers (`443`) or port + ranges (`8000-9000`). + +`--with-app-protocol` *APP_PROTOCOL*,... +: Filters targets by the app protocol names. + +`--with-payload` *STRING* +: Filters targets by the payload substring. + +`--with-payload-regex` `/`*REGEX*`/` +: Filters targets how's payload string matches the regular expression. + +## AUTHOR + +Postmodern + +## SEE ALSO + +[ronin-masscan-print](ronin-masscan-print.1.md) diff --git a/spec/cli/commands/grep_spec.rb b/spec/cli/commands/grep_spec.rb new file mode 100644 index 0000000..5531665 --- /dev/null +++ b/spec/cli/commands/grep_spec.rb @@ -0,0 +1,248 @@ +require 'spec_helper' +require 'ronin/masscan/cli/commands/grep' +require_relative 'man_page_example' + +require 'stringio' + +describe Ronin::Masscan::CLI::Commands::Grep do + include_examples "man_page" + + let(:stdout) { StringIO.new } + + before { allow(stdout).to receive(:tty?).and_return(true) } + subject { described_class.new(stdout: stdout) } + + let(:fixtures_dir) { File.expand_path(File.join(__dir__,'..','..','fixtures')) } + let(:masscan_path) { File.join(fixtures_dir, 'masscan.json') } + let(:output_file) { Masscan::OutputFile.new(masscan_path) } + let(:records) { output_file.each } + + let(:pattern) { 'html' } + let(:highlighted_pattern) { subject.colors.red(pattern) } + + describe "#run" do + it "must print the masscan banner records from the masscan file that contain the pattern" do + subject.run(pattern,masscan_path) + + expect(stdout.string).to eq( + <<~OUTPUT + [ 93.184.216.34 ] + + 80/tcp + #{highlighted_pattern}_title 404 - Not Found + http + HTTP/1.0 404 Not Found + Content-Type: text/#{highlighted_pattern} + Date: Thu, 26 Aug 2021 06:50:24 GMT + Server: ECS (sec/974D) + Content-Length: 345 + Connection: close + + OUTPUT + ) + end + end + + describe "#grep_records" do + it "must return a lazy enumerator with all banner records that contain the pattern in their #app_protocol or #payload fields" do + matching_records = subject.grep_records(output_file,pattern) + + expect(matching_records).to be_kind_of(Enumerator::Lazy) + expect(matching_records.to_a).to eq( + records.filter do |record| + record.kind_of?(Masscan::Banner) && ( + record.app_protocol.match(pattern) || + record.payload.match(pattern) + ) + end + ) + end + end + + describe "#match_record" do + context "when given a Masscan::Status record" do + let(:record) do + records.find { |record| record.kind_of?(Masscan::Status) } + end + + it "must return false" do + expect(subject.match_record(record,pattern)).to be_falsy + end + end + + context "when given a Masscan::Banner record" do + let(:record) do + records.find { |record| record.kind_of?(Masscan::Banner) } + end + + context "and the pattern exists within the #app_protocol field" do + let(:pattern) { 'html' } + let(:record) do + records.find do |record| + record.kind_of?(Masscan::Banner) && + record.app_protocol == :html_title + end + end + + it "must return true" do + expect(subject.match_record(record,pattern)).to be_truthy + end + end + + context "and the pattern exists within the #payload field" do + let(:pattern) { 'ECS' } + let(:record) do + records.find do |record| + record.kind_of?(Masscan::Banner) && + record.app_protocol == :http + end + end + + it "must return true" do + expect(subject.match_record(record,pattern)).to be_truthy + end + end + + context "but the pattern does not exists within the #app_protocol or #payload fields" do + let(:pattern) { 'foo' } + let(:record) do + records.find { |record| record.kind_of?(Masscan::Banner) } + end + + it "must return false" do + expect(subject.match_record(record,pattern)).to be_falsy + end + end + end + end + + describe "#highlight_records" do + let(:matching_records) do + records.filter do |record| + record.kind_of?(Masscan::Banner) && ( + record.app_protocol.match(pattern) || + record.payload.match(pattern) + ) + end + end + + it "must print the records, grouped by IP and port/protocol, with the pattern highlighted" do + subject.highlight_records(matching_records,pattern) + + expect(stdout.string).to eq( + <<~OUTPUT + [ 93.184.216.34 ] + + 80/tcp + #{highlighted_pattern}_title 404 - Not Found + http + HTTP/1.0 404 Not Found + Content-Type: text/#{highlighted_pattern} + Date: Thu, 26 Aug 2021 06:50:24 GMT + Server: ECS (sec/974D) + Content-Length: 345 + Connection: close + + OUTPUT + ) + end + end + + describe "#highlight_record" do + context "when given a Masscan::Banner record" do + let(:record) do + records.find { |record| record.kind_of?(Masscan::Banner) } + end + + context "and the pattern exists within the #app_protocol field" do + let(:pattern) { 'html' } + let(:record) do + records.find do |record| + record.kind_of?(Masscan::Banner) && + record.app_protocol == :html_title + end + end + + it "must print the banner record with the pattern highlighted in the #app_protocol part" do + subject.highlight_record(record,pattern) + + expect(stdout.string).to eq( + "#{highlighted_pattern}_title 404 - Not Found#{$/}" + ) + end + end + + context "and the pattern exists within the #payload field" do + let(:pattern) { 'ECS' } + let(:record) do + records.find do |record| + record.kind_of?(Masscan::Banner) && + record.app_protocol == :http + end + end + + it "must print the banner record with the pattern highlighted in the #payload part" do + subject.highlight_record(record,pattern) + + expect(stdout.string).to eq( + <<~OUTPUT + http + HTTP/1.0 404 Not Found + Content-Type: text/html + Date: Thu, 26 Aug 2021 06:50:24 GMT + Server: #{highlighted_pattern} (sec/974D) + Content-Length: 345 + Connection: close + OUTPUT + ) + end + end + end + end + + describe "#highlight_banner_record" do + context "and the pattern exists within the #app_protocol field" do + let(:pattern) { 'html' } + let(:record) do + records.find do |record| + record.kind_of?(Masscan::Banner) && + record.app_protocol == :html_title + end + end + + it "must print the banner record with the pattern highlighted in the #app_protocol part" do + subject.highlight_banner_record(record,pattern) + + expect(stdout.string).to eq( + "#{highlighted_pattern}_title 404 - Not Found#{$/}" + ) + end + end + + context "and the pattern exists within the #payload field" do + let(:pattern) { 'ECS' } + let(:record) do + records.find do |record| + record.kind_of?(Masscan::Banner) && + record.app_protocol == :http + end + end + + it "must print the banner record with the pattern highlighted in the #payload part" do + subject.highlight_banner_record(record,pattern) + + expect(stdout.string).to eq( + <<~OUTPUT + http + HTTP/1.0 404 Not Found + Content-Type: text/html + Date: Thu, 26 Aug 2021 06:50:24 GMT + Server: #{highlighted_pattern} (sec/974D) + Content-Length: 345 + Connection: close + OUTPUT + ) + end + end + end +end