1 files changed, 193 insertions, 0 deletions
diff --git a/scripts/http-useragent-tester.nse b/scripts/http-useragent-tester.nse
new file mode 100644
index 0000000..d6e0aca
--- /dev/null
+++ b/scripts/http-useragent-tester.nse
@@ -0,0 +1,193 @@
+description = [[
+Checks if various crawling utilities are allowed by the host.
+]]
+
+---
+-- @usage nmap -p80 --script http-useragent-tester.nse <host>
+--
+-- This script sets various User-Agent headers that are used by different
+-- utilities and crawling libraries (for example CURL or wget). If the request is
+-- redirected to a page different than a (valid) browser request would be, that
+-- means that this utility is banned.
+--
+-- @args http-useragent-tester.useragents A table with more User-Agent headers.
+--       Default: nil
+--
+-- @output
+-- PORT   STATE SERVICE REASON
+-- 80/tcp open  http    syn-ack
+-- | http-useragent-tester:
+-- |   Status for browser useragent: 200
+-- |   Redirected To: https://www.example.com/
+-- |   Allowed User Agents:
+-- |     Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)
+-- |     libwww
+-- |     lwp-trivial
+-- |     libcurl-agent/1.0
+-- |     PHP/
+-- |     GT::WWW
+-- |     Snoopy
+-- |     MFC_Tear_Sample
+-- |     HTTP::Lite
+-- |     PHPCrawl
+-- |     URI::Fetch
+-- |     Zend_Http_Client
+-- |     http client
+-- |     PECL::HTTP
+-- |     WWW-Mechanize/1.34
+-- |   Change in Status Code:
+-- |     Python-urllib/2.5: 403
+-- |_    Wget/1.13.4 (linux-gnu): 403
+--
+-- @xmloutput
+-- <elem key="Status for browser useragent">200</elem>
+-- <elem key="Redirected To">https://www.example.com/</elem>
+-- <table key="Allowed User Agents">
+--   <elem>Mozilla/5.0 (compatible; Nmap Scripting Engine;
+--   https://nmap.org/book/nse.html)</elem>
+--   <elem>libwww</elem>
+--   <elem>lwp-trivial</elem>
+--   <elem>libcurl-agent/1.0</elem>
+--   <elem>PHP/</elem>
+--   <elem>GT::WWW</elem>
+--   <elem>Snoopy</elem>
+--   <elem>MFC_Tear_Sample</elem>
+--   <elem>HTTP::Lite</elem>
+--   <elem>PHPCrawl</elem>
+--   <elem>URI::Fetch</elem>
+--   <elem>Zend_Http_Client</elem>
+--   <elem>http client</elem>
+--   <elem>PECL::HTTP</elem>
+--   <elem>WWW-Mechanize/1.34</elem>
+-- </table>
+-- <table key="Change in Status Code">
+--   <elem key="Python-urllib/2.5">403</elem>
+--   <elem key="Wget/1.13.4 (linux-gnu)">403</elem>
+-- </table>
+---
+
+categories = {"discovery", "safe"}
+author = "George Chatzisofroniou"
+license = "Same as Nmap--See https://nmap.org/book/man-legal.html"
+
+local http = require "http"
+local target = require "target"
+local httpspider = require "httpspider"
+local shortport = require "shortport"
+local stdnse = require "stdnse"
+local table = require "table"
+local url = require "url"
+
+getLastLoc = function(host, port, useragent)
+
+  local options
+
+  options = {header={}, no_cache=true, bypass_cache=true, redirect_ok=function(host,port)
+      local c = 3
+      return function(url)
+        if ( c==0 ) then return false end
+        c = c - 1
+        return true
+      end
+  end }
+
+
+  options['header']['User-Agent'] = useragent
+
+  stdnse.debug2("Making a request with User-Agent: " .. useragent)
+
+  local response = http.get(host, port, '/', options)
+  if response.location then
+    return response.location[#response.location],response.status or false, response.status
+  end
+
+  return false, response.status
+
+end
+
+portrule = shortport.port_or_service( {80, 443}, {"http", "https"}, "tcp", "open")
+
+action = function(host, port)
+
+  local moreagents = stdnse.get_script_args("http-useragent-tester.useragents") or nil
+  local newtargets = stdnse.get_script_args("newtargets") or nil
+  local output = stdnse.output_table()
+
+  -- We don't crawl any site. We initialize a crawler to use its iswithinhost method.
+  local crawler = httpspider.Crawler:new(host, port, '/', { scriptname = SCRIPT_NAME } )
+
+  local HTTPlibs = {
+    http.USER_AGENT,
+    "libwww",
+    "lwp-trivial",
+    "libcurl-agent/1.0",
+    "PHP/",
+    "Python-urllib/2.5",
+    "GT::WWW",
+    "Snoopy",
+    "MFC_Tear_Sample",
+    "HTTP::Lite",
+    "PHPCrawl",
+    "URI::Fetch",
+    "Zend_Http_Client",
+    "http client",
+    "PECL::HTTP",
+    "Wget/1.13.4 (linux-gnu)",
+    "WWW-Mechanize/1.34"
+  }
+
+  if moreagents then
+    for _, l in ipairs(moreagents) do
+      table.insert(HTTPlibs, l)
+    end
+  end
+
+  -- We perform a normal browser request and get the returned location
+  local loc, status = getLastLoc(host, port, "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.57 Safari/537.17")
+  output['Status for browser useragent'] = status
+
+  if loc then
+    output['Redirected To'] = loc
+  end
+
+  local allowed, forb, status_changed = {}, {}, {}
+
+  for _, l in ipairs(HTTPlibs) do
+
+    local libloc, libstatus = getLastLoc(host, port, l)
+
+    -- If the library's request returned a different location, that means the request was redirected somewhere else, hence is forbidden.
+    if libloc and loc ~= libloc then
+      forb[l] = {}
+      local libhost = url.parse(libloc)
+      if not crawler:iswithinhost(libhost.host) then
+        forb[l]['Different Host'] = tostring(libloc)
+        if newtargets then
+          target.add(libhost.host)
+        end
+      else
+        forb[l]['Same Host'] = tostring(libloc)
+      end
+    elseif status ~= libstatus then
+      status_changed[l] = libstatus
+    else
+      table.insert(allowed, l)
+    end
+
+  end
+
+  if next(allowed) ~= nil then
+    output['Allowed User Agents'] = allowed
+  end
+
+  if next(forb) ~= nil then
+    output['Forbidden/Redirected User Agents'] = forb
+  end
+
+  if next(status_changed) ~= nil then
+    output['Change in Status Code'] = status_changed
+  end
+
+  return output
+
+end