1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
|
local http = require "http"
local nmap = require "nmap"
local shortport = require "shortport"
local strbuf = require "strbuf"
local table = require "table"
description = [[
Checks for disallowed entries in <code>/robots.txt</code> on a web server.
The higher the verbosity or debug level, the more disallowed entries are shown.
]]
---
--@output
-- 80/tcp open http syn-ack
-- | http-robots.txt: 156 disallowed entries (40 shown)
-- | /news?output=xhtml& /search /groups /images /catalogs
-- | /catalogues /news /nwshp /news?btcid=*& /news?btaid=*&
-- | /setnewsprefs? /index.html? /? /addurl/image? /pagead/ /relpage/
-- | /relcontent /sorry/ /imgres /keyword/ /u/ /univ/ /cobrand /custom
-- | /advanced_group_search /googlesite /preferences /setprefs /swr /url /default
-- | /m? /m/? /m/lcb /m/news? /m/setnewsprefs? /m/search? /wml?
-- |_ /wml/? /wml/search?
author = "Eddie Bell"
license = "Same as Nmap--See https://nmap.org/book/man-legal.html"
categories = {"default", "discovery", "safe"}
portrule = shortport.http
local last_len = 0
-- split the output in 50 character length lines
local function buildOutput(output, w)
local nl
if w:len() == 0 then
return nil
end
-- check for duplicates
for i,v in ipairs(output) do
if w == v or w == v:sub(2, v:len()) then
return nil
end
end
-- format lines
if last_len == 0 or last_len + w:len() <= 50 then
last_len = last_len + w:len()
nl = ''
else
last_len = 0
nl = '\n'
end
output = output .. (nl .. w)
end
-- parse all disallowed entries in body and add them to a strbuf
local function parse_robots(body, output)
for line in body:gmatch("[^\r\n]+") do
for w in line:gmatch('[Dd]isallow:%s*(.*)') do
w = w:gsub("%s*#.*", "")
buildOutput(output, w)
end
end
return #output
end
action = function(host, port)
local dis_count, noun
local answer = http.get(host, port, "/robots.txt" )
if answer.status ~= 200 then
return nil
end
local v_level = nmap.verbosity() + (nmap.debugging()*2)
local output = strbuf.new()
local detail = 15
dis_count = parse_robots(answer.body, output)
if dis_count == 0 then
return
end
-- verbose/debug mode, print 50 entries
if v_level > 1 and v_level < 5 then
detail = 40
-- double debug mode, print everything
elseif v_level >= 5 then
detail = dis_count
end
-- check we have enough entries
if detail > dis_count then
detail = dis_count
end
noun = dis_count == 1 and "entry " or "entries "
local shown = (detail == 0 or detail == dis_count)
and "\n" or '(' .. detail .. ' shown)\n'
return dis_count .. " disallowed " .. noun ..
shown .. table.concat(output, ' ', 1, detail)
end
|
