---
-- Library methods for handling MongoDB, creating and parsing packets.
--
-- @author Martin Holst Swende
-- @copyright Same as Nmap--See http://nmap.org/book/man-legal.html
--
-- Version 0.2
--
-- @args mongodb.db - the database to use for authentication
-- Created 01/13/2010 - v0.1 - created by Martin Holst Swende <martin@swende.se>
-- Revised 01/03/2012 - v0.2 - added authentication support <patrik@cqure.net>
local bin = require "bin"
local nmap = require "nmap"
local stdnse = require "stdnse"
local string = require "string"
local table = require "table"
local openssl = stdnse.silent_require "openssl"
_ENV = stdnse.module("mongodb", stdnse.seeall)
-- this is not yet widely implemented but at least used for authentication
-- ideally, it would be used to set the database against which operations,
-- that do not require a specific database, should run
local arg_DB = stdnse.get_script_args("mongodb.db")
-- Some lazy shortcuts
local function dbg(str,...)
stdnse.print_debug(3, "MngoDb:"..str, ...)
end
--local dbg =stdnse.print_debug
local err =stdnse.print_debug
----------------------------------------------------------------------
-- First of all comes a Bson parsing library. This can easily be moved out into a separate library should other
-- services start to use Bson
----------------------------------------------------------------------
-- Library methods for handling the BSON format
--
-- For more documentation about the BSON format,
---and more details about it's implementations, check out the
-- python BSON implementation which is available at
-- http://github.com/mongodb/mongo-python-driver/blob/master/pymongo/bson.py
-- and licensed under the Apache License, Version 2.0 http://www.apache.org/licenses/LICENSE-2.0)
--
-- @author Martin Holst Swende
-- @copyright Same as Nmap--See http://nmap.org/book/man-legal.html
--
-- Version 0.1
-- Created 01/13/2010 - v0.1 - created by Martin Holst Swende <martin@swende.se>
--module("bson", package.seeall)
--require("bin")
local function dbg_err(str,...)
stdnse.print_debug("Bson-ERR:"..str, ...)
end
--local err =stdnse.log_error
-- Packs data into nullterminated string
--@param input the string to pack
--@return the packed nullterminated string
local function make_nullterminated_string(input)
return bin.pack("z",input)
end
--Converts an element (key, value) into bson binary data
--@param key the key name, must *NOT* contain . (period) or start with $
--@param value, the element value
--@return status : true if ok, false if error
--@return result : the packed binary data OR error message
local function _element_to_bson(key, value)
--Some constraints-checking
if type(key) ~= 'string' then
return false, "Documents must have only string keys, key was " .. type(key)
end
if key:sub(1,1) == "$" then
return false, "key must not start with $: ".. key
end
if key:find("%.") then
return false, ("key %r must not contain '.'"):format(tostring(key))
end
local name =bin.pack("z",key) -- null-terminated string
if type(value) == 'string' then
local cstring = bin.pack("z",value) -- null-terminated string
local length = bin.pack("<i", cstring:len())
local op = bin.pack('H','02')
return true, op .. name .. length .. cstring
elseif type(value) =='table' then
return true, bin.pack('H','02') .. name .. toBson(value)
elseif type(value)== 'boolean' then
return true, bin.pack('H','08') + name + bin.pack('H',value and '01' or '00')
elseif type(value) == 'number' then
--return bin.pack('H','10').. name .. bin.pack("<i", value)
-- Use 01 - double for - works better than 10
return true, bin.pack('H','01') .. name .. bin.pack("<d", value)
end
local _ = ("cannot convert value of type %s to bson"):format(type(value))
return false, _
end
--Converts a table of elements to binary bson format
--@param dict the table
--@return status : true if ok, false if error
--@return result : a string of binary data OR error message
function toBson(dict)
local elements = ""
--Put id first
if dict._id then
local status,res = _element_to_bson("_id", dict._id)
if not status then return false, res end
elements = elements..res
elseif ( dict._cmd ) then
for k, v in pairs(dict._cmd) do
local status,res = _element_to_bson(k, v)
if not status then return false, res end
elements = elements..res
end
end
--Concatenate binary values
for key, value in pairs( dict ) do
if key ~= "_id" and key ~= "_cmd" then
dbg("dictionary to bson : key,value =(%s,%s)",key,value)
local status,res = _element_to_bson(key,value)
if not status then return false, res end
elements = elements..res
end
end
-- Get length
local length = #elements + 5
if length > 4 * 1024 * 1024 then
return false, "document too large - BSON documents are limited to 4 MB"
end
dbg("Packet length is %d",length)
--Final pack
return true, bin.pack("I", length) .. elements .. bin.pack('H',"00")
end
-- Reads a null-terminated string. If length is supplied, it is just cut
-- out from the data, otherwise the data is scanned for at null-char.
--@param data the data which starts with a c-string
--@param length optional length of the string
--@return the string
--@return the remaining data (*without* null-char)
local function get_c_string(data,length)
if not length then
local index = data:find(string.char(0))
if index == nil then
error({code="C-string did not contain NULL char"})
end
length = index
end
local value = data:sub(1,length-1)
--dbg("Found char at pos %d, data is %s c-string is %s",length, data, value)
return value, data:sub(length+1)
end
-- Element parser. Parse data elements
-- @param data String containing binary data
-- @return Position in the data string where parsing stopped
-- @return Unpacked value
-- @return error string if error occurred
local function parse(code,data)
if 1 == code then -- double
return bin.unpack("<d", data)
elseif 2 == code then -- string
-- data length = first four bytes
local _,len = bin.unpack("<i",data)
-- string data = data[5] -->
local value = get_c_string(data:sub(5), len)
-- Count position as header (=4) + length of string (=len)+ null char (=1)
return 4+len+1,value
elseif 3 == code or 4 == code then -- table or array
local object, err
-- Need to know the length, to return later
local _,obj_size = bin.unpack("<i", data)
-- Now, get the data object
dbg("Recursing into bson array")
object, data, err = fromBson(data)
dbg("Recurse finished, got data object")
-- And return where the parsing stopped
return obj_size+1, object
--6 = _get_null
--7 = _get_oid
elseif 8 == code then -- Boolean
return 2, data:byte(1) == 1
elseif 9 == code then -- int64, UTC datetime
return bin.unpack("<l", data)
elseif 10 == code then -- nullvalue
return 0,nil
--11= _get_regex
--12= _get_ref
--13= _get_string, # code
--14= _get_string, # symbol
--15= _get_code_w_scope
elseif 16 == code then -- 4 byte integer
return bin.unpack("<i", data)
--17= _get_timestamp
elseif 18 == code then -- long
return bin.unpack("<l", data)
end
local err = ("Getter for %d not implemented"):format(code)
return 0, data, err
end
-- Reads an element from binary to BSon
--@param data a string of data to convert
--@return Name of the element
--@return Value of the element
--@return Residual data not used
--@return any error that occurred
local function _element_to_dict(data)
local element_type, element_name, err, pos, value
--local element_size = data:byte(1)
element_type = data:byte(1)
element_name, data = get_c_string(data:sub(2))
dbg(" Read element name '%s' (type:%s), data left: %d",element_name, element_type,data:len())
--pos,value,err = parsers.get(element_type)(data)
pos,value,err = parse(element_type,data)
if(err ~= nil) then
dbg_err(err)
return nil,nil, data, err
end
data=data:sub(pos)
dbg(" Read element value '%s', data left: %d",tostring(value), data:len())
return element_name, value, data
end
--Reads all elements from binary to BSon
--@param data the data to read from
--@return the resulting table
local function _elements_to_dict(data)
local result = {}
local key,value
while data and data:len() > 1 do
key, value, data = _element_to_dict(data)
dbg("Parsed (%s='%s'), data left : %d", tostring(key),tostring(value), data:len())
if type(value) ~= 'table' then value=tostring(value) end
result[key] = value
end
return result
end
--Checks if enough data to parse the result is captured
--@data binary bson data read from socket
--@return true if the full bson table is contained in the data, false if data is incomplete
--@return required size of packet, if known, otherwise nil
function isPacketComplete(data)
-- First, we check that the header is complete
if data:len() < 4 then
local err_msg = "Not enough data in buffer, at least 4 bytes header info expected"
return false
end
local _,obj_size = bin.unpack("<i", data)
dbg("BSon packet size is %s", obj_size)
-- Check that all data is read and the packet is complete
if data:len() < obj_size then
return false,obj_size
end
return true,obj_size
end
-- Converts bson binary data read from socket into a table
-- of elements
--@param data: binary data
--@return table containing elements
--@return remaining data
--@return error message if not enough data was in packet
function fromBson(data)
dbg("Decoding, got %s bytes of data", data:len())
local complete, object_size = isPacketComplete(data)
if not complete then
local err_msg = ("Not enough data in buffer, expected %s but only has %d"):format(object_size or "?", data:len())
dbg(err_msg)
return {},data, err_msg
end
local element_portion = data:sub(5,object_size)
local remainder = data:sub(object_size+1)
return _elements_to_dict(element_portion), remainder
end
----------------------------------------------------------------------------------
-- Test-code for debugging purposes below
----------------------------------------------------------------------------------
function testBson()
local p = toBson({hello="world", test="ok"})
print( "Encoded something ok")
local orig = fromBson(p)
print(" Decoded something else ok")
for i,v in pairs(orig) do
print(i,v)
end
end
--testBson()
--------------------------------------------------------------------------------------------------------------
--- END of BSON part
--------------------------------------------------------------------------------------------------------------
--[[ MongoDB wire protocol format
Standard message header :
struct {
int32 messageLength; // total size of the message, including the 4 bytes of length
int32 requestID; // client or database-generated identifier for this message
int32 responseTo; // requestID from the original request (used in reponses from db)
int32 opCode; // request type - see table below
}
Opcodes :
OP_REPLY 1 Reply to a client request. responseTo is set
OP_MSG 1000 generic msg command followed by a string
OP_UPDATE 2001 update document
OP_INSERT 2002 insert new document
OP_GET_BY_OID 2003 is this used?
OP_QUERY 2004 query a collection
OP_GET_MORE 2005 Get more data from a query. See Cursors
OP_DELETE 2006 Delete documents
OP_KILL_CURSORS 2007 Tell database client is done with a cursor
Query message :
struct {
MsgHeader header; // standard message header
int32 opts; // query options. See below for details.
cstring fullCollectionName; // "dbname.collectionname"
int32 numberToSkip; // number of documents to skip when returning results
int32 numberToReturn; // number of documents to return in the first OP_REPLY
BSON query ; // query object. See below for details.
[ BSON returnFieldSelector; ] // OPTIONAL : selector indicating the fields to return. See below for details.
}
For more info about the MongoDB wire protocol, see http://www.mongodb.org/display/DOCS/Mongo+Wire+Protocol
--]]
-- DIY lua-class to create Mongo packets
--@usage call MongoData:new({opCode=MongoData.OP.QUERY}) to create query object
MongoData ={
uniqueRequestId = 12345,
-- Opcodes used by Mongo db
OP = {
REPLY = 1,
MSG = 1000,
UPDATE = 2001,
INSERT = 2002,
GET_BY_IOD =2003,
QUERY = 2004,
GET_MORE = 2005,
DELETE = 2006,
KILL_CURSORS = 2007,
},
-- Lua-DIY constructor
new = function (self,o,opCode,responseTo)
o = o or {} -- create object if user does not provide one
setmetatable(o, self) -- DIY inheritance a'la javascript
self.__index = self
self.valueString = ''
self.requestID = MongoData.uniqueRequestId -- Create unique id for message
MongoData.uniqueRequestId = MongoData.uniqueRequestId +1
return o
end
}
--Adds unsigned int32 to the message body
--@param value the value to add
function MongoData:addUnsignedInt32(value)
self.valueString = self.valueString..bin.pack("I",value)
end
-- Adds a string to the message body
--@param value the string to add
function MongoData:addString(value)
self.valueString = self.valueString..bin.pack('z',value)
end
-- Add a table as a BSon object to the body
--@param dict the table to be converted to BSon
--@return status : true if ok, false if error occurred
--@return Error message if error occurred
function MongoData:addBSON(dict)
-- status, res = bson.toBson(dict)
local status, res = toBson(dict)
if not status then
dbg(res)
return status,res
end
self.valueString = self.valueString..res
return true
end
-- Returns the data in this packet in a raw string format to be sent on socket
-- This method creates necessary header information and puts it with the body
function MongoData:data()
local header = MongoData:new()
header:addUnsignedInt32( self.valueString:len()+4+4+4+4)
header:addUnsignedInt32( self.requestID)
header:addUnsignedInt32( self.responseTo or 0xFFFFFFFF)
header:addUnsignedInt32( self.opCode)
return header.valueString .. self.valueString
end
-- Creates a query
-- @param collectionName string specifying the collection to run query against
-- @param a table containing the query
--@return status : true for OK, false for error
--@return packet data OR error message
local function createQuery(collectionName, query)
local packet = MongoData:new({opCode=MongoData.OP.QUERY})
packet:addUnsignedInt32(0); -- options
packet:addString(collectionName);
packet:addUnsignedInt32(0) -- number to skip
packet:addUnsignedInt32(-1) -- number to return : no limit
local status, error = packet:addBSON(query)
if not status then
return status, error
end
return true, packet:data()
end
-- Creates a get last error query
-- @param responseTo optional identifier this packet is a response to
--@return status : true for OK, false for error
--@return packet data OR error message
function lastErrorQuery(responseTo)
local collectionName = "test.$cmd"
local query = {getlasterror=1}
return createQuery(collectionName, query)
end
-- Creates a server status query
-- @param responseTo optional identifier this packet is a response to
--@return status : true for OK, false for error
--@return packet data OR error message
function serverStatusQuery(responseTo)
local collectionName = "test.$cmd"
local query = {serverStatus = 1}
return createQuery(collectionName, query)
end
-- Creates a optime query
-- @param responseTo optional identifier this packet is a response to
--@return status : true for OK, false for error
--@return packet data OR error message
function opTimeQuery(responseTo)
local collectionName = "test.$cmd"
local query = {getoptime = 1}
return createQuery(collectionName, query)
end
-- Creates a list databases query
-- @param responseTo optional identifier this packet is a response to
--@return status : true for OK, false for error
--@return packet data OR error message
function listDbQuery(responseTo)
local collectionName = "admin.$cmd"
local query = {listDatabases = 1}
return createQuery(collectionName, query)
end
-- Creates a build info query
-- @param responseTo optional identifier this packet is a response to
--@return status : true for OK, false for error
--@return packet data OR error message
--@return status : true for OK, false for error
--@return packet data OR error message
function buildInfoQuery(responseTo)
local collectionName = "admin.$cmd"
local query = {buildinfo = 1}
return createQuery(collectionName, query)
end
--Reads an int32 from data
--@return int32 value
--@return data unread
local function parseInt32(data)
local pos,val = bin.unpack("<i",data)
return val, data:sub(pos)
end
local function parseInt64(data)
local pos,val = bin.unpack("<l",data)
return val, data:sub(pos)
end
-- Parses response header
-- The response header looks like this :
--[[
struct {
MsgHeader header; // standard message header
int32 responseFlag; // normally zero, non-zero on query failure
int64 cursorID; // id of the cursor created for this query response
int32 startingFrom; // indicates where in the cursor this reply is starting
int32 numberReturned; // number of documents in the reply
BSON[] documents; // documents
}
--]]
--@param the data from socket
--@return a table containing the header data
local function parseResponseHeader(data)
local response= {}
local hdr, rflag, cID, sfrom, nRet, docs
-- First std message header
hdr ={}
hdr["messageLength"], data = parseInt32(data)
hdr["requestID"], data = parseInt32(data)
hdr["responseTo"], data = parseInt32(data)
hdr["opCode"], data = parseInt32(data)
response["header"] = hdr
-- Some additional fields
response["responseFlag"] ,data = parseInt32(data)
response["cursorID"] ,data = parseInt64(data)
response["startingFrom"] ,data = parseInt32(data)
response["numberReturned"] ,data = parseInt32(data)
response["bson"] = data
return response
end
--Checks if enough data to parse the result is captured
--@data binary mongodb data read from socket
--@return true if the full mongodb packet is contained in the data, false if data is incomplete
--@return required size of packet, if known, otherwise nil
function isPacketComplete(data)
-- First, we check that the header is complete
if data:len() < 4 then
local err_msg = "Not enough data in buffer, at least 4 bytes header info expected"
return false
end
local _,obj_size = bin.unpack("<i", data)
dbg("MongoDb Packet size is %s, (got %d)", obj_size,data:len())
-- Check that all data is read and the packet is complete
if data:len() < obj_size then
return false,obj_size
end
return true,obj_size
end
-- Sends a packet over a socket, reads the response
-- and parses it into a table
--@return status : true if ok; false if bad
--@return result : table of status ok, error msg if bad
--@return if status ok : remaining data read from socket but not used
function query(socket, data)
--Create an error handler
local catch = function()
socket:close()
stdnse.print_debug(string.format("Query failed"))
end
local try = nmap.new_try(catch)
try( socket:send( data ) )
local data = ""
local result = {}
local err_msg
local isComplete, pSize
while not isComplete do
dbg("mongo: waiting for data from socket, got %d bytes so far...",data:len())
data = data .. try( socket:receive() )
isComplete, pSize = isPacketComplete(data)
end
-- All required data shold be read now
local packetData = data:sub(1,pSize)
local residualData = data:sub(pSize+1)
local responseHeader = parseResponseHeader(packetData)
if responseHeader["responseFlag"] ~= 0 then
dbg("Response flag not zero : %d, some error occurred", responseHeader["responseFlag"])
end
local bsonData = responseHeader["bson"]
if #bsonData == 0 then
dbg("No BSon data returned ")
return false, "No Bson data returned"
end
-- result, data, err_msg = bson.fromBson(bsonData)
result, data, err_msg = fromBson(bsonData)
if err_msg then
dbg("Got error converting from bson: %s" , err_msg)
return false, ("Got error converting from bson: %s"):format(err_msg)
end
return true,result, residualData
end
function login(socket, db, username, password)
local collectionName = ("%s.$cmd"):format(arg_DB or db)
local query = { getnonce = 1 }
local status, packet = createQuery(collectionName, query)
local response
status, response = query(socket, packet)
if ( not(status) or not(response.nonce) ) then
return false, "Failed to retrieve nonce"
end
local nonce = response.nonce
local pwdigest = stdnse.tohex(openssl.md5(username .. ':mongo:' ..password))
local digest = stdnse.tohex(openssl.md5(nonce .. username .. pwdigest))
query = { user = username, nonce = nonce, key = digest }
query._cmd = { authenticate = 1 }
local status, packet = createQuery(collectionName, query)
status, response = query(socket, packet)
if ( not(status) ) then
return status, response
elseif ( response.errmsg == "auth fails" ) then
return false, "Authentication failed"
elseif ( response.errmsg ) then
return false, response.errmsg
end
return status, response
end
--- Converts a quert result as received from MongoDB query into nmap "result" table
-- @param resultTable table as returned from a quer
-- @return table suitable for <code>stdnse.format_output</code>
function queryResultToTable( resultTable )
local result = {}
for k,v in pairs( resultTable ) do
if type(v) == 'table' then
table.insert(result,k)
table.insert(result,queryResultToTable(v))
else
table.insert(result,(("%s = %s"):format(tostring(k), tostring(v))))
end
end
return result
end
----------------------------------------------------------------------------------
-- Test-code for debugging purposes below
----------------------------------------------------------------------------------
--- Prints data (string) as Hex values, e.g so it more easily can
-- be compared with a packet dump
-- @param strData the data in a string format
local function printBuffer(strData)
local out = ''
local ch
for i = 1,strData:len() do
out = out .." "
ch =strData:byte(i)
if(ch < 16) then
ch = string.format("0%x",ch)
else ch = string.format("%x",ch)
end
--if ch > 64 and ch < 123 then
-- out = out .. string.char(ch)
--else
out = out .. ch
--end
end
print(out)
end
-- function test()
-- local res
-- res = versionQuery()
-- print(type(res),res:len(),res)
-- local out= bin.unpack('C'..#res,res)
-- printBuffer(res)
-- end
--test()
return _ENV;
@KyuuKazami