diff options
Diffstat (limited to 'htp/htp_request.c')
| -rw-r--r-- | htp/htp_request.c | 1173 |
1 files changed, 1173 insertions, 0 deletions
diff --git a/htp/htp_request.c b/htp/htp_request.c new file mode 100644 index 0000000..9fddbd8 --- /dev/null +++ b/htp/htp_request.c @@ -0,0 +1,1173 @@ +/*************************************************************************** + * Copyright (c) 2009-2010 Open Information Security Foundation + * Copyright (c) 2010-2013 Qualys, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + + * - Neither the name of the Qualys, Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ***************************************************************************/ + +/** + * @file + * @author Ivan Ristic <ivanr@webkreator.com> + */ + +#include "htp_config_auto.h" + +#include "htp_private.h" + +#define IN_TEST_NEXT_BYTE_OR_RETURN(X) \ +if ((X)->in_current_read_offset >= (X)->in_current_len) { \ + return HTP_DATA; \ +} + +#define IN_PEEK_NEXT(X) \ +if ((X)->in_current_read_offset >= (X)->in_current_len) { \ + (X)->in_next_byte = -1; \ +} else { \ + (X)->in_next_byte = (X)->in_current_data[(X)->in_current_read_offset]; \ +} + +#define IN_NEXT_BYTE(X) \ +if ((X)->in_current_read_offset < (X)->in_current_len) { \ + (X)->in_next_byte = (X)->in_current_data[(X)->in_current_read_offset]; \ + (X)->in_current_read_offset++; \ + (X)->in_current_consume_offset++; \ + (X)->in_stream_offset++; \ +} else { \ + (X)->in_next_byte = -1; \ +} + +#define IN_NEXT_BYTE_OR_RETURN(X) \ +if ((X)->in_current_read_offset < (X)->in_current_len) { \ + (X)->in_next_byte = (X)->in_current_data[(X)->in_current_read_offset]; \ + (X)->in_current_read_offset++; \ + (X)->in_current_consume_offset++; \ + (X)->in_stream_offset++; \ +} else { \ + return HTP_DATA; \ +} + +#define IN_COPY_BYTE_OR_RETURN(X) \ +if ((X)->in_current_read_offset < (X)->in_current_len) { \ + (X)->in_next_byte = (X)->in_current_data[(X)->in_current_read_offset]; \ + (X)->in_current_read_offset++; \ + (X)->in_stream_offset++; \ +} else { \ + return HTP_DATA_BUFFER; \ +} + +/** + * Sends outstanding connection data to the currently active data receiver hook. + * + * @param[in] connp + * @param[in] is_last + * @return HTP_OK, or a value returned from a callback. + */ +static htp_status_t htp_connp_req_receiver_send_data(htp_connp_t *connp, int is_last) { + if (connp->in_data_receiver_hook == NULL) return HTP_OK; + + htp_tx_data_t d; + d.tx = connp->in_tx; + d.data = connp->in_current_data + connp->in_current_receiver_offset; + d.len = connp->in_current_read_offset - connp->in_current_receiver_offset; + d.is_last = is_last; + + htp_status_t rc = htp_hook_run_all(connp->in_data_receiver_hook, &d); + if (rc != HTP_OK) return rc; + + connp->in_current_receiver_offset = connp->in_current_read_offset; + + return HTP_OK; +} + +/** + * Configures the data receiver hook. If there is a previous hook, it will be finalized and cleared. + * + * @param[in] connp + * @param[in] data_receiver_hook + * @return HTP_OK, or a value returned from a callback. + */ +static htp_status_t htp_connp_req_receiver_set(htp_connp_t *connp, htp_hook_t *data_receiver_hook) { + htp_status_t rc = htp_connp_req_receiver_finalize_clear(connp); + + connp->in_data_receiver_hook = data_receiver_hook; + connp->in_current_receiver_offset = connp->in_current_read_offset; + + return rc; +} + +/** + * Finalizes an existing data receiver hook by sending any outstanding data to it. The + * hook is then removed so that it receives no more data. + * + * @param[in] connp + * @return HTP_OK, or a value returned from a callback. + */ +htp_status_t htp_connp_req_receiver_finalize_clear(htp_connp_t *connp) { + if (connp->in_data_receiver_hook == NULL) return HTP_OK; + + htp_status_t rc = htp_connp_req_receiver_send_data(connp, 1 /* last */); + + connp->in_data_receiver_hook = NULL; + + return rc; +} + +/** + * Handles request parser state changes. At the moment, this function is used only + * to configure data receivers, which are sent raw connection data. + * + * @param[in] connp + * @return HTP_OK, or a value returned from a callback. + */ +static htp_status_t htp_req_handle_state_change(htp_connp_t *connp) { + if (connp->in_state_previous == connp->in_state) return HTP_OK; + + if (connp->in_state == htp_connp_REQ_HEADERS) { + htp_status_t rc = HTP_OK; + + switch (connp->in_tx->request_progress) { + case HTP_REQUEST_HEADERS: + rc = htp_connp_req_receiver_set(connp, connp->in_tx->cfg->hook_request_header_data); + break; + + case HTP_REQUEST_TRAILER: + rc = htp_connp_req_receiver_set(connp, connp->in_tx->cfg->hook_request_trailer_data); + break; + + default: + // Do nothing; receivers are currently used only for header blocks. + break; + } + + if (rc != HTP_OK) return rc; + } + + // Initially, I had the finalization of raw data sending here, but that + // caused the last REQUEST_HEADER_DATA hook to be invoked after the + // REQUEST_HEADERS hook -- which I thought made no sense. For that reason, + // the finalization is now initiated from the request header processing code, + // which is less elegant but provides a better user experience. Having some + // (or all) hooks to be invoked on state change might work better. + + connp->in_state_previous = connp->in_state; + + return HTP_OK; +} + +/** + * If there is any data left in the inbound data chunk, this function will preserve + * it for later consumption. The maximum amount accepted for buffering is controlled + * by htp_config_t::field_limit_hard. + * + * @param[in] connp + * @return HTP_OK, or HTP_ERROR on fatal failure. + */ +static htp_status_t htp_connp_req_buffer(htp_connp_t *connp) { + if (connp->in_current_data == NULL) return HTP_OK; + + unsigned char *data = connp->in_current_data + connp->in_current_consume_offset; + size_t len = connp->in_current_read_offset - connp->in_current_consume_offset; + + if (len == 0) + return HTP_OK; + + // Check the hard (buffering) limit. + + size_t newlen = connp->in_buf_size + len; + + // When calculating the size of the buffer, take into account the + // space we're using for the request header buffer. + if (connp->in_header != NULL) { + newlen += bstr_len(connp->in_header); + } + + if (newlen > connp->in_tx->cfg->field_limit_hard) { + htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Request buffer over the limit: size %zd limit %zd.", + newlen, connp->in_tx->cfg->field_limit_hard); + return HTP_ERROR; + } + + // Copy the data remaining in the buffer. + + if (connp->in_buf == NULL) { + connp->in_buf = malloc(len); + if (connp->in_buf == NULL) return HTP_ERROR; + memcpy(connp->in_buf, data, len); + connp->in_buf_size = len; + } else { + size_t newsize = connp->in_buf_size + len; + unsigned char *newbuf = realloc(connp->in_buf, newsize); + if (newbuf == NULL) return HTP_ERROR; + connp->in_buf = newbuf; + memcpy(connp->in_buf + connp->in_buf_size, data, len); + connp->in_buf_size = newsize; + } + + // Reset the consumer position. + connp->in_current_consume_offset = connp->in_current_read_offset; + + return HTP_OK; +} + +/** + * Returns to the caller the memory region that should be processed next. This function + * hides away the buffering process from the rest of the code, allowing it to work with + * non-buffered data that's in the inbound chunk, or buffered data that's in our structures. + * + * @param[in] connp + * @param[out] data + * @param[out] len + * @return HTP_OK + */ +static htp_status_t htp_connp_req_consolidate_data(htp_connp_t *connp, unsigned char **data, size_t *len) { + if (connp->in_buf == NULL) { + // We do not have any data buffered; point to the current data chunk. + *data = connp->in_current_data + connp->in_current_consume_offset; + *len = connp->in_current_read_offset - connp->in_current_consume_offset; + } else { + // We already have some data in the buffer. Add the data from the current + // chunk to it, and point to the consolidated buffer. + if (htp_connp_req_buffer(connp) != HTP_OK) { + return HTP_ERROR; + } + + *data = connp->in_buf; + *len = connp->in_buf_size; + } + + return HTP_OK; +} + +/** + * Clears buffered inbound data and resets the consumer position to the reader position. + * + * @param[in] connp + */ +static void htp_connp_req_clear_buffer(htp_connp_t *connp) { + connp->in_current_consume_offset = connp->in_current_read_offset; + + if (connp->in_buf != NULL) { + free(connp->in_buf); + connp->in_buf = NULL; + connp->in_buf_size = 0; + } +} + +/** + * Performs a check for a CONNECT transaction to decide whether inbound + * parsing needs to be suspended. + * + * @param[in] connp + * @return HTP_OK if the request does not use CONNECT, HTP_DATA_OTHER if + * inbound parsing needs to be suspended until we hear from the + * other side + */ +htp_status_t htp_connp_REQ_CONNECT_CHECK(htp_connp_t *connp) { + // If the request uses the CONNECT method, then there will + // not be a request body, but first we need to wait to see the + // response in order to determine if the tunneling request + // was a success. + if (connp->in_tx->request_method_number == HTP_M_CONNECT) { + connp->in_state = htp_connp_REQ_CONNECT_WAIT_RESPONSE; + connp->in_status = HTP_STREAM_DATA_OTHER; + return HTP_DATA_OTHER; + } + + // Continue to the next step to determine + // the presence of request body + connp->in_state = htp_connp_REQ_BODY_DETERMINE; + + return HTP_OK; +} + +/** + * Determines whether inbound parsing needs to continue or stop. In + * case the data appears to be plain text HTTP, we try to continue. + * + * @param[in] connp + * @return HTP_OK if the parser can resume parsing, HTP_DATA_BUFFER if + * we need more data. + */ +htp_status_t htp_connp_REQ_CONNECT_PROBE_DATA(htp_connp_t *connp) { + for (;;) {//;i < max_read; i++) { + IN_PEEK_NEXT(connp); + // Have we reached the end of the line? For some reason + // we can't test after IN_COPY_BYTE_OR_RETURN */ + if (connp->in_next_byte == LF || connp->in_next_byte == 0x00) + break; + + IN_COPY_BYTE_OR_RETURN(connp); + + } + + unsigned char *data; + size_t len; + if (htp_connp_req_consolidate_data(connp, &data, &len) != HTP_OK) { + return HTP_ERROR; + } +#ifdef HTP_DEBUG + fprint_raw_data(stderr, "PROBING", data, len); +#endif + + size_t pos = 0; + size_t mstart = 0; + // skip past leading whitespace. IIS allows this + while ((pos < len) && htp_is_space(data[pos])) + pos++; + if (pos) + mstart = pos; + // The request method starts at the beginning of the + // line and ends with the first whitespace character. + while ((pos < len) && (!htp_is_space(data[pos]))) + pos++; + + int methodi = HTP_M_UNKNOWN; + bstr *method = bstr_dup_mem(data + mstart, pos - mstart); + if (method) { + methodi = htp_convert_method_to_number(method); + bstr_free(method); + } + if (methodi != HTP_M_UNKNOWN) { +#ifdef HTP_DEBUG + fprint_raw_data(stderr, "htp_connp_REQ_CONNECT_PROBE_DATA: tunnel contains plain text HTTP", data, len); +#endif + return htp_tx_state_request_complete(connp->in_tx); + } else { +#ifdef HTP_DEBUG + fprint_raw_data(stderr, "htp_connp_REQ_CONNECT_PROBE_DATA: tunnel is not HTTP", data, len); +#endif + connp->in_status = HTP_STREAM_TUNNEL; + connp->out_status = HTP_STREAM_TUNNEL; + } + + // not calling htp_connp_req_clear_buffer, we're not consuming the data + + return HTP_OK; +} + +/** + * Determines whether inbound parsing, which was suspended after + * encountering a CONNECT transaction, can proceed (after receiving + * the response). + * + * @param[in] connp + * @return HTP_OK if the parser can resume parsing, HTP_DATA_OTHER if + * it needs to continue waiting. + */ +htp_status_t htp_connp_REQ_CONNECT_WAIT_RESPONSE(htp_connp_t *connp) { + // Check that we saw the response line of the current inbound transaction. + if (connp->in_tx->response_progress <= HTP_RESPONSE_LINE) { + return HTP_DATA_OTHER; + } + + // A 2xx response means a tunnel was established. Anything + // else means we continue to follow the HTTP stream. + if ((connp->in_tx->response_status_number >= 200) && (connp->in_tx->response_status_number <= 299)) { + // TODO Check that the server did not accept a connection to itself. + + // The requested tunnel was established: we are going + // to probe the remaining data on this stream to see + // if we need to ignore it or parse it + connp->in_state = htp_connp_REQ_CONNECT_PROBE_DATA; + } else { + // No tunnel; continue to the next transaction + connp->in_state = htp_connp_REQ_FINALIZE; + } + + return HTP_OK; +} + +/** + * Consumes bytes until the end of the current line. + * + * @param[in] connp + * @returns HTP_OK on state change, HTP_ERROR on error, or HTP_DATA when more data is needed. + */ +htp_status_t htp_connp_REQ_BODY_CHUNKED_DATA_END(htp_connp_t *connp) { + // TODO We shouldn't really see anything apart from CR and LF, + // so we should warn about anything else. + + for (;;) { + IN_NEXT_BYTE_OR_RETURN(connp); + + connp->in_tx->request_message_len++; + + if (connp->in_next_byte == LF) { + connp->in_state = htp_connp_REQ_BODY_CHUNKED_LENGTH; + return HTP_OK; + } + } + + return HTP_ERROR; +} + +/** + * Processes a chunk of data. + * + * @param[in] connp + * @returns HTP_OK on state change, HTP_ERROR on error, or HTP_DATA when more data is needed. + */ +htp_status_t htp_connp_REQ_BODY_CHUNKED_DATA(htp_connp_t *connp) { + // Determine how many bytes we can consume. + size_t bytes_to_consume; + if (connp->in_current_len - connp->in_current_read_offset >= connp->in_chunked_length) { + // Entire chunk available in the buffer; read all of it. + bytes_to_consume = connp->in_chunked_length; + } else { + // Partial chunk available in the buffer; read as much as we can. + bytes_to_consume = connp->in_current_len - connp->in_current_read_offset; + } + + #ifdef HTP_DEBUG + fprintf(stderr, "htp_connp_REQ_BODY_CHUNKED_DATA Consuming %zd bytes\n", bytes_to_consume); + #endif + + // If the input buffer is empty, ask for more data. + if (bytes_to_consume == 0) return HTP_DATA; + + // Consume the data. + htp_status_t rc = htp_tx_req_process_body_data_ex(connp->in_tx, connp->in_current_data + connp->in_current_read_offset, bytes_to_consume); + if (rc != HTP_OK) return rc; + + // Adjust counters. + connp->in_current_read_offset += bytes_to_consume; + connp->in_current_consume_offset += bytes_to_consume; + connp->in_stream_offset += bytes_to_consume; + connp->in_tx->request_message_len += bytes_to_consume; + connp->in_chunked_length -= bytes_to_consume; + + if (connp->in_chunked_length == 0) { + // End of the chunk. + connp->in_state = htp_connp_REQ_BODY_CHUNKED_DATA_END; + return HTP_OK; + } + + // Ask for more data. + return HTP_DATA; +} + +/** + * Extracts chunk length. + * + * @param[in] connp + * @returns HTP_OK on state change, HTP_ERROR on error, or HTP_DATA when more data is needed. + */ +htp_status_t htp_connp_REQ_BODY_CHUNKED_LENGTH(htp_connp_t *connp) { + for (;;) { + IN_COPY_BYTE_OR_RETURN(connp); + + // Have we reached the end of the line? + if (connp->in_next_byte == LF) { + unsigned char *data; + size_t len; + + if (htp_connp_req_consolidate_data(connp, &data, &len) != HTP_OK) { + return HTP_ERROR; + } + + connp->in_tx->request_message_len += len; + + #ifdef HTP_DEBUG + fprint_raw_data(stderr, "Chunk length line", data, len); + #endif + + htp_chomp(data, &len); + + int chunk_ext = 0; + connp->in_chunked_length = htp_parse_chunked_length(data, len, &chunk_ext); + if (chunk_ext == 1) { + htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Request chunk extension"); + } + + htp_connp_req_clear_buffer(connp); + + // Handle chunk length. + if (connp->in_chunked_length > 0) { + // More data available. + connp->in_state = htp_connp_REQ_BODY_CHUNKED_DATA; + } else if (connp->in_chunked_length == 0) { + // End of data. + connp->in_state = htp_connp_REQ_HEADERS; + connp->in_tx->request_progress = HTP_REQUEST_TRAILER; + } else { + // Invalid chunk length. + htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Request chunk encoding: Invalid chunk length"); + return HTP_ERROR; + } + + return HTP_OK; + } + } + + return HTP_ERROR; +} + +/** + * Processes identity request body. + * + * @param[in] connp + * @returns HTP_OK on state change, HTP_ERROR on error, or HTP_DATA when more data is needed. + */ +htp_status_t htp_connp_REQ_BODY_IDENTITY(htp_connp_t *connp) { + // Determine how many bytes we can consume. + size_t bytes_to_consume; + if (connp->in_current_len - connp->in_current_read_offset >= connp->in_body_data_left) { + bytes_to_consume = connp->in_body_data_left; + } else { + bytes_to_consume = connp->in_current_len - connp->in_current_read_offset; + } + + // If the input buffer is empty, ask for more data. + if (bytes_to_consume == 0) return HTP_DATA; + + // Consume data. + int rc = htp_tx_req_process_body_data_ex(connp->in_tx, connp->in_current_data + connp->in_current_read_offset, bytes_to_consume); + if (rc != HTP_OK) return rc; + + // Adjust counters. + connp->in_current_read_offset += bytes_to_consume; + connp->in_current_consume_offset += bytes_to_consume; + connp->in_stream_offset += bytes_to_consume; + connp->in_tx->request_message_len += bytes_to_consume; + connp->in_body_data_left -= bytes_to_consume; + + if (connp->in_body_data_left == 0) { + // End of request body. + connp->in_state = htp_connp_REQ_FINALIZE; + return HTP_OK; + } + + // Ask for more data. + return HTP_DATA; +} + +/** + * Determines presence (and encoding) of a request body. + * + * @param[in] connp + * @returns HTP_OK on state change, HTP_ERROR on error, or HTP_DATA when more data is needed. + */ +htp_status_t htp_connp_REQ_BODY_DETERMINE(htp_connp_t *connp) { + // Determine the next state based on the presence of the request + // body, and the coding used. + switch (connp->in_tx->request_transfer_coding) { + + case HTP_CODING_CHUNKED: + connp->in_state = htp_connp_REQ_BODY_CHUNKED_LENGTH; + connp->in_tx->request_progress = HTP_REQUEST_BODY; + break; + + case HTP_CODING_IDENTITY: + connp->in_content_length = connp->in_tx->request_content_length; + connp->in_body_data_left = connp->in_content_length; + + if (connp->in_content_length != 0) { + connp->in_state = htp_connp_REQ_BODY_IDENTITY; + connp->in_tx->request_progress = HTP_REQUEST_BODY; + } else { + connp->in_tx->connp->in_state = htp_connp_REQ_FINALIZE; + } + break; + + case HTP_CODING_NO_BODY: + // This request does not have a body, which + // means that we're done with it + connp->in_state = htp_connp_REQ_FINALIZE; + break; + + default: + // Should not be here + return HTP_ERROR; + break; + } + + return HTP_OK; +} + +/** + * Parses request headers. + * + * @param[in] connp + * @returns HTP_OK on state change, HTP_ERROR on error, or HTP_DATA when more data is needed. + */ +htp_status_t htp_connp_REQ_HEADERS(htp_connp_t *connp) { + for (;;) { + if (connp->in_status == HTP_STREAM_CLOSED) { + // Parse previous header, if any. + if (connp->in_header != NULL) { + if (connp->cfg->process_request_header(connp, bstr_ptr(connp->in_header), + bstr_len(connp->in_header)) != HTP_OK) + return HTP_ERROR; + bstr_free(connp->in_header); + connp->in_header = NULL; + } + + htp_connp_req_clear_buffer(connp); + + connp->in_tx->request_progress = HTP_REQUEST_TRAILER; + + // We've seen all the request headers. + return htp_tx_state_request_headers(connp->in_tx); + } + IN_COPY_BYTE_OR_RETURN(connp); + + // Have we reached the end of the line? + if (connp->in_next_byte == LF) { + unsigned char *data; + size_t len; + + if (htp_connp_req_consolidate_data(connp, &data, &len) != HTP_OK) { + return HTP_ERROR; + } + + #ifdef HTP_DEBUG + fprint_raw_data(stderr, __func__, data, len); + #endif + + // Should we terminate headers? + if (htp_connp_is_line_terminator(connp, data, len, 0)) { + // Parse previous header, if any. + if (connp->in_header != NULL) { + if (connp->cfg->process_request_header(connp, bstr_ptr(connp->in_header), + bstr_len(connp->in_header)) != HTP_OK) return HTP_ERROR; + + bstr_free(connp->in_header); + connp->in_header = NULL; + } + + htp_connp_req_clear_buffer(connp); + + // We've seen all the request headers. + return htp_tx_state_request_headers(connp->in_tx); + } + + htp_chomp(data, &len); + + // Check for header folding. + if (htp_connp_is_line_folded(data, len) == 0) { + // New header line. + + // Parse previous header, if any. + if (connp->in_header != NULL) { + if (connp->cfg->process_request_header(connp, bstr_ptr(connp->in_header), + bstr_len(connp->in_header)) != HTP_OK) return HTP_ERROR; + + bstr_free(connp->in_header); + connp->in_header = NULL; + } + + IN_PEEK_NEXT(connp); + + if (connp->in_next_byte != -1 && htp_is_folding_char(connp->in_next_byte) == 0) { + // Because we know this header is not folded, we can process the buffer straight away. + if (connp->cfg->process_request_header(connp, data, len) != HTP_OK) return HTP_ERROR; + } else { + // Keep the partial header data for parsing later. + connp->in_header = bstr_dup_mem(data, len); + if (connp->in_header == NULL) return HTP_ERROR; + } + } else { + // Folding; check that there's a previous header line to add to. + if (connp->in_header == NULL) { + // Invalid folding. + + // Warn only once per transaction. + if (!(connp->in_tx->flags & HTP_INVALID_FOLDING)) { + connp->in_tx->flags |= HTP_INVALID_FOLDING; + htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Invalid request field folding"); + } + + // Keep the header data for parsing later. + size_t trim = 0; + while(trim < len) { + if (!htp_is_folding_char(data[trim])) { + break; + } + trim++; + } + connp->in_header = bstr_dup_mem(data + trim, len - trim); + if (connp->in_header == NULL) return HTP_ERROR; + } else { + // Add to the existing header. + if (bstr_len(connp->in_header) < HTP_MAX_HEADER_FOLDED) { + bstr *new_in_header = bstr_add_mem(connp->in_header, data, len); + if (new_in_header == NULL) return HTP_ERROR; + connp->in_header = new_in_header; + } else { + htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Request field length exceeds folded maximum"); + } + } + } + + htp_connp_req_clear_buffer(connp); + } + } + + return HTP_ERROR; +} + +// HTTP/0.9 is supposed to be only a request line without protocol. +// Libhtp will still consider the request to be HTTP/0.9 if there +// are some junk whitespaces after that request line. +// Libhtp allows the small value of 16 extra bytes/whitespaces, +// otherwise we consider it to be a HTTP/1.x request with missing protocol. +// It is unlikely to meet HTTP/0.9, and we want to limit probing. +#define HTTP09_MAX_JUNK_LEN 16 + +/** + * Determines request protocol. + * + * @param[in] connp + * @returns HTP_OK on state change, HTP_ERROR on error, or HTP_DATA when more data is needed. + */ +htp_status_t htp_connp_REQ_PROTOCOL(htp_connp_t *connp) { + // Is this a short-style HTTP/0.9 request? If it is, + // we will not want to parse request headers. + if (connp->in_tx->is_protocol_0_9 == 0) { + // Switch to request header parsing. + connp->in_state = htp_connp_REQ_HEADERS; + connp->in_tx->request_progress = HTP_REQUEST_HEADERS; + } else { + // Let's check if the protocol was simply missing + int64_t pos = connp->in_current_read_offset; + // Probe if data looks like a header line + if (connp->in_current_len > connp->in_current_read_offset + HTTP09_MAX_JUNK_LEN) { + htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Request line: missing protocol"); + connp->in_tx->is_protocol_0_9 = 0; + // Switch to request header parsing. + connp->in_state = htp_connp_REQ_HEADERS; + connp->in_tx->request_progress = HTP_REQUEST_HEADERS; + return HTP_OK; + } + while (pos < connp->in_current_len) { + if (!htp_is_space(connp->in_current_data[pos])) { + htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Request line: missing protocol"); + connp->in_tx->is_protocol_0_9 = 0; + // Switch to request header parsing. + connp->in_state = htp_connp_REQ_HEADERS; + connp->in_tx->request_progress = HTP_REQUEST_HEADERS; + return HTP_OK; + } + pos++; + } + // We're done with this request. + connp->in_state = htp_connp_REQ_FINALIZE; + } + + return HTP_OK; +} + +/** + * Parse the request line. + * + * @param[in] connp + * @returns HTP_OK on succesful parse, HTP_ERROR on error. + */ +htp_status_t htp_connp_REQ_LINE_complete(htp_connp_t *connp) { + unsigned char *data; + size_t len; + + if (htp_connp_req_consolidate_data(connp, &data, &len) != HTP_OK) { + return HTP_ERROR; + } + + #ifdef HTP_DEBUG + fprint_raw_data(stderr, __func__, data, len); + #endif + if (len == 0) { + htp_connp_req_clear_buffer(connp); + return HTP_DATA; + } + + // Is this a line that should be ignored? + if (htp_connp_is_line_ignorable(connp, data, len)) { + // We have an empty/whitespace line, which we'll note, ignore and move on. + connp->in_tx->request_ignored_lines++; + + htp_connp_req_clear_buffer(connp); + + return HTP_OK; + } + + // Process request line. + + htp_chomp(data, &len); + + connp->in_tx->request_line = bstr_dup_mem(data, len); + if (connp->in_tx->request_line == NULL) + return HTP_ERROR; + + if (connp->cfg->parse_request_line(connp) != HTP_OK) + return HTP_ERROR; + + // Finalize request line parsing. + + if (htp_tx_state_request_line(connp->in_tx) != HTP_OK) + return HTP_ERROR; + + htp_connp_req_clear_buffer(connp); + + return HTP_OK; +} + +/** + * Parses request line. + * + * @param[in] connp + * @returns HTP_OK on state change, HTP_ERROR on error, or HTP_DATA when more data is needed. + */ +htp_status_t htp_connp_REQ_LINE(htp_connp_t *connp) { + for (;;) { + // Get one byte + IN_PEEK_NEXT(connp); + if (connp->in_status == HTP_STREAM_CLOSED && connp->in_next_byte == -1) { + return htp_connp_REQ_LINE_complete(connp); + } + IN_COPY_BYTE_OR_RETURN(connp); + + // Have we reached the end of the line? + if (connp->in_next_byte == LF) { + return htp_connp_REQ_LINE_complete(connp); + } + } + + return HTP_ERROR; +} + +htp_status_t htp_connp_REQ_FINALIZE(htp_connp_t *connp) { + if (connp->in_status != HTP_STREAM_CLOSED) { + IN_PEEK_NEXT(connp); + if (connp->in_next_byte == -1) { + return htp_tx_state_request_complete(connp->in_tx); + } + if (connp->in_next_byte != LF || connp->in_current_consume_offset >= connp->in_current_read_offset) { + for (;;) {//;i < max_read; i++) { + // peek until LF but do not mark it read so that REQ_LINE works + IN_PEEK_NEXT(connp); + if (connp->in_next_byte == LF) + break; + IN_COPY_BYTE_OR_RETURN(connp); + } + } + } + + unsigned char *data; + size_t len; + if (htp_connp_req_consolidate_data(connp, &data, &len) != HTP_OK) { + return HTP_ERROR; + } +#ifdef HTP_DEBUG + fprint_raw_data(stderr, "PROBING request finalize", data, len); +#endif + if (len == 0) { + //closing + return htp_tx_state_request_complete(connp->in_tx); + } + + size_t pos = 0; + size_t mstart = 0; + // skip past leading whitespace. IIS allows this + while ((pos < len) && htp_is_space(data[pos])) + pos++; + if (pos) + mstart = pos; + // The request method starts at the beginning of the + // line and ends with the first whitespace character. + while ((pos < len) && (!htp_is_space(data[pos]))) + pos++; + + if (pos > mstart) { + //non empty whitespace line + int methodi = HTP_M_UNKNOWN; + bstr *method = bstr_dup_mem(data + mstart, pos - mstart); + if (method) { + methodi = htp_convert_method_to_number(method); + bstr_free(method); + } + if (methodi != HTP_M_UNKNOWN) { + connp->in_body_data_left = -1; + return htp_tx_state_request_complete(connp->in_tx); + } // else continue + if (connp->in_body_data_left <= 0) { + // log only once per transaction + htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Unexpected request body"); + } else { + connp->in_body_data_left = 1; + } + } + //Adds linefeed to the buffer if there was one + if (connp->in_next_byte == LF) { + IN_COPY_BYTE_OR_RETURN(connp); + htp_connp_req_consolidate_data(connp, &data, &len); + } + // Interpret remaining bytes as body data + htp_status_t rc = htp_tx_req_process_body_data_ex(connp->in_tx, data, len); + htp_connp_req_clear_buffer(connp); + return rc; +} + +htp_status_t htp_connp_REQ_IGNORE_DATA_AFTER_HTTP_0_9(htp_connp_t *connp) { + // Consume whatever is left in the buffer. + + size_t bytes_left = connp->in_current_len - connp->in_current_read_offset; + + if (bytes_left > 0) { + connp->conn->flags |= HTP_CONN_HTTP_0_9_EXTRA; + } + + connp->in_current_read_offset += bytes_left; + connp->in_current_consume_offset += bytes_left; + connp->in_stream_offset += bytes_left; + + return HTP_DATA; +} + +/** + * The idle state is where the parser will end up after a transaction is processed. + * If there is more data available, a new request will be started. + * + * @param[in] connp + * @returns HTP_OK on state change, HTP_ERROR on error, or HTP_DATA when more data is needed. + */ +htp_status_t htp_connp_REQ_IDLE(htp_connp_t * connp) { + // We want to start parsing the next request (and change + // the state from IDLE) only if there's at least one + // byte of data available. Otherwise we could be creating + // new structures even if there's no more data on the + // connection. + IN_TEST_NEXT_BYTE_OR_RETURN(connp); + + connp->in_tx = htp_connp_tx_create(connp); + if (connp->in_tx == NULL) return HTP_ERROR; + + // Change state to TRANSACTION_START + htp_tx_state_request_start(connp->in_tx); + + return HTP_OK; +} + +/** + * Returns how many bytes from the current data chunks were consumed so far. + * + * @param[in] connp + * @return The number of bytes consumed. + */ +size_t htp_connp_req_data_consumed(htp_connp_t *connp) { + return connp->in_current_read_offset; +} + +int htp_connp_req_data(htp_connp_t *connp, const htp_time_t *timestamp, const void *data, size_t len) { + #ifdef HTP_DEBUG + fprintf(stderr, "htp_connp_req_data(connp->in_status %x)\n", connp->in_status); + fprint_raw_data(stderr, __func__, data, len); + #endif + + // Return if the connection is in stop state. + if (connp->in_status == HTP_STREAM_STOP) { + htp_log(connp, HTP_LOG_MARK, HTP_LOG_INFO, 0, "Inbound parser is in HTP_STREAM_STOP"); + return HTP_STREAM_STOP; + } + + // Return if the connection had a fatal error earlier + if (connp->in_status == HTP_STREAM_ERROR) { + htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Inbound parser is in HTP_STREAM_ERROR"); + + #ifdef HTP_DEBUG + fprintf(stderr, "htp_connp_req_data: returning HTP_STREAM_DATA (previous error)\n"); + #endif + + return HTP_STREAM_ERROR; + } + + // Sanity check: we must have a transaction pointer if the state is not IDLE (no inbound transaction) + if ((connp->in_tx == NULL)&&(connp->in_state != htp_connp_REQ_IDLE)) { + connp->in_status = HTP_STREAM_ERROR; + + htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Missing inbound transaction data"); + + return HTP_STREAM_ERROR; + } + + // If the length of the supplied data chunk is zero, proceed + // only if the stream has been closed. We do not allow zero-sized + // chunks in the API, but we use them internally to force the parsers + // to finalize parsing. + if (len == 0 && connp->in_status != HTP_STREAM_CLOSED) { + htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Zero-length data chunks are not allowed"); + + #ifdef HTP_DEBUG + fprintf(stderr, "htp_connp_req_data: returning HTP_STREAM_DATA (zero-length chunk)\n"); + #endif + + return HTP_STREAM_CLOSED; + } + + // Remember the timestamp of the current request data chunk + if (timestamp != NULL) { + memcpy(&connp->in_timestamp, timestamp, sizeof (*timestamp)); + } + + // Store the current chunk information + connp->in_current_data = (unsigned char *) data; + connp->in_current_len = len; + connp->in_current_read_offset = 0; + connp->in_current_consume_offset = 0; + connp->in_current_receiver_offset = 0; + connp->in_chunk_count++; + + htp_conn_track_inbound_data(connp->conn, len, timestamp); + + + // Return without processing any data if the stream is in tunneling + // mode (which it would be after an initial CONNECT transaction). + if (connp->in_status == HTP_STREAM_TUNNEL) { + #ifdef HTP_DEBUG + fprintf(stderr, "htp_connp_req_data: returning HTP_STREAM_TUNNEL\n"); + #endif + + return HTP_STREAM_TUNNEL; + } + + if (connp->out_status == HTP_STREAM_DATA_OTHER) { + connp->out_status = HTP_STREAM_DATA; + } + + // Invoke a processor, in a loop, until an error + // occurs or until we run out of data. Many processors + // will process a request, each pointing to the next + // processor that needs to run. + for (;;) { + #ifdef HTP_DEBUG + fprintf(stderr, "htp_connp_req_data: in state=%s, progress=%s\n", + htp_connp_in_state_as_string(connp), + htp_tx_request_progress_as_string(connp->in_tx)); + #endif + + // Return if there's been an error or if we've run out of data. We are relying + // on processors to supply error messages, so we'll keep quiet here. + + htp_status_t rc; + //handle gap + if (data == NULL && len > 0) { + //cannot switch over a function pointer in C + if (connp->in_state == htp_connp_REQ_BODY_IDENTITY || + connp->in_state == htp_connp_REQ_IGNORE_DATA_AFTER_HTTP_0_9) { + rc = connp->in_state(connp); + } else if (connp->in_state == htp_connp_REQ_FINALIZE) { + //simple version without probing + rc = htp_tx_state_request_complete(connp->in_tx); + } else { + // go to htp_connp_REQ_CONNECT_PROBE_DATA ? + htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Gaps are not allowed during this state"); + return HTP_STREAM_CLOSED; + } + } else { + rc = connp->in_state(connp); + } + if (rc == HTP_OK) { + if (connp->in_status == HTP_STREAM_TUNNEL) { + #ifdef HTP_DEBUG + fprintf(stderr, "htp_connp_req_data: returning HTP_STREAM_TUNNEL\n"); + #endif + + return HTP_STREAM_TUNNEL; + } + + rc = htp_req_handle_state_change(connp); + } + + if (rc != HTP_OK) { + // Do we need more data? + if ((rc == HTP_DATA) || (rc == HTP_DATA_BUFFER)) { + htp_connp_req_receiver_send_data(connp, 0 /* not last */); + + if (rc == HTP_DATA_BUFFER) { + if (htp_connp_req_buffer(connp) != HTP_OK) { + connp->in_status = HTP_STREAM_ERROR; + return HTP_STREAM_ERROR; + } + } + + #ifdef HTP_DEBUG + fprintf(stderr, "htp_connp_req_data: returning HTP_STREAM_DATA\n"); + #endif + + connp->in_status = HTP_STREAM_DATA; + + return HTP_STREAM_DATA; + } + + // Check for suspended parsing. + if (rc == HTP_DATA_OTHER) { + // We might have actually consumed the entire data chunk? + if (connp->in_current_read_offset >= connp->in_current_len) { + // Do not send STREAM_DATE_DATA_OTHER if we've consumed the entire chunk. + + #ifdef HTP_DEBUG + fprintf(stderr, "htp_connp_req_data: returning HTP_STREAM_DATA (suspended parsing)\n"); + #endif + + connp->in_status = HTP_STREAM_DATA; + + return HTP_STREAM_DATA; + } else { + // Partial chunk consumption. + + #ifdef HTP_DEBUG + fprintf(stderr, "htp_connp_req_data: returning HTP_STREAM_DATA_OTHER\n"); + #endif + + connp->in_status = HTP_STREAM_DATA_OTHER; + + return HTP_STREAM_DATA_OTHER; + } + } + + // Check for the stop signal. + if (rc == HTP_STOP) { + #ifdef HTP_DEBUG + fprintf(stderr, "htp_connp_req_data: returning HTP_STREAM_STOP\n"); + #endif + + connp->in_status = HTP_STREAM_STOP; + + return HTP_STREAM_STOP; + } + + #ifdef HTP_DEBUG + fprintf(stderr, "htp_connp_req_data: returning HTP_STREAM_ERROR\n"); + #endif + + // Permanent stream error. + connp->in_status = HTP_STREAM_ERROR; + + return HTP_STREAM_ERROR; + } + } +} |