/***************************************************************************
 *
 * Copyright (c) 2000-2015 BalaBit IT Ltd, Budapest, Hungary
 * Copyright (c) 2015-2018 BalaSys IT Ltd, Budapest, Hungary
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 *
 ***************************************************************************/

#include "http.h"
#include "httpfltr.h"

#include <zorpll/log.h>
#include <zorpll/io.h>
#include <sys/socket.h>
#include <netinet/tcp.h>

/* transfer states */

#define HTTP_SR_INITIAL           0
#define HTTP_SR_PUSH_HEADERS    10
#define HTTP_SR_PUSH_HEADERS_MAX  10

#define HTTP_SR_READ_INITIAL      20
#define HTTP_SR_READ_CHUNK_LENGTH 21
#define HTTP_SR_READ_CHUNK        22
#define HTTP_SR_READ_FOOTER       23
#define HTTP_SR_READ_ENTITY       24     /* reading content-length or EOF terminated entity */

#define HTTP_DW_INITIAL            0

#define HTTP_DW_POP_HEADERS        10
#define HTTP_DW_POP_HEADERS_CR     11
#define HTTP_DW_POP_HEADERS_LF     12
#define HTTP_DW_POP_HEADERS_MAX    12

#define HTTP_DW_WRITE_PREAMBLE     20
#define HTTP_DW_WRITE_INITIAL      21
#define HTTP_DW_FORMAT_CHUNK_LENGTH 22
#define HTTP_DW_WRITE_CHUNK_LENGTH 23
#define HTTP_DW_WRITE_CHUNK        24
#define HTTP_DW_WRITE_FOOTER       25

extern ZClass HttpTransfer__class;

/* HttpTransfer implementation */

static GIOStatus
http_transfer_src_read(ZTransfer2 *s, ZStream *stream, gchar *buf, gsize count, gsize *bytes_read, GError **err)
{
  HttpTransfer *self = Z_CAST(s, HttpTransfer);
  HttpProxy *owner = Z_CAST(s->owner, HttpProxy);
  GError *local_error = NULL;
  gsize br;
  GIOStatus res = G_IO_STATUS_NORMAL;

  if (self->src_read_state == HTTP_SR_INITIAL)
    self->src_read_state = HTTP_SR_PUSH_HEADERS;

  if (self->src_read_state >= HTTP_SR_PUSH_HEADERS && self->src_read_state <= HTTP_SR_PUSH_HEADERS_MAX)
    {
      if (self->push_mime_headers && self->stacked_preamble->len > 0)
        {
          gint move;

          *bytes_read = 0;

          move = MIN(count, self->stacked_preamble->len - self->stacked_preamble_ofs);
          memmove(buf, self->stacked_preamble->str + self->stacked_preamble_ofs, move);
          self->stacked_preamble_ofs += move;
          *bytes_read = move;

          if (self->stacked_preamble_ofs == self->stacked_preamble->len)
            {
              z_transfer2_set_proxy_out(s, FALSE);
              self->src_read_state = HTTP_SR_READ_INITIAL;
            }

          return G_IO_STATUS_NORMAL;
        }
      else
        {
          self->src_read_state = HTTP_SR_READ_INITIAL;
        }
    }

  *bytes_read = 0;

  if (self->src_chunked)
    {
      /* read as a chunked stream */
      switch (self->src_read_state)
        {
        case HTTP_SR_READ_INITIAL:
          self->src_whole_length = 0;
          self->src_read_state = HTTP_SR_READ_CHUNK_LENGTH;
          z_stream_line_set_poll_partial(stream, FALSE);
          /* fallthrough */
        case HTTP_SR_READ_CHUNK_LENGTH:
          {
            gchar line_buf[32], *line;
            gsize line_length;
            guint32 chunk_length;
            gchar *end;

            res = z_stream_line_get(stream, &line, &line_length, NULL);

            if (res == G_IO_STATUS_NORMAL)
              {
                /* a complete line was read, check if it is a valid chunk length */
                if (line_length >= sizeof(line_buf) - 1)
                  {
                    /*LOG
                      This message indicates that the chunk length line is too long.
                      It is likely caused by a buggy client or server.
                    */
                    z_proxy_log(self->super.owner, HTTP_VIOLATION, 1, "Chunk length line too long; line='%.*s'", (gint) line_length, line);
                    res = G_IO_STATUS_ERROR;
                    break;
                  }

                /* we already checked that line_buf is large enough */
                memcpy(line_buf, line, line_length);
                line_buf[line_length] = 0;
                chunk_length = strtoul(line_buf, &end, 16);

                if (end == line_buf)
                  {
                    /* hmm... invalid chunk length */
                    /*LOG
                      This message indicates that the chunk length is invalid.
                      It is likely caused by a buggy client or server.
                    */
                    z_proxy_log(self->super.owner, HTTP_VIOLATION, 1, "Invalid chunk length; line='%s'", line_buf);
                    res = G_IO_STATUS_ERROR;
                    break;
                  }

                /*
                 * NOTE: the string  pointed by end is NUL terminated, thus
                 * we will not overflow our buffer
                 */

                while (*end == ' ')
                  end++;

                if (*end == ';')
                  {
                    /* ignore and strip chunk extensions */
                    *end = 0;
                  }

                if (*end)
                  {
                    /* hmm... invalid chunk length */
                    /*LOG
                      This message indicates that the chunk length is invalid.
                      It is likely caused by a buggy client or server.
                    */
                    z_proxy_log(self->super.owner, HTTP_VIOLATION, 1, "Invalid chunk length; line='%s'", line_buf);
                    res = G_IO_STATUS_ERROR;
                    break;
                  }

                if ((owner->max_chunk_length && chunk_length > owner->max_chunk_length) ||
                    (chunk_length & 0x80000000))
                  {
                    /*LOG
                      This message indicates that the length of the chunk is larger than allowed
                      or is a negative number. Check the 'max_chunk_length' attribute.
                    */
                    z_proxy_log(self->super.owner, HTTP_POLICY, 2, "Chunk too large; length='%d', max_chunk_length='%d'", chunk_length, owner->max_chunk_length);
                    res = G_IO_STATUS_ERROR;
                    break;
                  }

                if (owner->max_body_length && (guint) self->src_whole_length + chunk_length > owner->max_body_length)
                  {
                    /* this chunk would be over body_length limit */

                    chunk_length = owner->max_body_length - self->src_whole_length;
                    self->src_chunk_left = chunk_length;
                    self->force_nonpersistent_mode = TRUE;
                    self->src_chunk_truncated = TRUE;
                  }

                self->src_chunk_left = chunk_length;
                self->src_last_chunk = chunk_length == 0;
                self->src_read_state = HTTP_SR_READ_CHUNK;
                z_stream_line_set_poll_partial(stream, TRUE);
                /* fall through */
              }
            else
              break;
          }
        case HTTP_SR_READ_CHUNK:

          if (!self->src_last_chunk)
            {
              res = z_stream_read(stream, buf, MIN(self->src_chunk_left, count), &br, &local_error);

              if (res == G_IO_STATUS_NORMAL)
                {
                  self->src_whole_length += br;
                  self->src_chunk_left -= br;
                  *bytes_read = br;
                }
              else if (res == G_IO_STATUS_EOF)
                {
                  /* unexpected eof */
                  /*LOG
                    This message indicates that Zorp unexpectedly got EOF during
                    chunk encoded data transfer. It is likely a caused by a buggy client
                    or server.
                  */
                  z_proxy_log(self->super.owner, HTTP_VIOLATION, 1, "Unexpected EOF while dechunking stream;");
                  res = G_IO_STATUS_ERROR;
                  break;
                }

              if (self->src_chunk_left == 0)
                {
                  self->src_read_state = HTTP_SR_READ_FOOTER;
                  z_stream_line_set_poll_partial(stream, FALSE);
                }

              break;
            }
          else
            {
              self->src_read_state = HTTP_SR_READ_FOOTER;
              z_stream_line_set_poll_partial(stream, FALSE);
              /* fallthrough */
            }

        case HTTP_SR_READ_FOOTER:
          {
            gchar *line;
            gsize line_length;

            if (!self->src_chunk_truncated)
              {
                res = z_stream_line_get(stream, &line, &line_length, NULL);
              }
            else
              {
                res = G_IO_STATUS_EOF;
              }

            if (res == G_IO_STATUS_NORMAL)
              {
                if (line_length != 0)
                  {
                    /*LOG
                      This message indicates that the chunk footer contains data.
                      It is likely caused by a buggy client or server.
                    */
                    z_proxy_log(self->super.owner, HTTP_VIOLATION, 1, "Chunk footer is not an empty line;");
                    res = G_IO_STATUS_ERROR;
                    break;
                  }

                if (self->src_last_chunk)
                  {
                    res = G_IO_STATUS_EOF;
                  }
                else
                  {
                    self->src_read_state = HTTP_SR_READ_CHUNK_LENGTH;
                    z_stream_line_set_poll_partial(stream, TRUE);
                    /* come back later */
                    res = G_IO_STATUS_AGAIN;
                  }

                break;
              }

            break;
          }
        }
    }
  else
    {
      /* copy until EOF or self->content_length bytes */
      if (self->content_length == HTTP_LENGTH_NONE)
        {
          res = G_IO_STATUS_EOF;
        }
      else
        {
          if (self->src_read_state == HTTP_SR_INITIAL)
            {
              self->src_whole_length = 0;
              self->src_read_state = HTTP_SR_READ_ENTITY;
            }

          if (self->content_length == HTTP_LENGTH_UNKNOWN)
            {
              if (owner->max_body_length && self->src_whole_length + count >= owner->max_body_length)
                {
                  count = owner->max_body_length - self->src_whole_length;
                }

              if (count == 0)
                {
                  self->force_nonpersistent_mode = TRUE;
                  res = G_IO_STATUS_EOF;
                }
              else
                res = z_stream_read(stream, buf, count, &br, &local_error);
            }
          else
            {
              /* for specified content-length, max_body_length has already
                 been processed, and content_length contains the number of
                 bytes to be transferred, but maximum max_body_length */
              if (self->content_length >= 0 && (guint64) self->content_length == self->src_whole_length)
                res = G_IO_STATUS_EOF;
              else
                res = z_stream_read(stream, buf, MIN(count, self->content_length - self->src_whole_length), &br, &local_error);
            }

          if (res == G_IO_STATUS_NORMAL)
            {
              self->src_whole_length += br;
              *bytes_read = br;
            }
        }
    }

  if (local_error)
    g_propagate_error(err, local_error);

  return res;
}

static GIOStatus
http_transfer_src_shutdown(ZTransfer2 * /* self */, ZStream * /* s */, GError ** /* err */)
{
  /* do nothing */
  return G_IO_STATUS_NORMAL;
}

static GIOStatus
http_transfer_dst_write_preamble(HttpTransfer *self, ZStream *stream, GError **err)
{
  HttpProxy *owner = Z_CAST(self->super.owner, HttpProxy);
  GIOStatus res = G_IO_STATUS_NORMAL;
  GError *local_error = NULL;
  gsize bw;

  http_log_headers(owner, self->transfer_from, "postfilter");
  res = z_stream_write(stream, &self->preamble->str[self->preamble_ofs], self->preamble->len - self->preamble_ofs, &bw, &local_error);

  if (res == G_IO_STATUS_NORMAL)
    {
      self->preamble_ofs += bw;

      if (self->preamble_ofs != self->preamble->len)
        {
          res = G_IO_STATUS_AGAIN;
        }
    }
  else if (self->src_read_state == HTTP_SR_INITIAL && g_error_matches(local_error, G_IO_CHANNEL_ERROR, G_IO_CHANNEL_ERROR_PIPE))
    {
      /* we can only reattempt a connection if nothing was read from the
       * client, that's the reason behind src_read_state == INITIAL above */

      /* FIXME: this is a hack. A better solution would be to propagate
       * GError to the proxy so this case can be handled there cleanly */

      ((HttpProxy *) self->super.owner)->reattempt_connection = TRUE;
    }

  if (local_error)
    g_propagate_error(err, local_error);

  return res;
}

/* NOTE: this function assumes that ZTransfer2 does not change the buffer
 * between successive calls when an G_IO_STATUS_AGAIN is returned. This is
 * required for performance reasons, as otherwise this function would have
 * to copy the buffer contents to a private buffer and return
 * G_IO_STATUS_AGAIN while it is being flushed.*/
static GIOStatus
http_transfer_dst_write(ZTransfer2 *s, ZStream *stream, const gchar *buf, gsize count, gsize *bytes_written, GError **err)
{
  HttpTransfer *self = Z_CAST(s, HttpTransfer);
  HttpProxy *owner = Z_CAST(self->super.owner, HttpProxy);
  GError *local_error = NULL;
  gsize bw;
  GIOStatus res = G_IO_STATUS_NORMAL;

  *bytes_written = 0;

  if (self->dst_write_state == HTTP_DW_INITIAL)
    self->dst_write_state = HTTP_DW_POP_HEADERS;

  if (self->dst_write_state >= HTTP_DW_POP_HEADERS && self->dst_write_state <= HTTP_DW_POP_HEADERS_MAX)
    {
      if (self->push_mime_headers)
        {
          gsize i;

          for (i = 0; i < count; i++)
            {
              switch (self->dst_write_state)
                {
                case HTTP_DW_POP_HEADERS:

                  switch (buf[i])
                    {
                    case '\r':
                      self->dst_write_state = HTTP_DW_POP_HEADERS_CR;
                      break;
                    case '\n':
                      self->dst_write_state = HTTP_DW_POP_HEADERS_LF;
                      break;
                    default:
                      break;
                    }

                  break;
                case HTTP_DW_POP_HEADERS_CR:

                  switch (buf[i])
                    {
                    case '\n':
                      self->dst_write_state = HTTP_DW_POP_HEADERS_LF;
                      break;
                    default:
                      self->dst_write_state = HTTP_DW_POP_HEADERS;
                      break;
                    }

                  break;
                }

              if (self->dst_write_state == HTTP_DW_POP_HEADERS_LF)
                {
                  /* new line found */
                  self->dst_eol_count++;

                  if (self->dst_eol_count == 2)
                    {
                      /* end of headers */
                      i++;
                      self->dst_write_state = HTTP_DW_WRITE_PREAMBLE;
                      break;
                    }
                  else
                    {
                      self->dst_write_state = HTTP_DW_POP_HEADERS;
                    }
                }
              else if (self->dst_write_state == HTTP_DW_POP_HEADERS)
                self->dst_eol_count = 0;

            }

          /* at this point we have either processed the whole buffer (i ==
           * count), or a small chunk of data is at the tail of our headers
           * to be ignored. In either case return G_IO_STATUS_NORMAL and the
           * number of processed bytes. */

          *bytes_written = i;
          self->stacked_preamble_read_bytes += i;

          return G_IO_STATUS_NORMAL;
        }
      else
        self->dst_write_state = HTTP_DW_WRITE_PREAMBLE;
    }

  if (self->dst_write_state == HTTP_DW_WRITE_PREAMBLE)
    {
      gboolean reformat_preamble = FALSE;

      if (self->super.child_content_length_hint_set)
        {
          /* we stay in persistent mode if possible */
          /*LOG
            This message reports that the stacked proxy sent a content-length hint on how much
            data will be sent and the http proxy is using it to set the Content-Length header.
          */
          z_proxy_log(owner, HTTP_DEBUG, 6, "Stacked proxy sent a content-length hint, using it; expected_content_length='%" G_GINT64_FORMAT "'", self->super.child_content_length_hint);
          g_string_sprintf(self->content_length_hdr->value, "%" G_GINT64_FORMAT, self->super.child_content_length_hint - self->stacked_preamble_read_bytes);
          self->content_length_hdr->present = TRUE;
          self->transfer_encoding_hdr->present = FALSE;
          self->dst_chunked = FALSE;
          reformat_preamble = TRUE;
        }
      else if (self->persistent_with_cl_hint_only)
        {
          /* when we require content-length hint to remain persistent and we
             have no content-length hint, we force ourselves to
             non-persistent mode */
          self->force_nonpersistent_mode = TRUE;
          self->content_length_hdr->present = FALSE;
          reformat_preamble = TRUE;
        }

      if (self->force_nonpersistent_mode && owner->connection_mode == HTTP_CONNECTION_KEEPALIVE)
        {
          owner->connection_mode = owner->server_connection_mode = HTTP_CONNECTION_CLOSE;

          /* we must change our connection header right here as our core has
           * already made its decision, but we must override that */

          if (owner->connection_hdr)
            {
              g_string_assign(owner->connection_hdr->value, "close");
              reformat_preamble = TRUE;
            }
        }

      if (reformat_preamble)
        self->format_preamble_func(owner, FALSE, self->preamble);

      /* take care about the preamble (request/response itself) */
      res = http_transfer_dst_write_preamble(self, stream, &local_error);

      if (res != G_IO_STATUS_NORMAL)
        {
          goto propagate_exit;
        }

      self->dst_write_state = HTTP_DW_WRITE_INITIAL;
    }

  /* ok, now take care about the data, and possibly enchunk it on the way */
  if (self->dst_chunked)
    {
      switch (self->dst_write_state)
        {
        case HTTP_DW_WRITE_INITIAL:
          self->dst_write_state = HTTP_DW_FORMAT_CHUNK_LENGTH;
          self->dst_whole_length = 0;
          /* fallthrough */
        case HTTP_DW_FORMAT_CHUNK_LENGTH:
          {
            if (count == 0)
              {
                res = G_IO_STATUS_NORMAL;
                break;
              }

            self->dst_chunk_length = 0;
            self->dst_chunk_left = count;
            g_snprintf(self->dst_chunk_length_buf, sizeof(self->dst_chunk_length_buf), "%" G_GINT64_MODIFIER "x\r\n", self->dst_chunk_left);
            self->dst_chunk_length_ofs = 0;
            self->dst_chunk_length_end = strlen(self->dst_chunk_length_buf);
            self->dst_write_state = HTTP_DW_WRITE_CHUNK_LENGTH;
          }
        case HTTP_DW_WRITE_CHUNK_LENGTH:
          {
            res = z_stream_write(stream, &self->dst_chunk_length_buf[self->dst_chunk_length_ofs], self->dst_chunk_length_end - self->dst_chunk_length_ofs, &bw, &local_error);

            if (res == G_IO_STATUS_NORMAL)
              {
                self->dst_chunk_length_ofs += bw;

                if (self->dst_chunk_length_ofs == self->dst_chunk_length_end)
                  self->dst_write_state = HTTP_DW_WRITE_CHUNK;
              }
            else
              {
                break;
              }

            /* fallthrough */
          }
        case HTTP_DW_WRITE_CHUNK:
          {
            /* NOTE: here lies the assumptions that ZTransfer2 neither
             * changes the buffer nor count when G_IO_STATUS_AGAIN
             * is returned */
            res = z_stream_write(stream, &buf[count - self->dst_chunk_left], self->dst_chunk_left, &bw, &local_error);

            if (res == G_IO_STATUS_NORMAL)
              {
                self->dst_chunk_length += bw;
                self->dst_chunk_left -= bw;

                if (self->dst_chunk_left == 0)
                  {
                    self->dst_write_state = HTTP_DW_WRITE_FOOTER;
                  }
                else
                  {
                    *bytes_written = 0;
                    res = G_IO_STATUS_AGAIN;
                    break;
                  }
              }
            else
              {
                break;
              }
          }
          self->dst_footer_ofs = 0;
        case HTTP_DW_WRITE_FOOTER:
          {
            gchar line_buf[] = "\r\n";
            g_assert(self->dst_footer_ofs < 2);
            res = z_stream_write(stream, &line_buf[self->dst_footer_ofs], 2 - self->dst_footer_ofs, &bw, &local_error);

            if (res == G_IO_STATUS_NORMAL)
              {
                self->dst_footer_ofs += bw;

                if (self->dst_footer_ofs == 2)
                  {
                    self->dst_whole_length += self->dst_chunk_length;
                    *bytes_written = self->dst_chunk_length;
                    self->dst_write_state = HTTP_DW_FORMAT_CHUNK_LENGTH;
                  }
              }
            else
              {
                break;
              }
          }
        }
    }
  else
    {
      res = z_stream_write(stream, buf, count, bytes_written, &local_error);
    }

 propagate_exit:

  if (local_error)
    g_propagate_error(err, local_error);

  return res;
}

static GIOStatus
http_transfer_dst_shutdown(ZTransfer2 *s, ZStream *stream, GError **err)
{
  HttpTransfer *self = Z_CAST(s, HttpTransfer);
  GIOStatus res = G_IO_STATUS_NORMAL;
  GError *local_error = NULL;
  gsize bw;
  gboolean delay_transfer;
  gboolean stack_error_or_reject;
  gboolean has_data_part;

  /* delay preamble and closing chunk if we want to write an error page, we can do this if:
   *
   * - we are not suppressing data entity (e.g. HEAD)
   * - we are expecting data (e.g. response) or the request contains data (e.g. POST)
   * - we are using MIME stacking
   */
  stack_error_or_reject = !!(s->status & (ZT2S_FAILED + ZT2S_ABORTED)) || (self->super.stack_decision != ZV_ACCEPT);
  has_data_part = self->expect_data || self->content_length != HTTP_LENGTH_NONE;

  delay_transfer = (self->dst_write_state == HTTP_DW_INITIAL && stack_error_or_reject) &&
                   !self->suppress_data && (has_data_part || self->stack_type == HTTP_STK_MIME);

  if (!delay_transfer)
    {
      if (self->dst_write_state == HTTP_DW_INITIAL || self->dst_write_state == HTTP_DW_WRITE_PREAMBLE)
        {
          self->dst_write_state = HTTP_DW_WRITE_PREAMBLE;
          res = http_transfer_dst_write_preamble(self, stream, &local_error);
        }

      if (res == G_IO_STATUS_NORMAL)
        {
          if (self->content_length != HTTP_LENGTH_NONE && self->dst_chunked)
            {
              gchar line_buf[] = "0\r\n\r\n";

              res = z_stream_write(stream, line_buf, 5, &bw, &local_error);

              if (bw != 5)
                {
                  res = G_IO_STATUS_ERROR;
                }
            }
        }
    }

  if (local_error)
    g_propagate_error(err, local_error);

  return res;
}

static gboolean
http_transfer_stack_proxy(ZTransfer2 *s, ZStackedProxy **stacked)
{
  HttpTransfer *self = Z_CAST(s, HttpTransfer);
  ZPolicyObj *proxy_stack_tuple = NULL, *stack_object = NULL;
  ZEndpoint side = self->transfer_from;
  gint stack_type = HTTP_STK_NONE;
  gboolean called;
  gboolean success = FALSE;
  HttpProxy *owner = Z_CAST(s->owner, HttpProxy);
  HttpHeaders *headers = &owner->headers[self->transfer_from];
  HttpHeader *hdr;

  /* query python for a stacked proxy */

  if (self->suppress_data || (self->transfer_type != HTTP_TRANSFER_NORMAL && self->transfer_type != HTTP_TRANSFER_TO_BLOB))
    {
      *stacked = NULL;
      return TRUE;
    }

  z_policy_lock(self->super.owner->thread);

  proxy_stack_tuple = z_policy_call(self->super.owner->handler, "requestStack", z_policy_var_build("(i)", side), &called, self->super.owner->session_id);

  if (called && !proxy_stack_tuple)
    {
      goto unref_unlock;
    }

  if (!z_policy_var_parse(proxy_stack_tuple, "(iO)", &stack_type, &stack_object))
    {
      /*LOG
        This message indicates that the request_stack or response_stack hash
        contains an invalid stacking tuple. It should contain a (stack_type, proxy_class) tuple.
        Check your Zorp configuration.
      */
      z_proxy_log(self->super.owner, HTTP_POLICY, 3, "Invalid stacking tuple returned by policy; side='%d'", side);
      z_proxy_report_invalid_policy(self->super.owner);
      goto unref_unlock;
    }

  self->stack_type = stack_type;
  if (stack_type < HTTP_STK_NONE || stack_type > HTTP_STK_MIME)
    {
      /*LOG
        This message indicates that the request_stack or response_stack hash
        contains an invalid stacking type. Check your Zorp configuration.
      */
      z_proxy_log(self->super.owner, HTTP_POLICY, 3, "Invalid stacking type; type='%d'", stack_type);
      stack_type = HTTP_STK_NONE;
      z_proxy_report_invalid_policy(self->super.owner);
      goto unref_unlock;
    }

  success = TRUE;

  if (stack_type == HTTP_STK_MIME)
    self->push_mime_headers = TRUE;

  /* we don't stack anything, if
     1) we suppress data
     2) data is not indicated by either the presence of some header fields nor do we expect data
  */
  if (!(self->expect_data || self->push_mime_headers || http_lookup_header(headers, "Transfer-Encoding", &hdr) || http_lookup_header(headers, "Content-Length", &hdr)))
    {
      *stacked = NULL;
      goto unref_unlock;
    }

  if (stack_type != HTTP_STK_NONE)
    success = z_proxy_stack_object(s->owner, stack_object, stacked, NULL);

 unref_unlock:
  if (!success)
    z_proxy_report_policy_abort(self->super.owner);

  z_policy_var_unref(proxy_stack_tuple);
  z_policy_unlock(self->super.owner->thread);
  return success;
}

static gboolean
http_transfer_setup(ZTransfer2 *s)
{
  HttpTransfer *self = Z_CAST(s, HttpTransfer);
  HttpProxy *owner = Z_CAST(s->owner, HttpProxy);
  HttpHeaders *headers = &owner->headers[self->transfer_from];
  gboolean chunked; /* original entity is chunked */

  z_proxy_enter(owner);

  if (!self->suppress_data)
    {
      if (http_lookup_header(headers, "Transfer-Encoding", &self->transfer_encoding_hdr))
        {
          chunked = self->transfer_encoding_hdr->present && strcasecmp(self->transfer_encoding_hdr->value->str, "chunked") == 0;
        }
      else
        {
          chunked = FALSE;
          self->transfer_encoding_hdr = http_add_header(headers, "Transfer-Encoding", 17, "", 0);
        }

      if (http_lookup_header(headers, "Content-Length", &self->content_length_hdr))
        {
          gchar *end;

          if (self->content_length_hdr->present)
            {
              self->content_length = strtoll(self->content_length_hdr->value->str, &end, 10);

              if (self->content_length < 0)
                {
                  self->content_length = self->expect_data ? HTTP_LENGTH_UNKNOWN : HTTP_LENGTH_NONE;
                }

              if ((guint) (end - self->content_length_hdr->value->str) != self->content_length_hdr->value->len)
                {
                  /* content-length not a number */
                  /*LOG
                    This message indicates that the Content-Length headers value
                    is not a valid number. It is likely caused by a buggy client or server.
                  */
                  z_proxy_log(owner, HTTP_VIOLATION, 1, "The header 'Content-Length' was present, but is not a number; content_length='%s'", self->content_length_hdr->value->str);
                  z_proxy_return(owner, FALSE);
                }
            }
          else
            {
              self->content_length = self->expect_data || chunked ? HTTP_LENGTH_UNKNOWN : HTTP_LENGTH_NONE;
            }
        }
      else
        {
          self->content_length_hdr = http_add_header(headers, "Content-Length", 14, "", 0);
          self->content_length = self->expect_data || chunked ? HTTP_LENGTH_UNKNOWN : HTTP_LENGTH_NONE;
        }

      self->transfer_encoding_hdr->present = FALSE;
      self->content_length_hdr->present = FALSE;

      if (self->push_mime_headers)
        {
          self->format_preamble_func((HttpProxy *) self->super.owner, TRUE, self->stacked_preamble);
          g_string_append(self->stacked_preamble, "\r\n");
          z_transfer2_set_proxy_out(s, TRUE);
        }

      self->src_chunked = FALSE;
      self->dst_chunked = FALSE;

      if (self->transfer_type != HTTP_TRANSFER_NORMAL)
        {
          if (self->transfer_type == HTTP_TRANSFER_TO_BLOB)
            {
              /* we are transferring into the request_data blob, we don't want chunking in the destination  */
              if (chunked)
                self->src_chunked = TRUE;
            }
          else
            {
              /* we are transferring from the request_data blob to the
               * server, we don't need chunking as we know the exact size of
               * the data */
              self->content_length = owner->request_data->size;
              g_string_sprintf(self->content_length_hdr->value, "%" G_GINT64_FORMAT, owner->request_data->size);
              self->content_length_hdr->present = TRUE;
            }
        }
      else if (self->super.stacked)
        {
          if (chunked && (self->transfer_to == EP_SERVER || (owner->proto_version[self->transfer_to] > 0x0100)))
            {
              self->src_chunked = TRUE;
              self->dst_chunked = TRUE;
            }
          else if (self->transfer_to == EP_CLIENT && self->expect_data)
            {
              if (owner->max_body_length && self->content_length != HTTP_LENGTH_UNKNOWN &&
                  (guint) self->content_length > owner->max_body_length)
                {
                  self->content_length = owner->max_body_length;
                  self->force_nonpersistent_mode = TRUE;
                }

              if (owner->proto_version[self->transfer_to] > 0x0100 && owner->proto_version[self->transfer_from] >= 0x0100)
                {
                  /* The original entity is not chunked, we enchunk it to
                   * avoid bufferring and keeping the connection open. For
                   * this we need HTTP/1.1 on the client and at least
                   * HTTP/1.0 on the server to be able to transmit headers.
                   *
                   * Transfer-Encoding is added, Content-Length is removed,
                   * version is bumped to HTTP/1.1
                   */
                  self->src_chunked = FALSE;
                  self->dst_chunked = TRUE;
                  owner->proto_version[self->transfer_from] = 0x0101;
                }
              else
                {
                  /* chunking is not supported, the entity's end is indicated by an EOF
                   * neither Content-Length nor Transfer-Encoding is added, persisency
                   * is retained only if a content-length hint is received.
                   */
                  if (owner->proto_version[self->transfer_from] >= 0x0100)
                    self->persistent_with_cl_hint_only = TRUE;
                  else
                    self->force_nonpersistent_mode = TRUE;

                  self->src_chunked = self->dst_chunked = FALSE;

                }
            }
          else
            {
              /* sending to the server, the client sends it unchunked, but we add chunking to avoid buffering */

              /* NOTE: some servers do not accept chunked data for methods other
               * than POST, but those do not usually have entity either */

              if (self->content_length > 0 || (self->expect_data && self->content_length == HTTP_LENGTH_UNKNOWN))
                {
                  self->src_chunked = FALSE;
                  self->dst_chunked = TRUE;

                  /* NOTE: we only change server protocol version as
                   * otherwise some defaults might change (e.g. connection
                   * mode) */
                  if (self->transfer_from == EP_SERVER)
                    owner->proto_version[self->transfer_from] = 0x0101;
                }
              else if (self->content_length != 0)
                {
                  self->content_length = HTTP_LENGTH_NONE;
                }
            }

        }
      else
        {
          /* there's no stacked proxy */
          if (chunked)
            {
              self->src_chunked = TRUE;
              self->dst_chunked = TRUE;
            }
          else if (self->content_length >= 0)
            {
              /* entity with specified length */
              if (owner->max_body_length && self->content_length != HTTP_LENGTH_UNKNOWN &&
                  (guint) self->content_length > owner->max_body_length)
                {
                  self->content_length = owner->max_body_length;
                  self->force_nonpersistent_mode = TRUE;
                }
            }
          else if (self->content_length == HTTP_LENGTH_UNKNOWN)
            {
              /* EOF terminated entity, can only happen for server->client direction */

              g_assert(self->transfer_from == EP_SERVER);

              if (owner->keep_persistent && owner->proto_version[self->transfer_to] > 0x0100 && owner->proto_version[self->transfer_from] >= 0x0100)
                {
                  /* client supports chunking, server can transfer headers, convert it */
                  self->src_chunked = FALSE;
                  self->dst_chunked = TRUE;
                  owner->server_connection_mode = HTTP_CONNECTION_CLOSE;
                  owner->proto_version[self->transfer_from] = 0x0101;
                }
              else if (owner->connection_mode == HTTP_CONNECTION_KEEPALIVE)
                {
                  /* client does not support chunking or we are not in keep_persistent mode,
                   * no way to keep it persistent */
                  g_string_assign(owner->connection_hdr->value, "close");
                  owner->connection_hdr->present = TRUE;
                  owner->connection_mode = HTTP_CONNECTION_CLOSE;
                }
            }
        }

      /* NOTE: the headers here are not final, if a content-length hint is received, the
       * content-length header will be added and transfer-encoding removed */
      if (self->dst_chunked)
        {
          self->content_length_hdr->present = FALSE;
          self->transfer_encoding_hdr->present = TRUE;
          g_string_assign(self->transfer_encoding_hdr->value, "chunked");
        }
      else if (self->content_length >= 0)
        {
          self->transfer_encoding_hdr->present = FALSE;
          g_string_sprintf(self->content_length_hdr->value, "%" G_GINT64_FORMAT, self->content_length);
          self->content_length_hdr->present = TRUE;

        }
    }
  else
    {
      self->content_length = HTTP_LENGTH_NONE;
    }

  switch (self->content_length)
    {
    case HTTP_LENGTH_UNKNOWN:
      self->super.our_content_length_hint_set = FALSE;
      break;

    case HTTP_LENGTH_NONE:
    default:
      self->super.our_content_length_hint_set = TRUE;
      self->super.our_content_length_hint = self->content_length;

      if (self->super.our_content_length_hint > 0)
        self->super.our_content_length_hint += self->stacked_preamble->len;

      break;
    }

  self->format_preamble_func((HttpProxy *) self->super.owner, FALSE, self->preamble);

  z_proxy_return(owner, TRUE);
}

gboolean
http_transfer_has_content(HttpTransfer *self)
{
  return (self->content_length != HTTP_LENGTH_NONE && self->content_length != 0) || self->push_mime_headers;
}

static ZTransfer2Result
http_transfer_run(ZTransfer2 *s)
{
  HttpTransfer *self = Z_CAST(s, HttpTransfer);
  GError *local_error = NULL;

  if (http_transfer_has_content(self))
    {
      return Z_SUPER(self, ZTransfer2)->run(&self->super);
    }

  if (z_transfer2_src_shutdown(s, z_transfer2_get_stream(s, ZT2E_SOURCE), &local_error) != G_IO_STATUS_NORMAL)
    {
      g_error_free(local_error);
      return ZT2_RESULT_ABORTED;
    }

  if (z_transfer2_dst_shutdown(s, z_transfer2_get_stream(s, ZT2E_DEST), &local_error) != G_IO_STATUS_NORMAL)
    {
      g_error_free(local_error);
      return ZT2_RESULT_ABORTED;
    }

  return ZT2_RESULT_FINISHED;
}

static HttpTransfer *
http_transfer_new(HttpProxy *owner,
                  gint transfer_type,
                  ZEndpoint from, ZStream *from_stream,
                  ZEndpoint to, ZStream *to_stream,
                  gboolean expect_data, gboolean suppress_data,
                  HttpTransferPreambleFunc format_preamble)
{
  HttpTransfer *self;

  z_proxy_enter(owner);
  self = Z_CAST(z_transfer2_new(Z_CLASS(HttpTransfer),
                                &owner->super, owner->poll,
                                from_stream, to_stream,
                                owner->buffer_size,
                                owner->timeout,
                                0),
                HttpTransfer);
  self->transfer_from = from;
  self->transfer_to = to;
  self->transfer_type = transfer_type;
  self->format_preamble_func = format_preamble;
  self->preamble = g_string_sized_new(0);
  self->stacked_preamble = g_string_sized_new(0);
  self->stack_type = HTTP_STK_NONE;
  self->force_nonpersistent_mode = FALSE;
  self->expect_data = expect_data;
  self->suppress_data = suppress_data;
  z_proxy_return(owner, self);
}

static void
http_transfer_free_method(ZObject *s)
{
  HttpTransfer *self = Z_CAST(s, HttpTransfer);

  g_string_free(self->preamble, TRUE);
  g_string_free(self->stacked_preamble, TRUE);
  z_transfer2_free_method(s);
}

ZTransfer2Funcs http_transfer_funcs =
  {
    {
      Z_FUNCS_COUNT(ZTransfer2),
      http_transfer_free_method,
    },
    /* .src_read = */ http_transfer_src_read,
    /* .dst_write = */ http_transfer_dst_write,
    /* .src_shutdown = */ http_transfer_src_shutdown,
    /* .dst_shutdown = */ http_transfer_dst_shutdown,
    /* .stack_proxy = */ http_transfer_stack_proxy,
    /* .setup = */ http_transfer_setup,
    /* .run = */ http_transfer_run,
    /* .progress = */ NULL
  };

Z_CLASS_DEF(HttpTransfer, ZTransfer2, http_transfer_funcs);

gboolean
http_data_transfer(HttpProxy *self, gint transfer_type, ZEndpoint from, ZStream *from_stream, ZEndpoint to, ZStream *to_stream, gboolean expect_data, gboolean suppress_data, HttpTransferPreambleFunc format_preamble)
{
  HttpTransfer *t;
  gboolean res = TRUE;
  ZTransfer2Result tr;

  /*
   * tell transfer not to send the preamble when data is expected (e.g.
   * response & response body). This makes it possible to display an error
   * page instead
   */

  if (transfer_type != HTTP_TRANSFER_TO_BLOB)
    {
      guint one = 1;
      guint fd = z_stream_get_fd(to_stream);
#if defined(__gnu_linux__)
#ifdef HAVE_DECL_TCP_CORK
      setsockopt(fd, IPPROTO_TCP, TCP_CORK, &one, sizeof(one));
#endif
#elif defined(__FreeBSD__)
#ifdef HAVE_DECL_TCP_NOPUSH
      setsockopt(fd, IPPROTO_TCP, TCP_NOPUSH, &one, sizeof(one));
#endif
#endif
    }

  t = http_transfer_new(self, transfer_type, from, from_stream, to, to_stream, expect_data, suppress_data, format_preamble);

  if (!t || !z_transfer2_start(&t->super))
    {
      /*LOG
        This message indicates that the processed request was invalid, and
        the data transfer failed.
      */
      z_proxy_log(self, HTTP_ERROR, 2, "Invalid request, data transfer failed;");
      g_string_assign(self->error_info, "Invalid request, data transfer failed;");

      if (t)
        z_object_unref(&t->super.super);

      return FALSE;
    }

  do
    {
      tr = z_transfer2_run(&t->super);
    }
  while (tr == ZT2_RESULT_SUSPENDED);

  if (transfer_type != HTTP_TRANSFER_TO_BLOB)
    {
      guint zero = 0;
      guint fd = z_stream_get_fd(to_stream);
#if defined(__gnu_linux__)
#ifdef HAVE_DECL_TCP_CORK
      setsockopt(fd, IPPROTO_TCP, TCP_CORK, &zero, sizeof(zero));
#endif
#elif defined(__FreeBSD__)
#ifdef HAVE_DECL_TCP_NOPUSH
      setsockopt(fd, IPPROTO_TCP, TCP_NOPUSH, &zero, sizeof(zero));
#endif
#endif
    }

  if (tr == ZT2_RESULT_FAILED)
    z_transfer2_rollback(&t->super);

  res = (tr != ZT2_RESULT_FAILED) && (tr != ZT2_RESULT_ABORTED);

  if (!res)
    {
      /* transfer was not successful */
      /*LOG
        This message reports that the data transfer failed.
      */
      z_proxy_log(self, HTTP_ERROR, 2, "Data transfer failed;");
      g_string_assign(self->error_info, "Data transfer failed.");
    }

  /* transfer was successful, check if the stacked proxy told us something important */
  if (t->super.stack_decision != ZV_ACCEPT)
    {
      /*LOG
        This message indicates that the stacked proxy returned the
        specified verdict about the content. Check the stacked proxy log
        for further information.
      */
      z_proxy_log(self, HTTP_ERROR, 2, "Stacked proxy decision; verdict='%d', info='%s'", t->super.stack_decision, t->super.stack_info->str);

      if (t->super.stack_info->len)
        g_string_assign(self->error_info, t->super.stack_info->str);
      else
        g_string_sprintf(self->error_info, "Stacked proxy did not accept this content (%d).", t->super.stack_decision);
    }
  else if (res)
    {
      /*LOG
        This message indicates that the stacked proxy accepted the
        content.
      */
      z_proxy_log(self, HTTP_DEBUG, 6, "Stacked proxy accepted data;");
    }

  if (t->dst_write_state == HTTP_DW_INITIAL)
    {
      /* our write state is HTTP_DW_INITIAL, this means that we never transmitted:
       *  * a request/response line
       *  * headers
       *  * data body
       * This means that we need to return with some kind of error code to ensure
       * that something gets back to the client */

      if (t->super.stack_decision != ZV_ACCEPT)
        self->error_code = HTTP_MSG_BAD_CONTENT;
      else
        self->error_code = HTTP_MSG_IO_ERROR;

      res = FALSE;
    }
  else if (!res)
    {
      /* unable to write error page */
      self->error_code = 0;
    }

  z_object_unref(&t->super.super);
  return res;
}
