Commit message: Fix CRLF injection in Wget host part

* src/url.c (url_parse): Reject control characters in host part of URL

Reported-by: Orange Tsai
CVE: CVE-2017-6508
CWE: CWE-93
Vulnerability description: CRLF injection vulnerability in the url_parse function in url.c in Wget through 1.19.1 allows remote attackers to inject arbitrary HTTP headers via CRLF sequences in the host subcomponent of a URL.

Project: wget

convert_fname (char *fname)
{
  char *converted_fname = fname;
  const char *from_encoding = opt.encoding_remote;
  const char *to_encoding = opt.locale;
  iconv_t cd;
  size_t len, done, inlen, outlen;
  char *s;
  const char *orig_fname = fname;

  /* Defaults for remote and local encodings.  */
  if (!from_encoding)
    from_encoding = "UTF-8";
  if (!to_encoding)
    to_encoding = nl_langinfo (CODESET);

  cd = iconv_open (to_encoding, from_encoding);
  if (cd == (iconv_t)(-1))
    logprintf (LOG_VERBOSE, _("Conversion from %s to %s isn't supported\n"),
	       quote (from_encoding), quote (to_encoding));
  else
    {
      inlen = strlen (fname);
      len = outlen = inlen * 2;
      converted_fname = s = xmalloc (outlen + 1);
      done = 0;

      for (;;)
	{
	  if (iconv (cd, (ICONV_CONST char **) &fname, &inlen, &s, &outlen) != (size_t)(-1)
	      && iconv (cd, NULL, NULL, &s, &outlen) != (size_t)(-1))
	    {
	      *(converted_fname + len - outlen - done) = '\0';
	      iconv_close(cd);
	      DEBUGP (("Converted file name '%s' (%s) -> '%s' (%s)\n",
		       orig_fname, from_encoding, converted_fname, to_encoding));
	      xfree (orig_fname);
	      return converted_fname;
	    }

	  /* Incomplete or invalid multibyte sequence */
	  if (errno == EINVAL || errno == EILSEQ)
	    {
	      logprintf (LOG_VERBOSE,
			 _("Incomplete or invalid multibyte sequence encountered\n"));
	      xfree (converted_fname);
	      converted_fname = (char *)orig_fname;
	      break;
	    }
	  else if (errno == E2BIG) /* Output buffer full */
	    {
	      done = len;
	      len = outlen = done + inlen * 2;
	      converted_fname = xrealloc (converted_fname, outlen + 1);
	      s = converted_fname + done;
	    }
	  else /* Weird, we got an unspecified error */
	    {
	      logprintf (LOG_VERBOSE, _("Unhandled errno %d\n"), errno);
	      xfree (converted_fname);
	      converted_fname = (char *)orig_fname;
	      break;
	    }
	}
      DEBUGP (("Failed to convert file name '%s' (%s) -> '?' (%s)\n",
	       orig_fname, from_encoding, to_encoding));
    }

    iconv_close(cd);

  return converted_fname;
}

Project: wget

url_file_name (const struct url *u, char *replaced_filename)
{
  struct growable fnres;        /* stands for "file name result" */
  struct growable temp_fnres;

  const char *u_file;
  char *fname, *unique, *fname_len_check;
  const char *index_filename = "index.html"; /* The default index file is index.html */
  size_t max_length;

  fnres.base = NULL;
  fnres.size = 0;
  fnres.tail = 0;

  temp_fnres.base = NULL;
  temp_fnres.size = 0;
  temp_fnres.tail = 0;

  /* If an alternative index file was defined, change index_filename */
  if (opt.default_page)
    index_filename = opt.default_page;


  /* Start with the directory prefix, if specified. */
  if (opt.dir_prefix)
    append_string (opt.dir_prefix, &fnres);

  /* If "dirstruct" is turned on (typically the case with -r), add
     the host and port (unless those have been turned off) and
     directory structure.  */
  if (opt.dirstruct)
    {
      if (opt.protocol_directories)
        {
          if (fnres.tail)
            append_char ('/', &fnres);
          append_string (supported_schemes[u->scheme].name, &fnres);
        }
      if (opt.add_hostdir)
        {
          if (fnres.tail)
            append_char ('/', &fnres);
          if (0 != strcmp (u->host, ".."))
            append_string (u->host, &fnres);
          else
            /* Host name can come from the network; malicious DNS may
               allow ".." to be resolved, causing us to write to
               "../<file>".  Defang such host names.  */
            append_string ("%2E%2E", &fnres);
          if (u->port != scheme_default_port (u->scheme))
            {
              char portstr[24];
              number_to_string (portstr, u->port);
              append_char (FN_PORT_SEP, &fnres);
              append_string (portstr, &fnres);
            }
        }

      append_dir_structure (u, &fnres);
    }

  if (!replaced_filename)
    {
      /* Create the filename. */
      u_file = *u->file ? u->file : index_filename;

      /* Append "?query" to the file name, even if empty,
       * and create fname_len_check. */
      if (u->query)
        fname_len_check = concat_strings (u_file, FN_QUERY_SEP_STR, u->query, NULL);
      else
        fname_len_check = strdupdelim (u_file, u_file + strlen (u_file));

      /* convert before concat with local path */
      fname_len_check = convert_fname (fname_len_check);
    }
  else
    {
      u_file = replaced_filename;
      fname_len_check = strdupdelim (u_file, u_file + strlen (u_file));
    }

  append_uri_pathel (fname_len_check,
    fname_len_check + strlen (fname_len_check), false, &temp_fnres);

  /* Zero-terminate the temporary file name. */
  append_char ('\0', &temp_fnres);

  /* Check that the length of the file name is acceptable. */
#ifdef WINDOWS
  if (MAX_PATH > (fnres.tail + CHOMP_BUFFER + 2))
    {
      max_length = MAX_PATH - (fnres.tail + CHOMP_BUFFER + 2);
      /* FIXME: In Windows a filename is usually limited to 255 characters.
      To really be accurate you could call GetVolumeInformation() to get
      lpMaximumComponentLength
      */
      if (max_length > 255)
        {
          max_length = 255;
        }
    }
  else
    {
      max_length = 0;
    }
#else
  max_length = get_max_length (fnres.base, fnres.tail, _PC_NAME_MAX) - CHOMP_BUFFER;
#endif
  if (max_length > 0 && strlen (temp_fnres.base) > max_length)
    {
      logprintf (LOG_NOTQUIET, "The name is too long, %lu chars total.\n",
          (unsigned long) strlen (temp_fnres.base));
      logprintf (LOG_NOTQUIET, "Trying to shorten...\n");

      /* Shorten the file name. */
      temp_fnres.base[max_length] = '\0';

      logprintf (LOG_NOTQUIET, "New name is %s.\n", temp_fnres.base);
    }

  xfree (fname_len_check);

  /* The filename has already been 'cleaned' by append_uri_pathel() above.  So,
   * just append it. */
  if (fnres.tail)
    append_char ('/', &fnres);
  append_string (temp_fnres.base, &fnres);

  fname = fnres.base;

  /* Make a final check that the path length is acceptable? */
  /* TODO: check fnres.base for path length problem */

  xfree (temp_fnres.base);

  /* Check the cases in which the unique extensions are not used:
     1) Clobbering is turned off (-nc).
     2) Retrieval with regetting.
     3) Timestamping is used.
     4) Hierarchy is built.
     5) Backups are specified.

     The exception is the case when file does exist and is a
     directory (see `mkalldirs' for explanation).  */

  if (ALLOW_CLOBBER
      && !(file_exists_p (fname) && !file_non_directory_p (fname)))
    {
      unique = fname;
    }
  else
    {
      unique = unique_name (fname, true);
      if (unique != fname)
        xfree (fname);
    }

/* On VMS, alter the name as required. */
#ifdef __VMS
  {
    char *unique2;

    unique2 = ods_conform( unique);
    if (unique2 != unique)
      {
        xfree (unique);
        unique = unique2;
      }
  }
#endif /* def __VMS */

  return unique;
}

Project: wget

scheme_leading_string (enum url_scheme scheme)
{
  return supported_schemes[scheme].leading_string;
}

Project: wget

url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)
{
  struct url *u;
  const char *p;
  bool path_modified, host_modified;

  enum url_scheme scheme;
  const char *seps;

  const char *uname_b,     *uname_e;
  const char *host_b,      *host_e;
  const char *path_b,      *path_e;
  const char *params_b,    *params_e;
  const char *query_b,     *query_e;
  const char *fragment_b,  *fragment_e;

  int port;
  char *user = NULL, *passwd = NULL;

  const char *url_encoded = NULL;

  int error_code;

  scheme = url_scheme (url);
  if (scheme == SCHEME_INVALID)
    {
      if (url_has_scheme (url))
        error_code = PE_UNSUPPORTED_SCHEME;
      else
        error_code = PE_MISSING_SCHEME;
      goto error;
    }

  url_encoded = url;

  if (iri && iri->utf8_encode)
    {
      char *new_url = NULL;

      iri->utf8_encode = remote_to_utf8 (iri, iri->orig_url ? iri->orig_url : url, &new_url);
      if (!iri->utf8_encode)
        new_url = NULL;
      else
        {
          xfree (iri->orig_url);
          iri->orig_url = xstrdup (url);
          url_encoded = reencode_escapes (new_url);
          if (url_encoded != new_url)
            xfree (new_url);
          percent_encode = false;
        }
    }

  if (percent_encode)
    url_encoded = reencode_escapes (url);

  p = url_encoded;
  p += strlen (supported_schemes[scheme].leading_string);
  uname_b = p;
  p = url_skip_credentials (p);
  uname_e = p;

  /* scheme://user:pass@host[:port]... */
  /*                    ^              */

  /* We attempt to break down the URL into the components path,
     params, query, and fragment.  They are ordered like this:

       scheme://host[:port][/path][;params][?query][#fragment]  */

  path_b     = path_e     = NULL;
  params_b   = params_e   = NULL;
  query_b    = query_e    = NULL;
  fragment_b = fragment_e = NULL;

  /* Initialize separators for optional parts of URL, depending on the
     scheme.  For example, FTP has params, and HTTP and HTTPS have
     query string and fragment. */
  seps = init_seps (scheme);

  host_b = p;

  if (*p == '[')
    {
      /* Handle IPv6 address inside square brackets.  Ideally we'd
         just look for the terminating ']', but rfc2732 mandates
         rejecting invalid IPv6 addresses.  */

      /* The address begins after '['. */
      host_b = p + 1;
      host_e = strchr (host_b, ']');

      if (!host_e)
        {
          error_code = PE_UNTERMINATED_IPV6_ADDRESS;
          goto error;
        }

#ifdef ENABLE_IPV6
      /* Check if the IPv6 address is valid. */
      if (!is_valid_ipv6_address(host_b, host_e))
        {
          error_code = PE_INVALID_IPV6_ADDRESS;
          goto error;
        }

      /* Continue parsing after the closing ']'. */
      p = host_e + 1;
#else
      error_code = PE_IPV6_NOT_SUPPORTED;
      goto error;
#endif

      /* The closing bracket must be followed by a separator or by the
         null char.  */
      /* http://[::1]... */
      /*             ^   */
      if (!strchr (seps, *p))
        {
          /* Trailing garbage after []-delimited IPv6 address. */
          error_code = PE_INVALID_HOST_NAME;
          goto error;
        }
    }
  else
    {
      p = strpbrk_or_eos (p, seps);
      host_e = p;
    }
  ++seps;                       /* advance to '/' */

  if (host_b == host_e)
    {
      error_code = PE_INVALID_HOST_NAME;
      goto error;
    }

  port = scheme_default_port (scheme);
  if (*p == ':')
    {
      const char *port_b, *port_e, *pp;

      /* scheme://host:port/tralala */
      /*              ^             */
      ++p;
      port_b = p;
      p = strpbrk_or_eos (p, seps);
      port_e = p;

      /* Allow empty port, as per rfc2396. */
      if (port_b != port_e)
        for (port = 0, pp = port_b; pp < port_e; pp++)
          {
            if (!c_isdigit (*pp))
              {
                /* http://host:12randomgarbage/blah */
                /*               ^                  */
                error_code = PE_BAD_PORT_NUMBER;
                goto error;
              }
            port = 10 * port + (*pp - '0');
            /* Check for too large port numbers here, before we have
               a chance to overflow on bogus port values.  */
            if (port > 0xffff)
              {
                error_code = PE_BAD_PORT_NUMBER;
                goto error;
              }
          }
    }
  /* Advance to the first separator *after* '/' (either ';' or '?',
     depending on the scheme).  */
  ++seps;

  /* Get the optional parts of URL, each part being delimited by
     current location and the position of the next separator.  */
#define GET_URL_PART(sepchar, var) do {                         \
  if (*p == sepchar)                                            \
    var##_b = ++p, var##_e = p = strpbrk_or_eos (p, seps);      \
  ++seps;                                                       \
} while (0)

  GET_URL_PART ('/', path);
  if (supported_schemes[scheme].flags & scm_has_params)
    GET_URL_PART (';', params);
  if (supported_schemes[scheme].flags & scm_has_query)
    GET_URL_PART ('?', query);
  if (supported_schemes[scheme].flags & scm_has_fragment)
    GET_URL_PART ('#', fragment);

#undef GET_URL_PART
  assert (*p == 0);

  if (uname_b != uname_e)
    {
      /* http://user:pass@host */
      /*        ^         ^    */
      /*     uname_b   uname_e */
      if (!parse_credentials (uname_b, uname_e - 1, &user, &passwd))
        {
          error_code = PE_INVALID_USER_NAME;
          goto error;
        }
    }

  u = xnew0 (struct url);
  u->scheme = scheme;
  u->host   = strdupdelim (host_b, host_e);
  u->port   = port;
  u->user   = user;
  u->passwd = passwd;

  u->path = strdupdelim (path_b, path_e);
  path_modified = path_simplify (scheme, u->path);
  split_path (u->path, &u->dir, &u->file);

  host_modified = lowercase_str (u->host);

  /* Decode %HH sequences in host name.  This is important not so much
     to support %HH sequences in host names (which other browser
     don't), but to support binary characters (which will have been
     converted to %HH by reencode_escapes).  */
  if (strchr (u->host, '%'))
    {
      url_unescape (u->host);
      host_modified = true;

      /* check for invalid control characters in host name */
      for (p = u->host; *p; p++)
        {
          if (c_iscntrl(*p))
            {
              url_free(u);
              error_code = PE_INVALID_HOST_NAME;
              goto error;
            }
        }

      /* Apply IDNA regardless of iri->utf8_encode status */
      if (opt.enable_iri && iri)
        {
          char *new = idn_encode (iri, u->host);
          if (new)
            {
              xfree (u->host);
              u->host = new;
              u->idn_allocated = true;
              host_modified = true;
            }
        }
    }

  if (params_b)
    u->params = strdupdelim (params_b, params_e);
  if (query_b)
    u->query = strdupdelim (query_b, query_e);
  if (fragment_b)
    u->fragment = strdupdelim (fragment_b, fragment_e);

  if (opt.enable_iri || path_modified || u->fragment || host_modified || path_b == path_e)
    {
      /* If we suspect that a transformation has rendered what
         url_string might return different from URL_ENCODED, rebuild
         u->url using url_string.  */
      u->url = url_string (u, URL_AUTH_SHOW);

      if (url_encoded != url)
        xfree (url_encoded);
    }
  else
    {
      if (url_encoded == url)
        u->url = xstrdup (url);
      else
        u->url = (char *) url_encoded;
    }

  return u;

 error:
  /* Cleanup in case of error: */
  if (url_encoded && url_encoded != url)
    xfree (url_encoded);

  /* Transmit the error code to the caller, if the caller wants to
     know.  */
  if (error)
    *error = error_code;
  return NULL;
}

Project: wget

url_free (struct url *url)
{
  if (url)
    {
      if (url->idn_allocated) {
        idn2_free (url->host);      /* A dummy if !defined(ENABLE_IRI) */
        url->host = NULL;
      }
      else
        xfree (url->host);

      xfree (url->path);
      xfree (url->url);

      xfree (url->params);
      xfree (url->query);
      xfree (url->fragment);
      xfree (url->user);
      xfree (url->passwd);

      xfree (url->dir);
      xfree (url->file);

      xfree (url);
    }
}

Project: wget

mkalldirs (const char *path)
{
  const char *p;
  char *t;
  struct stat st;
  int res;

  p = path + strlen (path);
  for (; *p != '/' && p != path; p--)
    ;

  /* Don't create if it's just a file.  */
  if ((p == path) && (*p != '/'))
    return 0;
  t = strdupdelim (path, p);

  /* Check whether the directory exists.  */
  if ((stat (t, &st) == 0))
    {
      if (S_ISDIR (st.st_mode))
        {
          xfree (t);
          return 0;
        }
      else
        {
          /* If the dir exists as a file name, remove it first.  This
             is *only* for Wget to work with buggy old CERN http
             servers.  Here is the scenario: When Wget tries to
             retrieve a directory without a slash, e.g.
             http://foo/bar (bar being a directory), CERN server will
             not redirect it too http://foo/bar/ -- it will generate a
             directory listing containing links to bar/file1,
             bar/file2, etc.  Wget will lose because it saves this
             HTML listing to a file `bar', so it cannot create the
             directory.  To work around this, if the file of the same
             name exists, we just remove it and create the directory
             anyway.  */
          DEBUGP (("Removing %s because of directory danger!\n", t));
          if (unlink (t))
            logprintf (LOG_NOTQUIET, "Failed to unlink %s (%d): %s\n",
                       t, errno, strerror(errno));
        }
    }
  res = make_directory (t);
  if (res != 0)
    logprintf (LOG_NOTQUIET, "%s: %s\n", t, strerror (errno));
  xfree (t);
  return res;
}