Commit message: Support non-ASCII URLs

* src/url.c [HAVE_ICONV]: Include iconv.h and langinfo.h.
(convert_fname): New function.
[HAVE_ICONV]: Convert file name from remote encoding to local
encoding.
(url_file_name): Call convert_fname.
(filechr_table): Don't consider bytes in 128..159 as control
characters.

* tests/Test-ftp-iri.px: Fix the expected file name to match the
new file-name recoding.  State the remote encoding explicitly on
the Wget command line.

* NEWS: Mention the URI recoding when built with libiconv.
CVE: CVE-2016-4971
CWE: NVD-CWE-noinfo
Vulnerability description: GNU wget before 1.18 allows remote servers to write to arbitrary files by redirecting a request from HTTP to a crafted FTP resource.

Project: wget

url_valid_scheme (const char *url)
{
  enum url_scheme scheme = url_scheme (url);
  return scheme != SCHEME_INVALID;
}

Project: wget

reencode_escapes (const char *s)
{
  const char *p1;
  char *newstr, *p2;
  int oldlen, newlen;

  int encode_count = 0;

  /* First pass: inspect the string to see if there's anything to do,
     and to calculate the new length.  */
  for (p1 = s; *p1; p1++)
    if (char_needs_escaping (p1))
      ++encode_count;

  if (!encode_count)
    /* The string is good as it is. */
    return (char *) s;          /* C const model sucks. */

  oldlen = p1 - s;
  /* Each encoding adds two characters (hex digits).  */
  newlen = oldlen + 2 * encode_count;
  newstr = xmalloc (newlen + 1);

  /* Second pass: copy the string to the destination address, encoding
     chars when needed.  */
  p1 = s;
  p2 = newstr;

  while (*p1)
    if (char_needs_escaping (p1))
      {
        unsigned char c = *p1++;
        *p2++ = '%';
        *p2++ = XNUM_TO_DIGIT (c >> 4);
        *p2++ = XNUM_TO_DIGIT (c & 0xf);
      }
    else
      *p2++ = *p1++;

  *p2 = '\0';
  assert (p2 - newstr == newlen);
  return newstr;
}

Project: wget

strpbrk_or_eos (const char *s, const char *accept)
{
  char *p = strpbrk (s, accept);
  if (!p)
    p = strchr (s, '\0');
  return p;
}

Project: wget

schemes_are_similar_p (enum url_scheme a, enum url_scheme b)
{
  if (a == b)
    return true;
#ifdef HAVE_SSL
  if ((a == SCHEME_HTTP && b == SCHEME_HTTPS)
      || (a == SCHEME_HTTPS && b == SCHEME_HTTP))
    return true;
#endif
  return false;
}

Project: wget

test_append_uri_pathel(void)
{
  unsigned i;
  static const struct {
    const char *original_url;
    const char *input;
    bool escaped;
    const char *expected_result;
  } test_array[] = {
    { "http://www.yoyodyne.com/path/", "somepage.html", false, "http://www.yoyodyne.com/path/somepage.html" },
  };

  for (i = 0; i < countof(test_array); ++i)
    {
      struct growable dest;
      const char *p = test_array[i].input;

      memset (&dest, 0, sizeof (dest));

      append_string (test_array[i].original_url, &dest);
      append_uri_pathel (p, p + strlen(p), test_array[i].escaped, &dest);

      mu_assert ("test_append_uri_pathel: wrong result",
                 strcmp (dest.base, test_array[i].expected_result) == 0);
      xfree (dest.base);
    }

  return NULL;
}

Project: wget

url_set_file (struct url *url, const char *newfile)
{
  xfree (url->file);
  url->file = xstrdup (newfile);
  sync_path (url);
}

Project: wget

run_test (const char *test, const char *expected_result, enum url_scheme scheme,
          bool expected_change)
{
  char *test_copy = xstrdup (test);
  bool modified = path_simplify (scheme, test_copy);

  if (0 != strcmp (test_copy, expected_result))
    {
      printf ("Failed path_simplify(\"%s\"): expected \"%s\", got \"%s\".\n",
              test, expected_result, test_copy);
      mu_assert ("", 0);
    }
  if (modified != expected_change)
    {
      if (expected_change)
        printf ("Expected modification with path_simplify(\"%s\").\n",
                test);
      else
        printf ("Expected no modification with path_simplify(\"%s\").\n",
                test);
    }
  xfree (test_copy);
  mu_assert ("", modified == expected_change);
  return NULL;
}

Project: wget

full_path_write (const struct url *url, char *where)
{
#define FROB(el, chr) do {                      \
  char *f_el = url->el;                         \
  if (f_el) {                                   \
    int l = strlen (f_el);                      \
    *where++ = chr;                             \
    memcpy (where, f_el, l);                    \
    where += l;                                 \
  }                                             \
} while (0)

  FROB (path, '/');
  FROB (params, ';');
  FROB (query, '?');

#undef FROB
}

Project: wget

lowercase_str (char *str)
{
  bool changed = false;
  for (; *str; str++)
    if (c_isupper (*str))
      {
        changed = true;
        *str = c_tolower (*str);
      }
  return changed;
}

Project: wget

append_uri_pathel (const char *b, const char *e, bool escaped,
                   struct growable *dest)
{
  const char *p;
  int quoted, outlen;

  int mask;
  if (opt.restrict_files_os == restrict_unix)
    mask = filechr_not_unix;
  else if (opt.restrict_files_os == restrict_vms)
    mask = filechr_not_vms;
  else
    mask = filechr_not_windows;
  if (opt.restrict_files_ctrl)
    mask |= filechr_control;

  /* Copy [b, e) to PATHEL and URL-unescape it. */
  if (escaped)
    {
      char *unescaped;
      BOUNDED_TO_ALLOCA (b, e, unescaped);
      url_unescape (unescaped);
      b = unescaped;
      e = unescaped + strlen (unescaped);
    }

  /* Defang ".." when found as component of path.  Remember that path
     comes from the URL and might contain malicious input.  */
  if (e - b == 2 && b[0] == '.' && b[1] == '.')
    {
      b = "%2E%2E";
      e = b + 6;
    }

  /* Walk the PATHEL string and check how many characters we'll need
     to quote.  */
  quoted = 0;
  for (p = b; p < e; p++)
    if (FILE_CHAR_TEST (*p, mask))
      ++quoted;

  /* Calculate the length of the output string.  e-b is the input
     string length.  Each quoted char introduces two additional
     characters in the string, hence 2*quoted.  */
  outlen = (e - b) + (2 * quoted);
  GROW (dest, outlen);

  if (!quoted)
    {
      /* If there's nothing to quote, we can simply append the string
         without processing it again.  */
      memcpy (TAIL (dest), b, outlen);
    }
  else
    {
      char *q = TAIL (dest);
      for (p = b; p < e; p++)
        {
          if (!FILE_CHAR_TEST (*p, mask))
            *q++ = *p;
          else
            {
              unsigned char ch = *p;
              *q++ = '%';
              *q++ = XNUM_TO_DIGIT (ch >> 4);
              *q++ = XNUM_TO_DIGIT (ch & 0xf);
            }
        }
      assert (q - TAIL (dest) == outlen);
    }

  /* Perform inline case transformation if required.  */
  if (opt.restrict_files_case == restrict_lowercase
      || opt.restrict_files_case == restrict_uppercase)
    {
      char *q;
      for (q = TAIL (dest); q < TAIL (dest) + outlen; ++q)
        {
          if (opt.restrict_files_case == restrict_lowercase)
            *q = c_tolower (*q);
          else
            *q = c_toupper (*q);
        }
    }

  TAIL_INCR (dest, outlen);
  append_null (dest);
}

Project: wget

unescape_single_char (char *str, char chr)
{
  const char c1 = XNUM_TO_DIGIT (chr >> 4);
  const char c2 = XNUM_TO_DIGIT (chr & 0xf);
  char *h = str;                /* hare */
  char *t = str;                /* tortoise */
  for (; *h; h++, t++)
    {
      if (h[0] == '%' && h[1] == c1 && h[2] == c2)
        {
          *t = chr;
          h += 2;
        }
      else
        *t = *h;
    }
  *t = '\0';
}

Project: wget

scheme_default_port (enum url_scheme scheme)
{
  return supported_schemes[scheme].default_port;
}

Project: wget

split_path (const char *path, char **dir, char **file)
{
  char *last_slash = strrchr (path, '/');
  if (!last_slash)
    {
      *dir = xstrdup ("");
      *file = xstrdup (path);
    }
  else
    {
      *dir = strdupdelim (path, last_slash);
      *file = xstrdup (last_slash + 1);
    }
  url_unescape (*dir);
  url_unescape (*file);
}

Project: wget

url_escape_dir (const char *dir)
{
  char *newdir = url_escape_1 (dir, urlchr_unsafe | urlchr_reserved, 1);
  if (newdir == dir)
    return (char *)dir;

  unescape_single_char (newdir, '/');
  return newdir;
}

Project: wget

sync_path (struct url *u)
{
  char *newpath, *efile, *edir;

  xfree (u->path);

  /* u->dir and u->file are not escaped.  URL-escape them before
     reassembling them into u->path.  That way, if they contain
     separators like '?' or even if u->file contains slashes, the
     path will be correctly assembled.  (u->file can contain slashes
     if the URL specifies it with %2f, or if an FTP server returns
     it.)  */
  edir = url_escape_dir (u->dir);
  efile = url_escape_1 (u->file, urlchr_unsafe | urlchr_reserved, 1);

  if (!*edir)
    newpath = xstrdup (efile);
  else
    {
      int dirlen = strlen (edir);
      int filelen = strlen (efile);

      /* Copy "DIR/FILE" to newpath. */
      char *p = newpath = xmalloc (dirlen + 1 + filelen + 1);
      memcpy (p, edir, dirlen);
      p += dirlen;
      *p++ = '/';
      memcpy (p, efile, filelen);
      p += filelen;
      *p = '\0';
    }

  u->path = newpath;

  if (edir != u->dir)
    xfree (edir);
  if (efile != u->file)
    xfree (efile);

  /* Regenerate u->url as well.  */
  xfree (u->url);
  u->url = url_string (u, URL_AUTH_SHOW);
}

Project: wget

url_unescape_1 (char *s, unsigned char mask)
{
  char *t = s;                  /* t - tortoise */
  char *h = s;                  /* h - hare     */

  for (; *h; h++, t++)
    {
      if (*h != '%')
        {
        copychar:
          *t = *h;
        }
      else
        {
          char c;
          /* Do nothing if '%' is not followed by two hex digits. */
          if (!h[1] || !h[2] || !(c_isxdigit (h[1]) && c_isxdigit (h[2])))
            goto copychar;
          c = X2DIGITS_TO_NUM (h[1], h[2]);
          if (urlchr_test(c, mask))
            goto copychar;
          /* Don't unescape %00 because there is no way to insert it
             into a C string without effectively truncating it. */
          if (c == '\0')
            goto copychar;
          *t = c;
          h += 2;
        }
    }
  *t = '\0';
}

Project: wget

url_string (const struct url *url, enum url_auth_mode auth_mode)
{
  int size;
  char *result, *p;
  char *quoted_host, *quoted_user = NULL, *quoted_passwd = NULL;

  int scheme_port = supported_schemes[url->scheme].default_port;
  const char *scheme_str = supported_schemes[url->scheme].leading_string;
  int fplen = full_path_length (url);

  bool brackets_around_host;

  assert (scheme_str != NULL);

  /* Make sure the user name and password are quoted. */
  if (url->user)
    {
      if (auth_mode != URL_AUTH_HIDE)
        {
          quoted_user = url_escape_allow_passthrough (url->user);
          if (url->passwd)
            {
              if (auth_mode == URL_AUTH_HIDE_PASSWD)
                quoted_passwd = (char *) HIDDEN_PASSWORD;
              else
                quoted_passwd = url_escape_allow_passthrough (url->passwd);
            }
        }
    }

  /* In the unlikely event that the host name contains non-printable
     characters, quote it for displaying to the user.  */
  quoted_host = url_escape_allow_passthrough (url->host);

  /* Undo the quoting of colons that URL escaping performs.  IPv6
     addresses may legally contain colons, and in that case must be
     placed in square brackets.  */
  if (quoted_host != url->host)
    unescape_single_char (quoted_host, ':');
  brackets_around_host = strchr (quoted_host, ':') != NULL;

  size = (strlen (scheme_str)
          + strlen (quoted_host)
          + (brackets_around_host ? 2 : 0)
          + fplen
          + 1);
  if (url->port != scheme_port)
    size += 1 + numdigit (url->port);
  if (quoted_user)
    {
      size += 1 + strlen (quoted_user);
      if (quoted_passwd)
        size += 1 + strlen (quoted_passwd);
    }

  p = result = xmalloc (size);

  APPEND (p, scheme_str);
  if (quoted_user)
    {
      APPEND (p, quoted_user);
      if (quoted_passwd)
        {
          *p++ = ':';
          APPEND (p, quoted_passwd);
        }
      *p++ = '@';
    }

  if (brackets_around_host)
    *p++ = '[';
  APPEND (p, quoted_host);
  if (brackets_around_host)
    *p++ = ']';
  if (url->port != scheme_port)
    {
      *p++ = ':';
      p = number_to_string (p, url->port);
    }

  full_path_write (url, p);
  p += fplen;
  *p++ = '\0';

  assert (p - result == size);

  if (quoted_user && quoted_user != url->user)
    xfree (quoted_user);
  if (quoted_passwd && auth_mode == URL_AUTH_SHOW
      && quoted_passwd != url->passwd)
    xfree (quoted_passwd);
  if (quoted_host != url->host)
    xfree (quoted_host);

  return result;
}

Project: wget

url_escape (const char *s)
{
  return url_escape_1 (s, urlchr_unsafe, false);
}

Project: wget

append_char (char ch, struct growable *dest)
{
  if (ch)
    {
      GROW (dest, 1);
      *TAIL (dest) = ch;
      TAIL_INCR (dest, 1);
    }

  append_null (dest);
}

Project: wget

char_needs_escaping (const char *p)
{
  if (*p == '%')
    {
      if (c_isxdigit (*(p + 1)) && c_isxdigit (*(p + 2)))
        return false;
      else
        /* Garbled %.. sequence: encode `%'. */
        return true;
    }
  else if (URL_UNSAFE_CHAR (*p) && !URL_RESERVED_CHAR (*p))
    return true;
  else
    return false;
}

Project: wget

url_set_dir (struct url *url, const char *newdir)
{
  xfree (url->dir);
  url->dir = xstrdup (newdir);
  sync_path (url);
}

Project: wget

mkalldirs (const char *path)
{
  const char *p;
  char *t;
  struct_stat st;
  int res;

  p = path + strlen (path);
  for (; *p != '/' && p != path; p--)
    ;

  /* Don't create if it's just a file.  */
  if ((p == path) && (*p != '/'))
    return 0;
  t = strdupdelim (path, p);

  /* Check whether the directory exists.  */
  if ((stat (t, &st) == 0))
    {
      if (S_ISDIR (st.st_mode))
        {
          xfree (t);
          return 0;
        }
      else
        {
          /* If the dir exists as a file name, remove it first.  This
             is *only* for Wget to work with buggy old CERN http
             servers.  Here is the scenario: When Wget tries to
             retrieve a directory without a slash, e.g.
             http://foo/bar (bar being a directory), CERN server will
             not redirect it too http://foo/bar/ -- it will generate a
             directory listing containing links to bar/file1,
             bar/file2, etc.  Wget will lose because it saves this
             HTML listing to a file `bar', so it cannot create the
             directory.  To work around this, if the file of the same
             name exists, we just remove it and create the directory
             anyway.  */
          DEBUGP (("Removing %s because of directory danger!\n", t));
          unlink (t);
        }
    }
  res = make_directory (t);
  if (res != 0)
    logprintf (LOG_NOTQUIET, "%s: %s", t, strerror (errno));
  xfree (t);
  return res;
}

Project: wget

append_null (struct growable *dest)
{
  GROW (dest, 1);
  *TAIL (dest) = 0;
}

Project: wget

full_path_length (const struct url *url)
{
  int len = 0;

#define FROB(el) if (url->el) len += 1 + strlen (url->el)

  FROB (path);
  FROB (params);
  FROB (query);

#undef FROB

  return len;
}

Project: wget

path_simplify (enum url_scheme scheme, char *path)
{
  char *h = path;               /* hare */
  char *t = path;               /* tortoise */
  char *beg = path;
  char *end = strchr (path, '\0');

  while (h < end)
    {
      /* Hare should be at the beginning of a path element. */

      if (h[0] == '.' && (h[1] == '/' || h[1] == '\0'))
        {
          /* Ignore "./". */
          h += 2;
        }
      else if (h[0] == '.' && h[1] == '.' && (h[2] == '/' || h[2] == '\0'))
        {
          /* Handle "../" by retreating the tortoise by one path
             element -- but not past beginning.  */
          if (t > beg)
            {
              /* Move backwards until T hits the beginning of the
                 previous path element or the beginning of path. */
              for (--t; t > beg && t[-1] != '/'; t--)
                ;
            }
          else if (scheme == SCHEME_FTP
#ifdef HAVE_SSL
              || scheme == SCHEME_FTPS
#endif
              )
            {
              /* If we're at the beginning, copy the "../" literally
                 and move the beginning so a later ".." doesn't remove
                 it.  This violates RFC 3986; but we do it for FTP
                 anyway because there is otherwise no way to get at a
                 parent directory, when the FTP server drops us in a
                 non-root directory (which is not uncommon). */
              beg = t + 3;
              goto regular;
            }
          h += 3;
        }
      else
        {
        regular:
          /* A regular path element.  If H hasn't advanced past T,
             simply skip to the next path element.  Otherwise, copy
             the path element until the next slash.  */
          if (t == h)
            {
              /* Skip the path element, including the slash.  */
              while (h < end && *h != '/')
                t++, h++;
              if (h < end)
                t++, h++;
            }
          else
            {
              /* Copy the path element, including the final slash.  */
              while (h < end && *h != '/')
                *t++ = *h++;
              if (h < end)
                *t++ = *h++;
            }
        }
    }

  if (t != h)
    *t = '\0';

  return t != h;
}

Project: wget

rewrite_shorthand_url (const char *url)
{
  const char *p;
  char *ret;

  if (url_scheme (url) != SCHEME_INVALID)
    return NULL;

  /* Look for a ':' or '/'.  The former signifies NcFTP syntax, the
     latter Netscape.  */
  p = strpbrk (url, ":/");
  if (p == url)
    return NULL;

  /* If we're looking at "://", it means the URL uses a scheme we
     don't support, which may include "https" when compiled without
     SSL support.  Don't bogusly rewrite such URLs.  */
  if (p && p[0] == ':' && p[1] == '/' && p[2] == '/')
    return NULL;

  if (p && *p == ':')
    {
      /* Colon indicates ftp, as in foo.bar.com:path.  Check for
         special case of http port number ("localhost:10000").  */
      int digits = strspn (p + 1, "0123456789");
      if (digits && (p[1 + digits] == '/' || p[1 + digits] == '\0'))
        goto http;

      /* Turn "foo.bar.com:path" to "ftp://foo.bar.com/path". */
      if ((ret = aprintf ("ftp://%s", url)) != NULL)
        ret[6 + (p - url)] = '/';
    }
  else
    {
    http:
      /* Just prepend "http://" to URL. */
      ret = aprintf ("http://%s", url);
    }
  return ret;
}

Project: wget

url_escape_allow_passthrough (const char *s)
{
  return url_escape_1 (s, urlchr_unsafe, true);
}

Project: wget

url_scheme (const char *url)
{
  int i;

  for (i = 0; supported_schemes[i].leading_string; i++)
    if (0 == strncasecmp (url, supported_schemes[i].leading_string,
                          strlen (supported_schemes[i].leading_string)))
      {
        if (!(supported_schemes[i].flags & scm_disabled))
          return (enum url_scheme) i;
        else
          return SCHEME_INVALID;
      }

  return SCHEME_INVALID;
}

Project: wget

test_path_simplify (void)
{
  static const struct {
    const char *test, *result;
    enum url_scheme scheme;
    bool should_modify;
  } tests[] = {
    { "",                       "",             SCHEME_HTTP, false },
    { ".",                      "",             SCHEME_HTTP, true },
    { "./",                     "",             SCHEME_HTTP, true },
    { "..",                     "",             SCHEME_HTTP, true },
    { "../",                    "",             SCHEME_HTTP, true },
    { "..",                     "..",           SCHEME_FTP,  false },
    { "../",                    "../",          SCHEME_FTP,  false },
    { "foo",                    "foo",          SCHEME_HTTP, false },
    { "foo/bar",                "foo/bar",      SCHEME_HTTP, false },
    { "foo///bar",              "foo///bar",    SCHEME_HTTP, false },
    { "foo/.",                  "foo/",         SCHEME_HTTP, true },
    { "foo/./",                 "foo/",         SCHEME_HTTP, true },
    { "foo./",                  "foo./",        SCHEME_HTTP, false },
    { "foo/../bar",             "bar",          SCHEME_HTTP, true },
    { "foo/../bar/",            "bar/",         SCHEME_HTTP, true },
    { "foo/bar/..",             "foo/",         SCHEME_HTTP, true },
    { "foo/bar/../x",           "foo/x",        SCHEME_HTTP, true },
    { "foo/bar/../x/",          "foo/x/",       SCHEME_HTTP, true },
    { "foo/..",                 "",             SCHEME_HTTP, true },
    { "foo/../..",              "",             SCHEME_HTTP, true },
    { "foo/../../..",           "",             SCHEME_HTTP, true },
    { "foo/../../bar/../../baz", "baz",         SCHEME_HTTP, true },
    { "foo/../..",              "..",           SCHEME_FTP,  true },
    { "foo/../../..",           "../..",        SCHEME_FTP,  true },
    { "foo/../../bar/../../baz", "../../baz",   SCHEME_FTP,  true },
    { "a/b/../../c",            "c",            SCHEME_HTTP, true },
    { "./a/../b",               "b",            SCHEME_HTTP, true }
  };
  unsigned i;

  for (i = 0; i < countof (tests); i++)
    {
      const char *message;
      const char *test = tests[i].test;
      const char *expected_result = tests[i].result;
      enum url_scheme scheme = tests[i].scheme;
      bool  expected_change = tests[i].should_modify;

      message = run_test (test, expected_result, scheme, expected_change);
      if (message) return message;
    }
  return NULL;
}

Project: wget

test_are_urls_equal(void)
{
  unsigned i;
  static const struct {
    const char *url1;
    const char *url2;
    bool expected_result;
  } test_array[] = {
    { "http://www.adomain.com/apath/", "http://www.adomain.com/apath/",       true },
    { "http://www.adomain.com/apath/", "http://www.adomain.com/anotherpath/", false },
    { "http://www.adomain.com/apath/", "http://www.anotherdomain.com/path/",  false },
    { "http://www.adomain.com/~path/", "http://www.adomain.com/%7epath/",     true },
    { "http://www.adomain.com/longer-path/", "http://www.adomain.com/path/",  false },
    { "http://www.adomain.com/path%2f", "http://www.adomain.com/path/",       false },
  };

  for (i = 0; i < countof(test_array); ++i)
    {
      mu_assert ("test_are_urls_equal: wrong result",
                 are_urls_equal (test_array[i].url1, test_array[i].url2) == test_array[i].expected_result);
    }

  return NULL;
}

Project: wget

url_has_scheme (const char *url)
{
  const char *p = url;

  /* The first char must be a scheme char. */
  if (!*p || !SCHEME_CHAR (*p))
    return false;
  ++p;
  /* Followed by 0 or more scheme chars. */
  while (*p && SCHEME_CHAR (*p))
    ++p;
  /* Terminated by ':'. */
  return *p == ':';
}

Project: wget

url_escape_unsafe_and_reserved (const char *s)
{
  return url_escape_1 (s, urlchr_unsafe|urlchr_reserved, false);
}

Project: wget

ps (char *path)
{
  char *copy = xstrdup (path);
  path_simplify (copy);
  return copy;
}

Project: wget

getchar_from_escaped_string (const char *str, char *c)
{
  const char *p = str;

  assert (str && *str);
  assert (c);

  if (p[0] == '%')
    {
      if (!c_isxdigit(p[1]) || !c_isxdigit(p[2]))
        {
          *c = '%';
          return 1;
        }
      else
        {
          if (p[2] == 0)
            return 0; /* error: invalid string */

          *c = X2DIGITS_TO_NUM (p[1], p[2]);
          if (URL_RESERVED_CHAR(*c))
            {
              *c = '%';
              return 1;
            }
          else
            return 3;
        }
    }
  else
    {
      *c = p[0];
    }

  return 1;
}

Project: wget

url_free (struct url *url)
{
  if (url)
    {
      if (url->idn_allocated) {
        idn_free (url->host);      /* A dummy if !defined(ENABLE_IRI) */
        url->host = NULL;
      }
      else
        xfree (url->host);

      xfree (url->path);
      xfree (url->url);

      xfree (url->params);
      xfree (url->query);
      xfree (url->fragment);
      xfree (url->user);
      xfree (url->passwd);

      xfree (url->dir);
      xfree (url->file);

      xfree (url);
    }
}

Project: wget

scheme_disable (enum url_scheme scheme)
{
  supported_schemes[scheme].flags |= scm_disabled;
}

Project: wget

url_unescape_except_reserved (char *s)
{
  url_unescape_1 (s, urlchr_reserved);
}

Project: wget

url_error (const char *url, int error_code)
{
  assert (error_code >= 0 && ((size_t) error_code) < countof (parse_errors));

  if (error_code == PE_UNSUPPORTED_SCHEME)
    {
      char *error, *p;
      char *scheme = xstrdup (url);
      assert (url_has_scheme (url));

      if ((p = strchr (scheme, ':')))
        *p = '\0';
      if (!c_strcasecmp (scheme, "https"))
        error = aprintf (_("HTTPS support not compiled in"));
      else
        error = aprintf (_(parse_errors[error_code]), quote (scheme));
      xfree (scheme);

      return error;
    }
  else
    return xstrdup (_(parse_errors[error_code]));
}

Project: wget

uri_merge (const char *base, const char *link)
{
  int linklength;
  const char *end;
  char *merge;

  if (url_has_scheme (link))
    return xstrdup (link);

  /* We may not examine BASE past END. */
  end = path_end (base);
  linklength = strlen (link);

  if (!*link)
    {
      /* Empty LINK points back to BASE, query string and all. */
      return xstrdup (base);
    }
  else if (*link == '?')
    {
      /* LINK points to the same location, but changes the query
         string.  Examples: */
      /* uri_merge("path",         "?new") -> "path?new"     */
      /* uri_merge("path?foo",     "?new") -> "path?new"     */
      /* uri_merge("path?foo#bar", "?new") -> "path?new"     */
      /* uri_merge("path#foo",     "?new") -> "path?new"     */
      int baselength = end - base;
      merge = xmalloc (baselength + linklength + 1);
      memcpy (merge, base, baselength);
      memcpy (merge + baselength, link, linklength);
      merge[baselength + linklength] = '\0';
    }
  else if (*link == '#')
    {
      /* uri_merge("path",         "#new") -> "path#new"     */
      /* uri_merge("path#foo",     "#new") -> "path#new"     */
      /* uri_merge("path?foo",     "#new") -> "path?foo#new" */
      /* uri_merge("path?foo#bar", "#new") -> "path?foo#new" */
      int baselength;
      const char *end1 = strchr (base, '#');
      if (!end1)
        end1 = base + strlen (base);
      baselength = end1 - base;
      merge = xmalloc (baselength + linklength + 1);
      memcpy (merge, base, baselength);
      memcpy (merge + baselength, link, linklength);
      merge[baselength + linklength] = '\0';
    }
  else if (*link == '/' && *(link + 1) == '/')
    {
      /* LINK begins with "//" and so is a net path: we need to
         replace everything after (and including) the double slash
         with LINK. */

      /* uri_merge("foo", "//new/bar")            -> "//new/bar"      */
      /* uri_merge("//old/foo", "//new/bar")      -> "//new/bar"      */
      /* uri_merge("http://old/foo", "//new/bar") -> "http://new/bar" */

      int span;
      const char *slash;
      const char *start_insert;

      /* Look for first slash. */
      slash = memchr (base, '/', end - base);
      /* If found slash and it is a double slash, then replace
         from this point, else default to replacing from the
         beginning.  */
      if (slash && *(slash + 1) == '/')
        start_insert = slash;
      else
        start_insert = base;

      span = start_insert - base;
      merge = xmalloc (span + linklength + 1);
      if (span)
        memcpy (merge, base, span);
      memcpy (merge + span, link, linklength);
      merge[span + linklength] = '\0';
    }
  else if (*link == '/')
    {
      /* LINK is an absolute path: we need to replace everything
         after (and including) the FIRST slash with LINK.

         So, if BASE is "http://host/whatever/foo/bar", and LINK is
         "/qux/xyzzy", our result should be
         "http://host/qux/xyzzy".  */
      int span;
      const char *slash;
      const char *start_insert = NULL; /* for gcc to shut up. */
      const char *pos = base;
      bool seen_slash_slash = false;
      /* We're looking for the first slash, but want to ignore
         double slash. */
    again:
      slash = memchr (pos, '/', end - pos);
      if (slash && !seen_slash_slash)
        if (*(slash + 1) == '/')
          {
            pos = slash + 2;
            seen_slash_slash = true;
            goto again;
          }

      /* At this point, SLASH is the location of the first / after
         "//", or the first slash altogether.  START_INSERT is the
         pointer to the location where LINK will be inserted.  When
         examining the last two examples, keep in mind that LINK
         begins with '/'. */

      if (!slash && !seen_slash_slash)
        /* example: "foo" */
        /*           ^    */
        start_insert = base;
      else if (!slash && seen_slash_slash)
        /* example: "http://foo" */
        /*                     ^ */
        start_insert = end;
      else if (slash && !seen_slash_slash)
        /* example: "foo/bar" */
        /*           ^        */
        start_insert = base;
      else if (slash && seen_slash_slash)
        /* example: "http://something/" */
        /*                           ^  */
        start_insert = slash;

      span = start_insert - base;
      merge = xmalloc (span + linklength + 1);
      if (span)
        memcpy (merge, base, span);
      memcpy (merge + span, link, linklength);
      merge[span + linklength] = '\0';
    }
  else
    {
      /* LINK is a relative URL: we need to replace everything
         after last slash (possibly empty) with LINK.

         So, if BASE is "whatever/foo/bar", and LINK is "qux/xyzzy",
         our result should be "whatever/foo/qux/xyzzy".  */
      bool need_explicit_slash = false;
      int span;
      const char *start_insert;
      const char *last_slash = find_last_char (base, end, '/');
      if (!last_slash)
        {
          /* No slash found at all.  Replace what we have with LINK. */
          start_insert = base;
        }
      else if (last_slash && last_slash >= base + 2
               && last_slash[-2] == ':' && last_slash[-1] == '/')
        {
          /* example: http://host"  */
          /*                      ^ */
          start_insert = end + 1;
          need_explicit_slash = true;
        }
      else
        {
          /* example: "whatever/foo/bar" */
          /*                        ^    */
          start_insert = last_slash + 1;
        }

      span = start_insert - base;
      merge = xmalloc (span + linklength + 1);
      if (span)
        memcpy (merge, base, span);
      if (need_explicit_slash)
        merge[span - 1] = '/';
      memcpy (merge + span, link, linklength);
      merge[span + linklength] = '\0';
    }

  return merge;
}

Project: wget

path_end (const char *url)
{
  enum url_scheme scheme = url_scheme (url);
  const char *seps;
  if (scheme == SCHEME_INVALID)
    scheme = SCHEME_HTTP;       /* use http semantics for rel links */
  /* +2 to ignore the first two separators ':' and '/' */
  seps = init_seps (scheme) + 2;
  return strpbrk_or_eos (url, seps);
}

Project: wget

convert_fname (const char *fname)
{
  char *converted_fname = (char *)fname;
#if HAVE_ICONV
  const char *from_encoding = opt.encoding_remote;
  const char *to_encoding = opt.locale;
  iconv_t cd;
  size_t len, done, inlen, outlen;
  char *s;
  const char *orig_fname = fname;;

  /* Defaults for remote and local encodings.  */
  if (!from_encoding)
    from_encoding = "UTF-8";
  if (!to_encoding)
    to_encoding = nl_langinfo (CODESET);

  cd = iconv_open (to_encoding, from_encoding);
  if (cd == (iconv_t)(-1))
    logprintf (LOG_VERBOSE, _("Conversion from %s to %s isn't supported\n"),
	       quote (from_encoding), quote (to_encoding));
  else
    {
      inlen = strlen (fname);
      len = outlen = inlen * 2;
      converted_fname = s = xmalloc (outlen + 1);
      done = 0;

      for (;;)
	{
	  if (iconv (cd, &fname, &inlen, &s, &outlen) != (size_t)(-1)
	      && iconv (cd, NULL, NULL, &s, &outlen) != (size_t)(-1))
	    {
	      *(converted_fname + len - outlen - done) = '\0';
	      iconv_close(cd);
	      DEBUGP (("Converted file name '%s' (%s) -> '%s' (%s)\n",
		       orig_fname, from_encoding, converted_fname, to_encoding));
	      xfree (orig_fname);
	      return converted_fname;
	    }

	  /* Incomplete or invalid multibyte sequence */
	  if (errno == EINVAL || errno == EILSEQ)
	    {
	      logprintf (LOG_VERBOSE,
			 _("Incomplete or invalid multibyte sequence encountered\n"));
	      xfree (converted_fname);
	      converted_fname = (char *)orig_fname;
	      break;
	    }
	  else if (errno == E2BIG) /* Output buffer full */
	    {
	      done = len;
	      len = outlen = done + inlen * 2;
	      converted_fname = xrealloc (converted_fname, outlen + 1);
	      s = converted_fname + done;
	    }
	  else /* Weird, we got an unspecified error */
	    {
	      logprintf (LOG_VERBOSE, _("Unhandled errno %d\n"), errno);
	      xfree (converted_fname);
	      converted_fname = (char *)orig_fname;
	      break;
	    }
	}
      DEBUGP (("Failed to convert file name '%s' (%s) -> '?' (%s)\n",
	       orig_fname, from_encoding, to_encoding));
    }

    iconv_close(cd);
#endif

  return converted_fname;
}

Project: wget

url_file_name (const struct url *u, char *replaced_filename)
{
  struct growable fnres;        /* stands for "file name result" */
  struct growable temp_fnres;

  const char *u_file;
  char *fname, *unique, *fname_len_check;
  const char *index_filename = "index.html"; /* The default index file is index.html */
  size_t max_length;

  fnres.base = NULL;
  fnres.size = 0;
  fnres.tail = 0;

  temp_fnres.base = NULL;
  temp_fnres.size = 0;
  temp_fnres.tail = 0;

  /* If an alternative index file was defined, change index_filename */
  if (opt.default_page)
    index_filename = opt.default_page;


  /* Start with the directory prefix, if specified. */
  if (opt.dir_prefix)
    append_string (opt.dir_prefix, &fnres);

  /* If "dirstruct" is turned on (typically the case with -r), add
     the host and port (unless those have been turned off) and
     directory structure.  */
  if (opt.dirstruct)
    {
      if (opt.protocol_directories)
        {
          if (fnres.tail)
            append_char ('/', &fnres);
          append_string (supported_schemes[u->scheme].name, &fnres);
        }
      if (opt.add_hostdir)
        {
          if (fnres.tail)
            append_char ('/', &fnres);
          if (0 != strcmp (u->host, ".."))
            append_string (u->host, &fnres);
          else
            /* Host name can come from the network; malicious DNS may
               allow ".." to be resolved, causing us to write to
               "../<file>".  Defang such host names.  */
            append_string ("%2E%2E", &fnres);
          if (u->port != scheme_default_port (u->scheme))
            {
              char portstr[24];
              number_to_string (portstr, u->port);
              append_char (FN_PORT_SEP, &fnres);
              append_string (portstr, &fnres);
            }
        }

      append_dir_structure (u, &fnres);
    }

  if (!replaced_filename)
    {
      /* Create the filename. */
      u_file = *u->file ? u->file : index_filename;

      /* Append "?query" to the file name, even if empty,
       * and create fname_len_check. */
      if (u->query)
        fname_len_check = concat_strings (u_file, FN_QUERY_SEP_STR, u->query, NULL);
      else
        fname_len_check = strdupdelim (u_file, u_file + strlen (u_file));
    }
  else
    {
      u_file = replaced_filename;
      fname_len_check = strdupdelim (u_file, u_file + strlen (u_file));
    }

  append_uri_pathel (fname_len_check,
    fname_len_check + strlen (fname_len_check), false, &temp_fnres);

  /* Zero-terminate the temporary file name. */
  append_char ('\0', &temp_fnres);

  /* Check that the length of the file name is acceptable. */
#ifdef WINDOWS
  if (MAX_PATH > (fnres.tail + CHOMP_BUFFER + 2))
    {
      max_length = MAX_PATH - (fnres.tail + CHOMP_BUFFER + 2);
      /* FIXME: In Windows a filename is usually limited to 255 characters.
      To really be accurate you could call GetVolumeInformation() to get
      lpMaximumComponentLength
      */
      if (max_length > 255)
        {
          max_length = 255;
        }
    }
  else
    {
      max_length = 0;
    }
#else
  max_length = get_max_length (fnres.base, fnres.tail, _PC_NAME_MAX) - CHOMP_BUFFER;
#endif
  if (max_length > 0 && strlen (temp_fnres.base) > max_length)
    {
      logprintf (LOG_NOTQUIET, "The name is too long, %lu chars total.\n",
          (unsigned long) strlen (temp_fnres.base));
      logprintf (LOG_NOTQUIET, "Trying to shorten...\n");

      /* Shorten the file name. */
      temp_fnres.base[max_length] = '\0';

      logprintf (LOG_NOTQUIET, "New name is %s.\n", temp_fnres.base);
    }

  xfree (fname_len_check);

  /* The filename has already been 'cleaned' by append_uri_pathel() above.  So,
   * just append it. */
  if (fnres.tail)
    append_char ('/', &fnres);
  append_string (temp_fnres.base, &fnres);

  fname = fnres.base;

  /* Make a final check that the path length is acceptable? */
  /* TODO: check fnres.base for path length problem */

  xfree (temp_fnres.base);

  fname = convert_fname (fname);

  /* Check the cases in which the unique extensions are not used:
     1) Clobbering is turned off (-nc).
     2) Retrieval with regetting.
     3) Timestamping is used.
     4) Hierarchy is built.
     5) Backups are specified.

     The exception is the case when file does exist and is a
     directory (see `mkalldirs' for explanation).  */

  if (ALLOW_CLOBBER
      && !(file_exists_p (fname) && !file_non_directory_p (fname)))
    {
      unique = fname;
    }
  else
    {
      unique = unique_name (fname, true);
      if (unique != fname)
        xfree (fname);
    }

/* On VMS, alter the name as required. */
#ifdef __VMS
  {
    char *unique2;

    unique2 = ods_conform( unique);
    if (unique2 != unique)
      {
        xfree (unique);
        unique = unique2;
      }
  }
#endif /* def __VMS */

  return unique;
}

Project: wget

url_escape_1 (const char *s, unsigned char mask, bool allow_passthrough)
{
  const char *p1;
  char *p2, *newstr;
  int newlen;
  int addition = 0;

  for (p1 = s; *p1; p1++)
    if (urlchr_test (*p1, mask))
      addition += 2;            /* Two more characters (hex digits) */

  if (!addition)
    return allow_passthrough ? (char *)s : xstrdup (s);

  newlen = (p1 - s) + addition;
  newstr = xmalloc (newlen + 1);

  p1 = s;
  p2 = newstr;
  while (*p1)
    {
      /* Quote the characters that match the test mask. */
      if (urlchr_test (*p1, mask))
        {
          unsigned char c = *p1++;
          *p2++ = '%';
          *p2++ = XNUM_TO_DIGIT (c >> 4);
          *p2++ = XNUM_TO_DIGIT (c & 0xf);
        }
      else
        *p2++ = *p1++;
    }
  assert (p2 - newstr == newlen);
  *p2 = '\0';

  return newstr;
}

Project: wget

url_unescape (char *s)
{
  url_unescape_1 (s, 0);
}

Project: wget

append_string (const char *str, struct growable *dest)
{
  int l = strlen (str);

  if (l)
    {
      GROW (dest, l);
      memcpy (TAIL (dest), str, l);
      TAIL_INCR (dest, l);
    }

  append_null (dest);
}

Project: wget

url_full_path (const struct url *url)
{
  int length = full_path_length (url);
  char *full_path = xmalloc (length + 1);

  full_path_write (url, full_path);
  full_path[length] = '\0';

  return full_path;
}

Project: wget

url_skip_credentials (const char *url)
{
  /* Look for '@' that comes before terminators, such as '/', '?',
     '#', or ';'.  */
  const char *p = (const char *)strpbrk (url, "@/?#;");
  if (!p || *p != '@')
    return url;
  return p + 1;
}

Project: wget

are_urls_equal (const char *u1, const char *u2)
{
  const char *p, *q;
  int pp, qq;
  char ch1, ch2;
  assert(u1 && u2);

  p = u1;
  q = u2;

  while (*p && *q
         && (pp = getchar_from_escaped_string (p, &ch1))
         && (qq = getchar_from_escaped_string (q, &ch2))
         && (c_tolower(ch1) == c_tolower(ch2)))
    {
      p += pp;
      q += qq;
    }

  return (*p == 0 && *q == 0 ? true : false);
}

Project: wget

init_seps (enum url_scheme scheme)
{
  static char seps[8] = ":/";
  char *p = seps + 2;
  int flags = supported_schemes[scheme].flags;

  if (flags & scm_has_params)
    *p++ = ';';
  if (flags & scm_has_query)
    *p++ = '?';
  if (flags & scm_has_fragment)
    *p++ = '#';
  *p = '\0';
  return seps;
}

Project: wget

append_dir_structure (const struct url *u, struct growable *dest)
{
  char *pathel, *next;
  int cut = opt.cut_dirs;

  /* Go through the path components, de-URL-quote them, and quote them
     (if necessary) as file names.  */

  pathel = u->path;
  for (; (next = strchr (pathel, '/')) != NULL; pathel = next + 1)
    {
      if (cut-- > 0)
        continue;
      if (pathel == next)
        /* Ignore empty pathels.  */
        continue;

      if (dest->tail)
        append_char ('/', dest);
      append_uri_pathel (pathel, next, true, dest);
    }
}

Project: wget

parse_credentials (const char *beg, const char *end, char **user, char **passwd)
{
  char *colon;
  const char *userend;

  if (beg == end)
    return false;               /* empty user name */

  colon = memchr (beg, ':', end - beg);
  if (colon == beg)
    return false;               /* again empty user name */

  if (colon)
    {
      *passwd = strdupdelim (colon + 1, end);
      userend = colon;
      url_unescape (*passwd);
    }
  else
    {
      *passwd = NULL;
      userend = end;
    }
  *user = strdupdelim (beg, userend);
  url_unescape (*user);
  return true;
}