Commit message: Don't save user/pw with --xattr

Also the Referer info is reduced to scheme+host+port.

* src/ftp.c (getftp): Change params of set_file_metadata()
* src/http.c (gethttp): Change params of set_file_metadata()
* src/xattr.c (set_file_metadata): Remove user/password from origin URL,
  reduce Referer value to scheme/host/port.
* src/xattr.h: Change prototype of set_file_metadata()
CVE: CVE-2018-20483
CWE: CWE-200
Vulnerability description: set_file_metadata in xattr.c in GNU Wget before 1.20.1 stores a file's origin URL in the user.xdg.origin.url metadata attribute of the extended attributes of the downloaded file, which allows local users to obtain sensitive information (e.g., credentials contained in the URL) by reading this attribute, as demonstrated by getfattr. This also applies to Referer information in the user.xdg.referrer.url metadata attribute. According to 2016-07-22 in the Wget ChangeLog, user.xdg.origin.url was partially based on the behavior of fwrite_xattr in tool_xattr.c in curl.

Project: wget

get_ftp_greeting (int csock, ccon *con)
{
  uerr_t err = 0;

  /* Get the server's greeting */
  err = ftp_greeting (csock);
  if (err != FTPOK)
    {
      logputs (LOG_NOTQUIET, "Error in server response. Closing.\n");
      fd_close (csock);
      con->csock = -1;
    }

  return err;
}

Project: wget

init_control_ssl_connection (int csock, struct url *u, bool *using_control_security)
{
  bool using_security = false;

  /* If '--ftps-implicit' was passed, perform the SSL handshake directly,
   * and do not send an AUTH command.
   * Otherwise send an AUTH sequence before login,
   * and perform the SSL handshake if accepted by server.
   */
  if (!opt.ftps_implicit && !opt.server_response)
    logputs (LOG_VERBOSE, "==> AUTH TLS ... ");
  if (opt.ftps_implicit || ftp_auth (csock, SCHEME_FTPS) == FTPOK)
    {
      if (!ssl_connect_wget (csock, u->host, NULL))
        {
          fd_close (csock);
          return CONSSLERR;
        }
      else if (!ssl_check_certificate (csock, u->host))
        {
          fd_close (csock);
          return VERIFCERTERR;
        }

      if (!opt.ftps_implicit && !opt.server_response)
        logputs (LOG_VERBOSE, " done.\n");

      /* If implicit FTPS was requested, we act as "normal" FTP, but over SSL.
       * We're not using RFC 2228 commands.
       */
      using_security = true;
    }
  else
    {
      /* The server does not support 'AUTH TLS'.
       * Check if --ftps-fallback-to-ftp was passed. */
      if (opt.ftps_fallback_to_ftp)
        {
          logputs (LOG_NOTQUIET, "Server does not support AUTH TLS. Falling back to FTP.\n");
          using_security = false;
        }
      else
        {
          fd_close (csock);
          return FTPNOAUTH;
        }
    }

  *using_control_security = using_security;
  return NOCONERROR;
}

Project: wget

resp_new (char *head)
{
  char *hdr;
  int count, size;

  struct response *resp = xnew0 (struct response);
  resp->data = head;

  if (*head == '\0')
    {
      /* Empty head means that we're dealing with a headerless
         (HTTP/0.9) response.  In that case, don't set HEADERS at
         all.  */
      return resp;
    }

  /* Split HEAD into header lines, so that resp_header_* functions
     don't need to do this over and over again.  */

  size = count = 0;
  hdr = head;
  while (1)
    {
      DO_REALLOC (resp->headers, size, count + 1, const char *);
      resp->headers[count++] = hdr;

      /* Break upon encountering an empty line. */
      if (!hdr[0] || (hdr[0] == '\r' && hdr[1] == '\n') || hdr[0] == '\n')
        break;

      /* Find the end of HDR, including continuations. */
      for (;;)
        {
          char *end = strchr (hdr, '\n');

          if (!end)
            {
              hdr += strlen (hdr);
              break;
            }

          hdr = end + 1;

          if (*hdr != ' ' && *hdr != '\t')
            break;

          // continuation, transform \r and \n into spaces
          *end = ' ';
          if (end > head && end[-1] == '\r')
            end[-1] = ' ';
        }
    }
  DO_REALLOC (resp->headers, size, count + 1, const char *);
  resp->headers[count] = NULL;

  return resp;
}

Project: wget

write_xattr_metadata (const char *name, const char *value, FILE *fp)
{
  (void)name;
  (void)value;
  (void)fp;

  return 0;
}

Project: wget

ftp_retrieve_glob (struct url *u, struct url *original_url,
                   ccon *con, int action)
{
  struct fileinfo *f, *start;
  uerr_t res;

  con->cmd |= LEAVE_PENDING;

  res = ftp_get_listing (u, original_url, con, &start);
  if (res != RETROK)
    return res;

  // Set the function used for glob matching.
  int (*matcher) (const char *, const char *, int)
    = opt.ignore_case ? fnmatch_nocase : fnmatch;

  // Set the function used to compare strings
#ifdef __VMS
  /* 2009-09-09 SMS.
   * Odd-ball compiler ("HP C V7.3-009 on OpenVMS Alpha V7.3-2")
   * bug causes spurious %CC-E-BADCONDIT complaint with this
   * "?:" statement.  (Different linkage attributes for strcmp()
   * and strcasecmp().)  Converting to "if" changes the
   * complaint to %CC-W-PTRMISMATCH on "cmp = strcmp;".  Adding
   * the senseless type cast clears the complaint, and looks
   * harmless.
   */
  int (*cmp) (const char *, const char *)
    = opt.ignore_case ? strcasecmp : (int (*)())strcmp;
#else /* def __VMS */
  int (*cmp) (const char *, const char *)
    = opt.ignore_case ? strcasecmp : strcmp;
#endif /* def __VMS [else] */

  f = start;
  while (f)
    {

      // Weed out files that do not confirm to the global rules given in
      // opt.accepts and opt.rejects
      if ((opt.accepts || opt.rejects) &&
          f->type != FT_DIRECTORY && !acceptable (f->name))
        {
          logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
                     quote (f->name));
          f = delelement (f, &start);
          continue;
        }


      // Identify and eliminate possibly harmful names or invalid entries.
      if (has_insecure_name_p (f->name) || is_invalid_entry (f))
        {
          logprintf (LOG_VERBOSE, _("Rejecting %s (Invalid Entry).\n"),
                     quote (f->name));
          f = delelement (f, &start);
          continue;
        }

      if (!accept_url (f->name))
        {
          logprintf (LOG_VERBOSE, _("%s is excluded/not-included through regex.\n"), f->name);
          f = delelement (f, &start);
          continue;
        }

      /* Now weed out the files that do not match our globbing pattern.
         If we are dealing with a globbing pattern, that is.  */
      if (*u->file)
        {
          if (action == GLOB_GLOBALL)
            {
              int matchres = matcher (u->file, f->name, 0);
              if (matchres == -1)
                {
                  logprintf (LOG_NOTQUIET, _("Error matching %s against %s: %s\n"),
                             u->file, quotearg_style (escape_quoting_style, f->name),
                             strerror (errno));
                  freefileinfo (start);
                  return RETRBADPATTERN;
                }
              if (matchres == FNM_NOMATCH)
                {
                  f = delelement (f, &start); /* delete the element from the list */
                  continue;
                }
            }
          else if (action == GLOB_GETONE)
            {
              if (0 != cmp(u->file, f->name))
                {
                  f = delelement (f, &start);
                  continue;
                }
            }
        }
      f = f->next;
    }

  /*
   * Now that preprocessing of the file listing is over, let's try to download
   * all the remaining files in our listing.
   */
  if (start)
    {
      /* Just get everything.  */
      res = ftp_retrieve_list (u, original_url, start, con);
    }
  else
    {
      if (action == GLOB_GLOBALL)
        {
          /* No luck.  */
          /* #### This message SUCKS.  We should see what was the
             reason that nothing was retrieved.  */
          logprintf (LOG_VERBOSE, _("No matches on pattern %s.\n"),
                     quote (u->file));
        }
      else if (action == GLOB_GETONE) /* GLOB_GETONE or GLOB_GETALL */
        {
          /* Let's try retrieving it anyway.  */
          con->st |= ON_YOUR_OWN;
          res = ftp_loop_internal (u, original_url, NULL, con, NULL, false);
          return res;
        }

      /* If action == GLOB_GETALL, and the file list is empty, there's
         no point in trying to download anything or in complaining about
         it.  (An empty directory should not cause complaints.)
      */
    }
  freefileinfo (start);
  if (opt.quota && total_downloaded_bytes > opt.quota)
    return QUOTEXC;
  else
    return res;
}

Project: wget

ftp_loop (struct url *u, struct url *original_url, char **local_file, int *dt,
          struct url *proxy, bool recursive, bool glob)
{
  ccon con;                     /* FTP connection */
  uerr_t res;

  *dt = 0;

  xzero (con);

  con.csock = -1;
  con.st = ON_YOUR_OWN;
  con.rs = ST_UNIX;
  con.id = NULL;
  con.proxy = proxy;

  /* If the file name is empty, the user probably wants a directory
     index.  We'll provide one, properly HTML-ized.  Unless
     opt.htmlify is 0, of course.  :-) */
  if (!*u->file && !recursive)
    {
      struct fileinfo *f;
      res = ftp_get_listing (u, original_url, &con, &f);

      if (res == RETROK)
        {
          if (opt.htmlify && !opt.spider)
            {
              struct url *url_file = opt.trustservernames ? u : original_url;
              char *filename = (opt.output_document
                                ? xstrdup (opt.output_document)
                                : (con.target ? xstrdup (con.target)
                                   : url_file_name (url_file, NULL)));
              res = ftp_index (filename, u, f);
              if (res == FTPOK && opt.verbose)
                {
                  if (!opt.output_document)
                    {
                      struct stat st;
                      wgint sz;
                      if (stat (filename, &st) == 0)
                        sz = st.st_size;
                      else
                        sz = -1;
                      logprintf (LOG_NOTQUIET,
                                 _("Wrote HTML-ized index to %s [%s].\n"),
                                 quote (filename), number_to_static_string (sz));
                    }
                  else
                    logprintf (LOG_NOTQUIET,
                               _("Wrote HTML-ized index to %s.\n"),
                               quote (filename));
                }
              xfree (filename);
            }
          freefileinfo (f);
        }
    }
  else
    {
      bool ispattern = false;
      if (glob)
        {
          /* Treat the URL as a pattern if the file name part of the
             URL path contains wildcards.  (Don't check for u->file
             because it is unescaped and therefore doesn't leave users
             the option to escape literal '*' as %2A.)  */
          char *file_part = strrchr (u->path, '/');
          if (!file_part)
            file_part = u->path;
          ispattern = has_wildcards_p (file_part);
        }
      if (ispattern || recursive || opt.timestamping || opt.preserve_perm)
        {
          /* ftp_retrieve_glob is a catch-all function that gets called
             if we need globbing, time-stamping, recursion or preserve
             permissions.  Its third argument is just what we really need.  */
          res = ftp_retrieve_glob (u, original_url, &con,
                                   ispattern ? GLOB_GLOBALL : GLOB_GETONE);
        }
      else
        {
          res = ftp_loop_internal (u, original_url, NULL, &con, local_file, false);
        }
    }
  if (res == FTPOK)
    res = RETROK;
  if (res == RETROK)
    *dt |= RETROKF;
  /* If a connection was left, quench it.  */
  if (con.csock != -1)
    fd_close (con.csock);
  xfree (con.id);
  xfree (con.target);
  return res;
}

Project: wget

http_loop (const struct url *u, struct url *original_url, char **newloc,
           char **local_file, const char *referer, int *dt, struct url *proxy,
           struct iri *iri)
{
  int count;
  bool got_head = false;         /* used for time-stamping and filename detection */
  bool time_came_from_head = false;
  bool got_name = false;
  char *tms;
  const char *tmrate;
  uerr_t err, ret = TRYLIMEXC;
  time_t tmr = -1;               /* remote time-stamp */
  struct http_stat hstat;        /* HTTP status */
  struct stat st;
  bool send_head_first = true;
  bool force_full_retrieve = false;


  /* If we are writing to a WARC file: always retrieve the whole file. */
  if (opt.warc_filename != NULL)
    force_full_retrieve = true;


  /* Assert that no value for *LOCAL_FILE was passed. */
  assert (local_file == NULL || *local_file == NULL);

  /* Set LOCAL_FILE parameter. */
  if (local_file && opt.output_document)
    *local_file = HYPHENP (opt.output_document) ? NULL : xstrdup (opt.output_document);

  /* Reset NEWLOC parameter. */
  *newloc = NULL;

  /* This used to be done in main, but it's a better idea to do it
     here so that we don't go through the hoops if we're just using
     FTP or whatever. */
  if (opt.cookies)
    load_cookies ();

  /* Warn on (likely bogus) wildcard usage in HTTP. */
  if (opt.ftp_glob && has_wildcards_p (u->path))
    logputs (LOG_VERBOSE, _("Warning: wildcards not supported in HTTP.\n"));

  /* Setup hstat struct. */
  xzero (hstat);
  hstat.referer = referer;

  if (opt.output_document)
    {
      hstat.local_file = xstrdup (opt.output_document);
      got_name = true;
    }
  else if (!opt.content_disposition)
    {
      hstat.local_file =
        url_file_name (opt.trustservernames ? u : original_url, NULL);
      got_name = true;
    }

  if (got_name && file_exists_p (hstat.local_file, NULL) && opt.noclobber && !opt.output_document)
    {
      /* If opt.noclobber is turned on and file already exists, do not
         retrieve the file. But if the output_document was given, then this
         test was already done and the file didn't exist. Hence the !opt.output_document */
      get_file_flags (hstat.local_file, dt);
      ret = RETROK;
      goto exit;
    }

  /* Reset the counter. */
  count = 0;

  /* Reset the document type. */
  *dt = 0;

  /* Skip preliminary HEAD request if we're not in spider mode.  */
  if (!opt.spider)
    send_head_first = false;

  /* Send preliminary HEAD request if --content-disposition and -c are used
     together.  */
  if (opt.content_disposition && opt.always_rest)
    send_head_first = true;

#ifdef HAVE_METALINK
  if (opt.metalink_over_http)
    {
      *dt |= METALINK_METADATA;
      send_head_first = true;
    }
#endif

  if (opt.timestamping)
    {
      /* Use conditional get request if requested
       * and if timestamp is known at this moment.  */
      if (opt.if_modified_since && !send_head_first && got_name && file_exists_p (hstat.local_file, NULL))
        {
          *dt |= IF_MODIFIED_SINCE;
          {
            uerr_t timestamp_err = set_file_timestamp (&hstat);
            if (timestamp_err != RETROK)
              return timestamp_err;
          }
        }
        /* Send preliminary HEAD request if -N is given and we have existing
         * destination file or content disposition is enabled.  */
      else if (opt.content_disposition || file_exists_p (hstat.local_file, NULL))
        send_head_first = true;
    }

  /* THE loop */
  do
    {
      /* Increment the pass counter.  */
      ++count;
      sleep_between_retrievals (count);

      /* Get the current time string.  */
      tms = datetime_str (time (NULL));

      if (opt.spider && !got_head)
        logprintf (LOG_VERBOSE,
			  _("Spider mode enabled. Check if remote file exists.\n"));

      /* Print fetch message, if opt.verbose.  */
      if (opt.verbose)
        {
          char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);

          if (count > 1)
            {
              char tmp[256];
              sprintf (tmp, _("(try:%2d)"), count);
              logprintf (LOG_NOTQUIET, "--%s--  %s  %s\n",
                         tms, tmp, hurl);
            }
          else
            {
              logprintf (LOG_NOTQUIET, "--%s--  %s\n",
                         tms, hurl);
            }

#ifdef WINDOWS
          ws_changetitle (hurl);
#endif
          xfree (hurl);
        }

      /* Default document type is empty.  However, if spider mode is
         on or time-stamping is employed, HEAD_ONLY commands is
         encoded within *dt.  */
      if (send_head_first && !got_head)
        *dt |= HEAD_ONLY;
      else
        *dt &= ~HEAD_ONLY;

      /* Decide whether or not to restart.  */
      if (force_full_retrieve)
        hstat.restval = hstat.len;
      else if (opt.start_pos >= 0)
        hstat.restval = opt.start_pos;
      else if (opt.always_rest
          && got_name
          && stat (hstat.local_file, &st) == 0
          && S_ISREG (st.st_mode))
        /* When -c is used, continue from on-disk size.  (Can't use
           hstat.len even if count>1 because we don't want a failed
           first attempt to clobber existing data.)  */
        hstat.restval = st.st_size;
      else if (count > 1)
        /* otherwise, continue where the previous try left off */
        hstat.restval = hstat.len;
      else
        hstat.restval = 0;

      /* Decide whether to send the no-cache directive.  We send it in
         two cases:
           a) we're using a proxy, and we're past our first retrieval.
              Some proxies are notorious for caching incomplete data, so
              we require a fresh get.
           b) caching is explicitly inhibited. */
      if ((proxy && count > 1)        /* a */
          || !opt.allow_cache)        /* b */
        *dt |= SEND_NOCACHE;
      else
        *dt &= ~SEND_NOCACHE;

      /* Try fetching the document, or at least its head.  */
      err = gethttp (u, original_url, &hstat, dt, proxy, iri, count);

      /* Time?  */
      tms = datetime_str (time (NULL));

      /* Get the new location (with or without the redirection).  */
      if (hstat.newloc)
        *newloc = xstrdup (hstat.newloc);

      switch (err)
        {
        case HERR: case HEOF: case CONSOCKERR:
        case CONERROR: case READERR: case WRITEFAILED:
        case RANGEERR: case FOPEN_EXCL_ERR: case GATEWAYTIMEOUT:
          /* Non-fatal errors continue executing the loop, which will
             bring them to "while" statement at the end, to judge
             whether the number of tries was exceeded.  */
          printwhat (count, opt.ntry);
          xfree (hstat.message);
          xfree (hstat.error);
          continue;
        case FWRITEERR: case FOPENERR:
          /* Another fatal error.  */
          logputs (LOG_VERBOSE, "\n");
          logprintf (LOG_NOTQUIET, _("Cannot write to %s (%s).\n"),
                     quote (hstat.local_file), strerror (errno));
          ret = err;
          goto exit;
        case HOSTERR:
          /* Fatal unless option set otherwise. */
          if ( opt.retry_on_host_error )
            {
              printwhat (count, opt.ntry);
              xfree (hstat.message);
              xfree (hstat.error);
              continue;
            }
          ret = err;
          goto exit;
        case CONIMPOSSIBLE: case PROXERR: case SSLINITFAILED:
        case CONTNOTSUPPORTED: case VERIFCERTERR: case FILEBADFILE:
        case UNKNOWNATTR:
          /* Fatal errors just return from the function.  */
          ret = err;
          goto exit;
        case ATTRMISSING:
          /* A missing attribute in a Header is a fatal Protocol error. */
          logputs (LOG_VERBOSE, "\n");
          logprintf (LOG_NOTQUIET, _("Required attribute missing from Header received.\n"));
          ret = err;
          goto exit;
        case AUTHFAILED:
          logputs (LOG_VERBOSE, "\n");
          logprintf (LOG_NOTQUIET, _("Username/Password Authentication Failed.\n"));
          ret = err;
          goto exit;
        case WARC_ERR:
          /* A fatal WARC error. */
          logputs (LOG_VERBOSE, "\n");
          logprintf (LOG_NOTQUIET, _("Cannot write to WARC file.\n"));
          ret = err;
          goto exit;
        case WARC_TMP_FOPENERR: case WARC_TMP_FWRITEERR:
          /* A fatal WARC error. */
          logputs (LOG_VERBOSE, "\n");
          logprintf (LOG_NOTQUIET, _("Cannot write to temporary WARC file.\n"));
          ret = err;
          goto exit;
        case CONSSLERR:
          /* Another fatal error.  */
          logprintf (LOG_NOTQUIET, _("Unable to establish SSL connection.\n"));
          ret = err;
          goto exit;
        case UNLINKERR:
          /* Another fatal error.  */
          logputs (LOG_VERBOSE, "\n");
          logprintf (LOG_NOTQUIET, _("Cannot unlink %s (%s).\n"),
                     quote (hstat.local_file), strerror (errno));
          ret = err;
          goto exit;
        case NEWLOCATION:
        case NEWLOCATION_KEEP_POST:
          /* Return the new location to the caller.  */
          if (!*newloc)
            {
              logprintf (LOG_NOTQUIET,
                         _("ERROR: Redirection (%d) without location.\n"),
                         hstat.statcode);
              ret = WRONGCODE;
            }
          else
            {
              ret = err;
            }
          goto exit;
        case RETRUNNEEDED:
          /* The file was already fully retrieved. */
          ret = RETROK;
          goto exit;
        case RETRFINISHED:
          /* Deal with you later.  */
          break;
#ifdef HAVE_METALINK
        case RETR_WITH_METALINK:
          {
            if (hstat.metalink == NULL)
              {
                logputs (LOG_NOTQUIET,
                         _("Could not find Metalink data in HTTP response. "
                           "Downloading file using HTTP GET.\n"));
                *dt &= ~METALINK_METADATA;
                *dt &= ~HEAD_ONLY;
                got_head = true;
                continue;
              }

            logputs (LOG_VERBOSE,
                     _("Metalink headers found. "
                       "Switching to Metalink mode.\n"));

            ret = retrieve_from_metalink (hstat.metalink);
            goto exit;
          }
          break;
#endif
        default:
          /* All possibilities should have been exhausted.  */
          abort ();
        }

      if (!(*dt & RETROKF))
        {
          char *hurl = NULL;
          if (!opt.verbose)
            {
              /* #### Ugly ugly ugly! */
              hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
              logprintf (LOG_NONVERBOSE, "%s:\n", hurl);
            }

          /* Fall back to GET if HEAD fails with a 500 or 501 error code. */
          if (*dt & HEAD_ONLY
              && (hstat.statcode == 500 || hstat.statcode == 501))
            {
              got_head = true;
              xfree (hurl);
              continue;
            }
          /* Maybe we should always keep track of broken links, not just in
           * spider mode.
           * Don't log error if it was UTF-8 encoded because we will try
           * once unencoded. */
          else if (opt.spider && !iri->utf8_encode)
            {
              /* #### Again: ugly ugly ugly! */
              if (!hurl)
                hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
              nonexisting_url (hurl);
              logprintf (LOG_NOTQUIET, _("\
Remote file does not exist -- broken link!!!\n"));
            }
          else if (check_retry_on_http_error (hstat.statcode))
            {
              printwhat (count, opt.ntry);
              xfree (hurl);
              continue;
            }
          else
            {
              logprintf (LOG_NOTQUIET, _("%s ERROR %d: %s.\n"),
                         tms, hstat.statcode,
                         quotearg_style (escape_quoting_style, hstat.error));
            }
          logputs (LOG_VERBOSE, "\n");
          ret = WRONGCODE;
          xfree (hurl);
          goto exit;
        }

      /* Did we get the time-stamp? */
      if (!got_head || (opt.spider && !opt.recursive))
        {
          got_head = true;    /* no more time-stamping */

          if (opt.timestamping && !hstat.remote_time)
            {
              logputs (LOG_NOTQUIET, _("\
Last-modified header missing -- time-stamps turned off.\n"));
            }
          else if (hstat.remote_time)
            {
              /* Convert the date-string into struct tm.  */
              tmr = http_atotm (hstat.remote_time);
              if (tmr == (time_t) (-1))
                logputs (LOG_VERBOSE, _("\
Last-modified header invalid -- time-stamp ignored.\n"));
              if (*dt & HEAD_ONLY)
                time_came_from_head = true;
            }

          if (send_head_first)
            {
              /* The time-stamping section.  */
              if (opt.timestamping)
                {
                  if (hstat.orig_file_name) /* Perform the following
                                               checks only if the file
                                               we're supposed to
                                               download already exists.  */
                    {
                      if (hstat.remote_time &&
                          tmr != (time_t) (-1))
                        {
                          /* Now time-stamping can be used validly.
                             Time-stamping means that if the sizes of
                             the local and remote file match, and local
                             file is newer than the remote file, it will
                             not be retrieved.  Otherwise, the normal
                             download procedure is resumed.  */
                          if (hstat.orig_file_tstamp >= tmr)
                            {
                              if (hstat.contlen == -1
                                  || hstat.orig_file_size == hstat.contlen)
                                {
                                  logprintf (LOG_VERBOSE, _("\
Server file no newer than local file %s -- not retrieving.\n\n"),
                                             quote (hstat.orig_file_name));
                                  ret = RETROK;
                                  goto exit;
                                }
                              else
                                {
                                  logprintf (LOG_VERBOSE, _("\
The sizes do not match (local %s) -- retrieving.\n"),
                                             number_to_static_string (hstat.orig_file_size));
                                }
                            }
                          else
                            {
                              force_full_retrieve = true;
                              logputs (LOG_VERBOSE,
                                       _("Remote file is newer, retrieving.\n"));
                            }

                          logputs (LOG_VERBOSE, "\n");
                        }
                    }

                  /* free_hstat (&hstat); */
                  hstat.timestamp_checked = true;
                }

              if (opt.spider)
                {
                  bool finished = true;
                  if (opt.recursive)
                    {
                      if ((*dt & TEXTHTML) || (*dt & TEXTCSS))
                        {
                          logputs (LOG_VERBOSE, _("\
Remote file exists and could contain links to other resources -- retrieving.\n\n"));
                          finished = false;
                        }
                      else
                        {
                          logprintf (LOG_VERBOSE, _("\
Remote file exists but does not contain any link -- not retrieving.\n\n"));
                          ret = RETROK; /* RETRUNNEEDED is not for caller. */
                        }
                    }
                  else
                    {
                      if ((*dt & TEXTHTML) || (*dt & TEXTCSS))
                        {
                          logprintf (LOG_VERBOSE, _("\
Remote file exists and could contain further links,\n\
but recursion is disabled -- not retrieving.\n\n"));
                        }
                      else
                        {
                          logprintf (LOG_VERBOSE, _("\
Remote file exists.\n\n"));
                        }
                      ret = RETROK; /* RETRUNNEEDED is not for caller. */
                    }

                  if (finished)
                    {
                      logprintf (LOG_NONVERBOSE,
                                 _("%s URL: %s %2d %s\n"),
                                 tms, u->url, hstat.statcode,
                                 hstat.message ? quotearg_style (escape_quoting_style, hstat.message) : "");
                      goto exit;
                    }
                }

              got_name = true;
              *dt &= ~HEAD_ONLY;
              count = 0;          /* the retrieve count for HEAD is reset */
              xfree (hstat.message);
              xfree (hstat.error);
              continue;
            } /* send_head_first */
        } /* !got_head */

      if (opt.useservertimestamps
          && (tmr != (time_t) (-1))
          && ((hstat.len == hstat.contlen) ||
              ((hstat.res == 0) && (hstat.contlen == -1))))
        {
          const char *fl = NULL;
          set_local_file (&fl, hstat.local_file);
          if (fl)
            {
              time_t newtmr = -1;
              /* Reparse time header, in case it's changed. */
              if (time_came_from_head
                  && hstat.remote_time && hstat.remote_time[0])
                {
                  newtmr = http_atotm (hstat.remote_time);
                  if (newtmr != (time_t)-1)
                    tmr = newtmr;
                }
              touch (fl, tmr);
            }
        }
      /* End of time-stamping section. */

      tmrate = retr_rate (hstat.rd_size, hstat.dltime);
      total_download_time += hstat.dltime;

      if (hstat.len == hstat.contlen)
        {
          if (*dt & RETROKF || opt.content_on_error)
            {
              bool write_to_stdout = (opt.output_document && HYPHENP (opt.output_document));

              logprintf (LOG_VERBOSE,
                         write_to_stdout
                         ? _("%s (%s) - written to stdout %s[%s/%s]\n\n")
                         : _("%s (%s) - %s saved [%s/%s]\n\n"),
                         tms, tmrate,
                         write_to_stdout ? "" : quote (hstat.local_file),
                         number_to_static_string (hstat.len),
                         number_to_static_string (hstat.contlen));
              logprintf (LOG_NONVERBOSE,
                         "%s URL:%s [%s/%s] -> \"%s\" [%d]\n",
                         tms, u->url,
                         number_to_static_string (hstat.len),
                         number_to_static_string (hstat.contlen),
                         hstat.local_file, count);
            }
          ++numurls;
          total_downloaded_bytes += hstat.rd_size;

          /* Remember that we downloaded the file for later ".orig" code. */
          if (*dt & ADDED_HTML_EXTENSION)
            downloaded_file (FILE_DOWNLOADED_AND_HTML_EXTENSION_ADDED, hstat.local_file);
          else
            downloaded_file (FILE_DOWNLOADED_NORMALLY, hstat.local_file);

          ret = RETROK;
          goto exit;
        }
      else if (hstat.res == 0) /* No read error */
        {
          if (hstat.contlen == -1)  /* We don't know how much we were supposed
                                       to get, so assume we succeeded. */
            {
              if (*dt & RETROKF || opt.content_on_error)
                {
                  bool write_to_stdout = (opt.output_document && HYPHENP (opt.output_document));

                  logprintf (LOG_VERBOSE,
                             write_to_stdout
                             ? _("%s (%s) - written to stdout %s[%s]\n\n")
                             : _("%s (%s) - %s saved [%s]\n\n"),
                             tms, tmrate,
                             write_to_stdout ? "" : quote (hstat.local_file),
                             number_to_static_string (hstat.len));
                  logprintf (LOG_NONVERBOSE,
                             "%s URL:%s [%s] -> \"%s\" [%d]\n",
                             tms, u->url, number_to_static_string (hstat.len),
                             hstat.local_file, count);
                }
              ++numurls;
              total_downloaded_bytes += hstat.rd_size;

              /* Remember that we downloaded the file for later ".orig" code. */
              if (*dt & ADDED_HTML_EXTENSION)
                downloaded_file (FILE_DOWNLOADED_AND_HTML_EXTENSION_ADDED, hstat.local_file);
              else
                downloaded_file (FILE_DOWNLOADED_NORMALLY, hstat.local_file);

              ret = RETROK;
              goto exit;
            }
          else if (hstat.len < hstat.contlen) /* meaning we lost the
                                                 connection too soon */
            {
              logprintf (LOG_VERBOSE,
                         _("%s (%s) - Connection closed at byte %s. "),
                         tms, tmrate, number_to_static_string (hstat.len));
              printwhat (count, opt.ntry);
              continue;
            }
          else if (hstat.len != hstat.restval)
            /* Getting here would mean reading more data than
               requested with content-length, which we never do.  */
            abort ();
          else
            {
              /* Getting here probably means that the content-length was
               * _less_ than the original, local size. We should probably
               * truncate or re-read, or something. FIXME */
              ret = RETROK;
              goto exit;
            }
        }
      else /* from now on hstat.res can only be -1 */
        {
          if (hstat.contlen == -1)
            {
              logprintf (LOG_VERBOSE,
                         _("%s (%s) - Read error at byte %s (%s)."),
                         tms, tmrate, number_to_static_string (hstat.len),
                         hstat.rderrmsg);
              printwhat (count, opt.ntry);
              continue;
            }
          else /* hstat.res == -1 and contlen is given */
            {
              logprintf (LOG_VERBOSE,
                         _("%s (%s) - Read error at byte %s/%s (%s). "),
                         tms, tmrate,
                         number_to_static_string (hstat.len),
                         number_to_static_string (hstat.contlen),
                         hstat.rderrmsg);
              printwhat (count, opt.ntry);
              continue;
            }
        }
      /* not reached */
    }
  while (!opt.ntry || (count < opt.ntry));

exit:
  if ((ret == RETROK || opt.content_on_error) && local_file)
    {
      xfree (*local_file);
      /* Bugfix: Prevent SIGSEGV when hstat.local_file was left NULL
         (i.e. due to opt.content_disposition).  */
      if (hstat.local_file)
        *local_file = xstrdup (hstat.local_file);
    }
  free_hstat (&hstat);

  return ret;
}

Project: wget

ftp_loop_internal (struct url *u, struct url *original_url, struct fileinfo *f,
                   ccon *con, char **local_file, bool force_full_retrieve)
{
  int count, orig_lp;
  wgint restval, len = 0, qtyread = 0;
  char *tms, *locf;
  const char *tmrate = NULL;
  uerr_t err;
  struct stat st;

  /* Declare WARC variables. */
  bool warc_enabled = (opt.warc_filename != NULL);
  FILE *warc_tmp = NULL;
  ip_address *warc_ip = NULL;
  wgint last_expected_bytes = 0;

  /* Get the target, and set the name for the message accordingly. */
  if ((f == NULL) && (con->target))
    {
      /* Explicit file (like ".listing"). */
      locf = con->target;
    }
  else
    {
      /* URL-derived file.  Consider "-O file" name. */
      xfree (con->target);
      con->target = url_file_name (opt.trustservernames || !original_url ? u : original_url, NULL);
      if (!opt.output_document)
        locf = con->target;
      else
        locf = opt.output_document;
    }

  /* If the output_document was given, then this check was already done and
     the file didn't exist. Hence the !opt.output_document */

  /* If we receive .listing file it is necessary to determine system type of the ftp
     server even if opn.noclobber is given. Thus we must ignore opt.noclobber in
     order to establish connection with the server and get system type. */
  if (opt.noclobber && !opt.output_document && file_exists_p (con->target, NULL)
      && !((con->cmd & DO_LIST) && !(con->cmd & DO_RETR)))
    {
      logprintf (LOG_VERBOSE,
                 _("File %s already there; not retrieving.\n"), quote (con->target));
      /* If the file is there, we suppose it's retrieved OK.  */
      return RETROK;
    }

  /* Remove it if it's a link.  */
  remove_link (con->target);

  count = 0;

  if (con->st & ON_YOUR_OWN)
    con->st = ON_YOUR_OWN;

  orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;

  /* THE loop.  */
  do
    {
      /* Increment the pass counter.  */
      ++count;
      sleep_between_retrievals (count);
      if (con->st & ON_YOUR_OWN)
        {
          con->cmd = 0;
          con->cmd |= (DO_RETR | LEAVE_PENDING);
          if (con->csock != -1)
            con->cmd &= ~ (DO_LOGIN | DO_CWD);
          else
            con->cmd |= (DO_LOGIN | DO_CWD);
        }
      else /* not on your own */
        {
          if (con->csock != -1)
            con->cmd &= ~DO_LOGIN;
          else
            con->cmd |= DO_LOGIN;
          if (con->st & DONE_CWD)
            con->cmd &= ~DO_CWD;
          else
            con->cmd |= DO_CWD;
        }

      /* For file RETR requests, we can write a WARC record.
         We record the file contents to a temporary file. */
      if (warc_enabled && (con->cmd & DO_RETR) && warc_tmp == NULL)
        {
          warc_tmp = warc_tempfile ();
          if (warc_tmp == NULL)
            return WARC_TMP_FOPENERR;

          if (!con->proxy && con->csock != -1)
            {
              warc_ip = (ip_address *) alloca (sizeof (ip_address));
              socket_ip_address (con->csock, warc_ip, ENDPOINT_PEER);
            }
        }

      /* Decide whether or not to restart.  */
      if (con->cmd & DO_LIST)
        restval = 0;
      else if (force_full_retrieve)
        restval = 0;
      else if (opt.start_pos >= 0)
        restval = opt.start_pos;
      else if (opt.always_rest
          && stat (locf, &st) == 0
          && S_ISREG (st.st_mode))
        /* When -c is used, continue from on-disk size.  (Can't use
           hstat.len even if count>1 because we don't want a failed
           first attempt to clobber existing data.)  */
        restval = st.st_size;
      else if (count > 1)
        restval = qtyread;          /* start where the previous run left off */
      else
        restval = 0;

      /* Get the current time string.  */
      tms = datetime_str (time (NULL));
      /* Print fetch message, if opt.verbose.  */
      if (opt.verbose)
        {
          char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
          char tmp[256];
          strcpy (tmp, "        ");
          if (count > 1)
            sprintf (tmp, _("(try:%2d)"), count);
          logprintf (LOG_VERBOSE, "--%s--  %s\n  %s => %s\n",
                     tms, hurl, tmp, quote (locf));
#ifdef WINDOWS
          ws_changetitle (hurl);
#endif
          xfree (hurl);
        }
      /* Send getftp the proper length, if fileinfo was provided.  */
      if (f && f->type != FT_SYMLINK)
        len = f->size;
      else
        len = 0;

      /* If we are working on a WARC record, getftp should also write
         to the warc_tmp file. */
      err = getftp (u, original_url, len, &qtyread, restval, con, count,
                    &last_expected_bytes, warc_tmp);

      if (con->csock == -1)
        con->st &= ~DONE_CWD;
      else
        con->st |= DONE_CWD;

      switch (err)
        {
        case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
        case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case FTPNOAUTH: case FTPNOPBSZ: case FTPNOPROT:
        case UNLINKERR: case WARC_TMP_FWRITEERR: case CONSSLERR: case CONTNOTSUPPORTED:
#ifdef HAVE_SSL
          if (err == FTPNOAUTH)
            logputs (LOG_NOTQUIET, "Server does not support AUTH TLS.\n");
          if (opt.ftps_implicit)
            logputs (LOG_NOTQUIET, "Server does not like implicit FTPS connections.\n");
#endif
          /* Fatal errors, give up.  */
          if (warc_tmp != NULL)
            {
              fclose (warc_tmp);
              warc_tmp = NULL;
            }
          return err;
        case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
        case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
        case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
        case FOPEN_EXCL_ERR:
          printwhat (count, opt.ntry);
          /* non-fatal errors */
          if (err == FOPEN_EXCL_ERR)
            {
              /* Re-determine the file name. */
              xfree (con->target);
              con->target = url_file_name (u, NULL);
              locf = con->target;
            }
          continue;
        case FTPRETRINT:
          /* If the control connection was closed, the retrieval
             will be considered OK if f->size == len.  */
          if (!f || qtyread != f->size)
            {
              printwhat (count, opt.ntry);
              continue;
            }
          break;
        case RETRFINISHED:
          /* Great!  */
          break;
        default:
          /* Not as great.  */
          abort ();
        }
      tms = datetime_str (time (NULL));
      if (!opt.spider)
        tmrate = retr_rate (qtyread - restval, con->dltime);

      /* If we get out of the switch above without continue'ing, we've
         successfully downloaded a file.  Remember this fact. */
      downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);

      if (con->st & ON_YOUR_OWN)
        {
          fd_close (con->csock);
          con->csock = -1;
        }
      if (!opt.spider)
        {
          bool write_to_stdout = (opt.output_document && HYPHENP (opt.output_document));

          logprintf (LOG_VERBOSE,
                     write_to_stdout
                     ? _("%s (%s) - written to stdout %s[%s]\n\n")
                     : _("%s (%s) - %s saved [%s]\n\n"),
                     tms, tmrate,
                     write_to_stdout ? "" : quote (locf),
                     number_to_static_string (qtyread));
        }
      if (!opt.verbose && !opt.quiet)
        {
          /* Need to hide the password from the URL.  The `if' is here
             so that we don't do the needless allocation every
             time. */
          char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
          logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
                     tms, hurl, number_to_static_string (qtyread), locf, count);
          xfree (hurl);
        }

      if (warc_enabled && (con->cmd & DO_RETR))
        {
          /* Create and store a WARC resource record for the retrieved file. */
          bool warc_res;

          warc_res = warc_write_resource_record (NULL, u->url, NULL, NULL,
                                                  warc_ip, NULL, warc_tmp, -1);

          if (! warc_res)
            return WARC_ERR;

          /* warc_write_resource_record has also closed warc_tmp. */
          warc_tmp = NULL;
        }

      if (con->cmd & DO_LIST)
        /* This is a directory listing file. */
        {
          if (!opt.remove_listing)
            /* --dont-remove-listing was specified, so do count this towards the
               number of bytes and files downloaded. */
            {
              total_downloaded_bytes += qtyread;
              numurls++;
            }

          /* Deletion of listing files is not controlled by --delete-after, but
             by the more specific option --dont-remove-listing, and the code
             to do this deletion is in another function. */
        }
      else if (!opt.spider)
        /* This is not a directory listing file. */
        {
          /* Unlike directory listing files, don't pretend normal files weren't
             downloaded if they're going to be deleted.  People seeding proxies,
             for instance, may want to know how many bytes and files they've
             downloaded through it. */
          total_downloaded_bytes += qtyread;
          numurls++;

          if (opt.delete_after && !input_file_url (opt.input_filename))
            {
              DEBUGP (("\
Removing file due to --delete-after in ftp_loop_internal():\n"));
              logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
              if (unlink (locf))
                logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
            }
        }

      /* Restore the original leave-pendingness.  */
      if (orig_lp)
        con->cmd |= LEAVE_PENDING;
      else
        con->cmd &= ~LEAVE_PENDING;

      if (local_file)
        *local_file = xstrdup (locf);

      if (warc_tmp != NULL)
        {
          fclose (warc_tmp);
          warc_tmp = NULL;
        }

      return RETROK;
    } while (!opt.ntry || (count < opt.ntry));

  if (con->csock != -1 && (con->st & ON_YOUR_OWN))
    {
      fd_close (con->csock);
      con->csock = -1;
    }

  if (warc_tmp != NULL)
    fclose (warc_tmp);

  return TRYLIMEXC;
}

Project: wget

write_xattr_metadata (const char *name, const char *value, FILE *fp)
{
  int retval = -1;

  if (name && value && fp)
    {
      retval = fsetxattr (fileno (fp), name, value, strlen (value), 0);
      /* FreeBSD's extattr_set_fd returns the length of the extended attribute. */
      retval = (retval < 0) ? retval : 0;
      if (retval)
        DEBUGP (("Failed to set xattr %s.\n", quote(name)));
    }

  return retval;
}

Project: wget

ftp_get_listing (struct url *u, struct url *original_url, ccon *con,
                 struct fileinfo **f)
{
  uerr_t err;
  char *uf;                     /* url file name */
  char *lf;                     /* list file name */
  char *old_target = con->target;

  con->st &= ~ON_YOUR_OWN;
  con->cmd |= (DO_LIST | LEAVE_PENDING);
  con->cmd &= ~DO_RETR;

  /* Find the listing file name.  We do it by taking the file name of
     the URL and replacing the last component with the listing file
     name.  */
  uf = url_file_name (u, NULL);
  lf = file_merge (uf, LIST_FILENAME);
  xfree (uf);
  DEBUGP ((_("Using %s as listing tmp file.\n"), quote (lf)));

  con->target = xstrdup (lf);
  xfree (lf);
  err = ftp_loop_internal (u, original_url, NULL, con, NULL, false);
  lf = xstrdup (con->target);
  xfree (con->target);
  con->target = old_target;

  if (err == RETROK)
    {
      *f = ftp_parse_ls (lf, con->rs);
      if (opt.remove_listing)
        {
          if (unlink (lf))
            logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
          else
            logprintf (LOG_VERBOSE, _("Removed %s.\n"), quote (lf));
        }
    }
  else
    *f = NULL;
  xfree (lf);
  con->cmd &= ~DO_LIST;
  return err;
}

Project: wget

getftp (struct url *u, struct url *original_url,
        wgint passed_expected_bytes, wgint *qtyread,
        wgint restval, ccon *con, int count, wgint *last_expected_bytes,
        FILE *warc_tmp)
{
  int csock, dtsock, local_sock, res;
  uerr_t err = RETROK;          /* appease the compiler */
  FILE *fp = NULL;
  char *respline, *tms;
  const char *user, *passwd, *tmrate;
  int cmd = con->cmd;
  wgint expected_bytes = 0;
  bool got_expected_bytes = false;
  bool rest_failed = false;
  int flags;
  wgint rd_size, previous_rd_size = 0;
  char type_char;
  bool try_again;
  bool list_a_used = false;
#ifdef HAVE_SSL
  enum prot_level prot = (opt.ftps_clear_data_connection ? PROT_CLEAR : PROT_PRIVATE);
  /* these variables tell whether the target server
   * accepts the security extensions (RFC 2228) or not,
   * and whether we're actually using any of them
   * (encryption at the control connection only,
   * or both at control and data connections) */
  bool using_control_security = false, using_data_security = false;
#endif

  assert (con != NULL);
  assert (con->target != NULL);

  /* Debug-check of the sanity of the request by making sure that LIST
     and RETR are never both requested (since we can handle only one
     at a time.  */
  assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
  /* Make sure that at least *something* is requested.  */
  assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);

  *qtyread = restval;

  /* Find the username with priority */
  if (u->user)
    user = u->user;
  else if (opt.user && (opt.use_askpass || opt.ask_passwd))
    user = opt.user;
  else if (opt.ftp_user)
    user = opt.ftp_user;
  else if (opt.user)
    user = opt.user;
  else
    user = NULL;

  /* Find the password with priority */
  if (u->passwd)
    passwd = u->passwd;
  else if (opt.passwd && (opt.use_askpass || opt.ask_passwd))
    passwd = opt.passwd;
  else if (opt.ftp_passwd)
    passwd = opt.ftp_passwd;
  else if (opt.passwd)
    passwd = opt.passwd;
  else
    passwd = NULL;

  /* Check for ~/.netrc if none of the above match */
  if (opt.netrc && (!user || !passwd))
    search_netrc (u->host, (const char **) &user, (const char **) &passwd, 1, NULL);

  if (!user) user = "anonymous";
  if (!passwd) passwd = "-wget@";

  dtsock = -1;
  local_sock = -1;
  con->dltime = 0;

#ifdef HAVE_SSL
  if (u->scheme == SCHEME_FTPS)
    {
      /* Initialize SSL layer first */
      if (!ssl_init ())
        {
          scheme_disable (SCHEME_FTPS);
          logprintf (LOG_NOTQUIET, _("Could not initialize SSL. It will be disabled."));
          err = SSLINITFAILED;
          return err;
        }

      /* If we're using the default FTP port and implicit FTPS was requested,
       * rewrite the port to the default *implicit* FTPS port.
       */
      if (opt.ftps_implicit && u->port == DEFAULT_FTP_PORT)
        {
          DEBUGP (("Implicit FTPS was specified. Rewriting default port to %d.\n", DEFAULT_FTPS_IMPLICIT_PORT));
          u->port = DEFAULT_FTPS_IMPLICIT_PORT;
        }
    }
#endif

  if (!(cmd & DO_LOGIN))
    {
      csock = con->csock;
#ifdef HAVE_SSL
      using_data_security = con->st & DATA_CHANNEL_SECURITY;
#endif
    }
  else                          /* cmd & DO_LOGIN */
    {
      char    *host = con->proxy ? con->proxy->host : u->host;
      int      port = con->proxy ? con->proxy->port : u->port;

      /* Login to the server: */

      /* First: Establish the control connection.  */

      csock = connect_to_host (host, port);
      if (csock == E_HOST)
          return HOSTERR;
      else if (csock < 0)
          return (retryable_socket_connect_error (errno)
                  ? CONERROR : CONIMPOSSIBLE);

      if (cmd & LEAVE_PENDING)
        con->csock = csock;
      else
        con->csock = -1;

#ifdef HAVE_SSL
      if (u->scheme == SCHEME_FTPS)
        {
          /* If we're in implicit FTPS mode, we have to set up SSL/TLS before everything else.
           * Otherwise we first read the server's greeting, and then send an "AUTH TLS".
           */
          if (opt.ftps_implicit)
            {
              err = init_control_ssl_connection (csock, u, &using_control_security);
              if (err != NOCONERROR)
                return err;
              err = get_ftp_greeting (csock, con);
              if (err != FTPOK)
                return err;
            }
          else
            {
              err = get_ftp_greeting (csock, con);
              if (err != FTPOK)
                return err;
              err = init_control_ssl_connection (csock, u, &using_control_security);
              if (err != NOCONERROR)
                return err;
            }
        }
      else
        {
          err = get_ftp_greeting (csock, con);
          if (err != FTPOK)
            return err;
        }
#else
      err = get_ftp_greeting (csock, con);
      if (err != FTPOK)
        return err;
#endif

      /* Second: Login with proper USER/PASS sequence.  */
      logprintf (LOG_VERBOSE, _("Logging in as %s ... "),
                 quotearg_style (escape_quoting_style, user));
      if (opt.server_response)
        logputs (LOG_ALWAYS, "\n");
      if (con->proxy)
        {
          /* If proxy is in use, log in as username@target-site. */
          char *logname = concat_strings (user, "@", u->host, (char *) 0);
          err = ftp_login (csock, logname, passwd);
          xfree (logname);
        }
      else
        err = ftp_login (csock, user, passwd);

      /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
      switch (err)
        {
        case FTPRERR:
          logputs (LOG_VERBOSE, "\n");
          logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
          fd_close (csock);
          con->csock = -1;
          return err;
        case FTPSRVERR:
          logputs (LOG_VERBOSE, "\n");
          logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
          fd_close (csock);
          con->csock = -1;
          return err;
        case WRITEFAILED:
          logputs (LOG_VERBOSE, "\n");
          logputs (LOG_NOTQUIET,
                   _("Write failed, closing control connection.\n"));
          fd_close (csock);
          con->csock = -1;
          return err;
        case FTPLOGREFUSED:
          logputs (LOG_VERBOSE, "\n");
          logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
          fd_close (csock);
          con->csock = -1;
          return FTPLOGREFUSED;
        case FTPLOGINC:
          logputs (LOG_VERBOSE, "\n");
          logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
          fd_close (csock);
          con->csock = -1;
          return FTPLOGINC;
        case FTPOK:
          if (!opt.server_response)
            logputs (LOG_VERBOSE, _("Logged in!\n"));
          break;
        default:
          abort ();
        }

#ifdef HAVE_SSL
      if (using_control_security)
        {
          /* Send the PBSZ and PROT commands, in that order.
           * If we are here it means that the server has already accepted
           * some form of FTPS. Thus, these commands must work.
           * If they don't work, that's an error. There's no sense in honoring
           * --ftps-fallback-to-ftp or similar options. */
          if (u->scheme == SCHEME_FTPS)
            {
              if (!opt.server_response)
                logputs (LOG_VERBOSE, "==> PBSZ 0 ... ");
              if ((err = ftp_pbsz (csock, 0)) == FTPNOPBSZ)
                {
                  logputs (LOG_NOTQUIET, _("Server did not accept the 'PBSZ 0' command.\n"));
                  return err;
                }
              if (!opt.server_response)
                logputs (LOG_VERBOSE, "done.");

              if (!opt.server_response)
                logprintf (LOG_VERBOSE, "  ==> PROT %c ... ", (int) prot);
              if ((err = ftp_prot (csock, prot)) == FTPNOPROT)
                {
                  logprintf (LOG_NOTQUIET, _("Server did not accept the 'PROT %c' command.\n"), (int) prot);
                  return err;
                }
              if (!opt.server_response)
                logputs (LOG_VERBOSE, "done.\n");

              if (prot != PROT_CLEAR)
                {
                  using_data_security = true;
                  con->st |= DATA_CHANNEL_SECURITY;
                }
            }
        }
#endif

      /* Third: Get the system type */
      if (!opt.server_response)
        logprintf (LOG_VERBOSE, "==> SYST ... ");
      err = ftp_syst (csock, &con->rs, &con->rsu);
      /* FTPRERR */
      switch (err)
        {
        case FTPRERR:
          logputs (LOG_VERBOSE, "\n");
          logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
          fd_close (csock);
          con->csock = -1;
          return err;
        case FTPSRVERR:
          logputs (LOG_VERBOSE, "\n");
          logputs (LOG_NOTQUIET,
                   _("Server error, can't determine system type.\n"));
          break;
        case FTPOK:
          /* Everything is OK.  */
          break;
        default:
          abort ();
        }
      if (!opt.server_response && err != FTPSRVERR)
        logputs (LOG_VERBOSE, _("done.    "));

      /* 2013-10-17 Andrea Urbani (matfanjol)
         According to the system type I choose which
         list command will be used.
         If I don't know that system, I will try, the
         first time of each session, "LIST -a" and
         "LIST". (see __LIST_A_EXPLANATION__ below) */
      switch (con->rs)
        {
        case ST_VMS:
          /* About ST_VMS there is an old note:
             2008-01-29  SMS.  For a VMS FTP server, where "LIST -a" may not
             fail, but will never do what is desired here,
             skip directly to the simple "LIST" command
             (assumed to be the last one in the list).  */
          DEBUGP (("\nVMS: I know it and I will use \"LIST\" as standard list command\n"));
          con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
          con->st |= AVOID_LIST_A;
          break;
        case ST_UNIX:
          if (con->rsu == UST_MULTINET)
            {
              DEBUGP (("\nUNIX MultiNet: I know it and I will use \"LIST\" "
                       "as standard list command\n"));
              con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
              con->st |= AVOID_LIST_A;
            }
          else if (con->rsu == UST_TYPE_L8)
            {
              DEBUGP (("\nUNIX TYPE L8: I know it and I will use \"LIST -a\" "
                       "as standard list command\n"));
              con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
              con->st |= AVOID_LIST;
            }
          break;
        default:
          break;
        }

      /* Fourth: Find the initial ftp directory */

      if (!opt.server_response)
        logprintf (LOG_VERBOSE, "==> PWD ... ");
      err = ftp_pwd (csock, &con->id);
      /* FTPRERR */
      switch (err)
        {
        case FTPRERR:
          logputs (LOG_VERBOSE, "\n");
          logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
          fd_close (csock);
          con->csock = -1;
          return err;
        case FTPSRVERR :
          /* PWD unsupported -- assume "/". */
          xfree (con->id);
          con->id = xstrdup ("/");
          break;
        case FTPOK:
          /* Everything is OK.  */
          break;
        default:
          abort ();
        }

#if 0
      /* 2004-09-17 SMS.
         Don't help me out.  Please.
         A reasonably recent VMS FTP server will cope just fine with
         UNIX file specifications.  This code just spoils things.
         Discarding the device name, for example, is not a wise move.
         This code was disabled but left in as an example of what not
         to do.
      */

      /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
         Convert it to "/INITIAL/FOLDER" */
      if (con->rs == ST_VMS)
        {
          char *path = strchr (con->id, '[');
          char *pathend = path ? strchr (path + 1, ']') : NULL;
          if (!path || !pathend)
            DEBUGP (("Initial VMS directory not in the form [...]!\n"));
          else
            {
              char *idir = con->id;
              DEBUGP (("Preprocessing the initial VMS directory\n"));
              DEBUGP (("  old = '%s'\n", con->id));
              /* We do the conversion in-place by copying the stuff
                 between [ and ] to the beginning, and changing dots
                 to slashes at the same time.  */
              *idir++ = '/';
              for (++path; path < pathend; path++, idir++)
                *idir = *path == '.' ? '/' : *path;
              *idir = '\0';
              DEBUGP (("  new = '%s'\n\n", con->id));
            }
        }
#endif /* 0 */

      if (!opt.server_response)
        logputs (LOG_VERBOSE, _("done.\n"));

      /* Fifth: Set the FTP type.  */
      type_char = ftp_process_type (u->params);
      if (!opt.server_response)
        logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
      err = ftp_type (csock, type_char);
      /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
      switch (err)
        {
        case FTPRERR:
          logputs (LOG_VERBOSE, "\n");
          logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
          fd_close (csock);
          con->csock = -1;
          return err;
        case WRITEFAILED:
          logputs (LOG_VERBOSE, "\n");
          logputs (LOG_NOTQUIET,
                   _("Write failed, closing control connection.\n"));
          fd_close (csock);
          con->csock = -1;
          return err;
        case FTPUNKNOWNTYPE:
          logputs (LOG_VERBOSE, "\n");
          logprintf (LOG_NOTQUIET,
                     _("Unknown type `%c', closing control connection.\n"),
                     type_char);
          fd_close (csock);
          con->csock = -1;
          return err;
        case FTPOK:
          /* Everything is OK.  */
          break;
        default:
          abort ();
        }
      if (!opt.server_response)
        logputs (LOG_VERBOSE, _("done.  "));
    } /* do login */

  if (cmd & DO_CWD)
    {
      if (!*u->dir)
        logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
      else
        {
          const char *targ = NULL;
          int cwd_count;
          int cwd_end;
          int cwd_start;

          char *target = u->dir;

          DEBUGP (("changing working directory\n"));

          /* Change working directory.  To change to a non-absolute
             Unix directory, we need to prepend initial directory
             (con->id) to it.  Absolute directories "just work".

             A relative directory is one that does not begin with '/'
             and, on non-Unix OS'es, one that doesn't begin with
             "[a-z]:".

             This is not done for OS400, which doesn't use
             "/"-delimited directories, nor does it support directory
             hierarchies.  "CWD foo" followed by "CWD bar" leaves us
             in "bar", not in "foo/bar", as would be customary
             elsewhere.  */

            /* 2004-09-20 SMS.
               Why is this wise even on UNIX?  It certainly fouls VMS.
               See below for a more reliable, more universal method.
            */

            /* 2008-04-22 MJC.
               I'm not crazy about it either. I'm informed it's useful
               for misconfigured servers that have some dirs in the path
               with +x but -r, but this method is not RFC-conformant. I
               understand the need to deal with crappy server
               configurations, but it's far better to use the canonical
               method first, and fall back to kludges second.
            */

          if (target[0] != '/'
              && !(con->rs != ST_UNIX
                   && c_isalpha (target[0])
                   && target[1] == ':')
              && (con->rs != ST_OS400)
              && (con->rs != ST_VMS))
            {
              int idlen = strlen (con->id);
              char *ntarget, *p;

              /* Strip trailing slash(es) from con->id. */
              while (idlen > 0 && con->id[idlen - 1] == '/')
                --idlen;
              p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
              memcpy (p, con->id, idlen);
              p += idlen;
              *p++ = '/';
              strcpy (p, target);

              DEBUGP (("Prepended initial PWD to relative path:\n"));
              DEBUGP (("   pwd: '%s'\n   old: '%s'\n  new: '%s'\n",
                       con->id, target, ntarget));
              target = ntarget;
            }

#if 0
          /* 2004-09-17 SMS.
             Don't help me out.  Please.
             A reasonably recent VMS FTP server will cope just fine with
             UNIX file specifications.  This code just spoils things.
             Discarding the device name, for example, is not a wise
             move.
             This code was disabled but left in as an example of what
             not to do.
          */

          /* If the FTP host runs VMS, we will have to convert the absolute
             directory path in UNIX notation to absolute directory path in
             VMS notation as VMS FTP servers do not like UNIX notation of
             absolute paths.  "VMS notation" is [dir.subdir.subsubdir]. */

          if (con->rs == ST_VMS)
            {
              char *tmpp;
              char *ntarget = (char *)alloca (strlen (target) + 2);
              /* We use a converted initial dir, so directories in
                 TARGET will be separated with slashes, something like
                 "/INITIAL/FOLDER/DIR/SUBDIR".  Convert that to
                 "[INITIAL.FOLDER.DIR.SUBDIR]".  */
              strcpy (ntarget, target);
              assert (*ntarget == '/');
              *ntarget = '[';
              for (tmpp = ntarget + 1; *tmpp; tmpp++)
                if (*tmpp == '/')
                  *tmpp = '.';
              *tmpp++ = ']';
              *tmpp = '\0';
              DEBUGP (("Changed file name to VMS syntax:\n"));
              DEBUGP (("  Unix: '%s'\n  VMS: '%s'\n", target, ntarget));
              target = ntarget;
            }
#endif /* 0 */

          /* 2004-09-20 SMS.
             A relative directory is relative to the initial directory.
             Thus, what _is_ useful on VMS (and probably elsewhere) is
             to CWD to the initial directory (ideally, whatever the
             server reports, _exactly_, NOT badly UNIX-ixed), and then
             CWD to the (new) relative directory.  This should probably
             be restructured as a function, called once or twice, but
             I'm lazy enough to take the badly indented loop short-cut
             for now.
          */

          /* Decide on one pass (absolute) or two (relative).
             The VMS restriction may be relaxed when the squirrely code
             above is reformed.
          */
          if ((con->rs == ST_VMS) && (target[0] != '/'))
            {
              cwd_start = 0;
              DEBUGP (("Using two-step CWD for relative path.\n"));
            }
          else
            {
              /* Go straight to the target. */
              cwd_start = 1;
            }

          /* At least one VMS FTP server (TCPware V5.6-2) can switch to
             a UNIX emulation mode when given a UNIX-like directory
             specification (like "a/b/c").  If allowed to continue this
             way, LIST interpretation will be confused, because the
             system type (SYST response) will not be re-checked, and
             future UNIX-format directory listings (for multiple URLs or
             "-r") will be horribly misinterpreted.

             The cheap and nasty work-around is to do a "CWD []" after a
             UNIX-like directory specification is used.  (A single-level
             directory is harmless.)  This puts the TCPware server back
             into VMS mode, and does no harm on other servers.

             Unlike the rest of this block, this particular behavior
             _is_ VMS-specific, so it gets its own VMS test.
          */
          if ((con->rs == ST_VMS) && (strchr (target, '/') != NULL))
            {
              cwd_end = 3;
              DEBUGP (("Using extra \"CWD []\" step for VMS server.\n"));
            }
          else
            {
              cwd_end = 2;
            }

          /* 2004-09-20 SMS. */
          /* Sorry about the deviant indenting.  Laziness. */

          for (cwd_count = cwd_start; cwd_count < cwd_end; cwd_count++)
            {
              switch (cwd_count)
                {
                  case 0:
                    /* Step one (optional): Go to the initial directory,
                       exactly as reported by the server.
                    */
                    targ = con->id;
                    break;

                  case 1:
                    /* Step two: Go to the target directory.  (Absolute or
                       relative will work now.)
                    */
                    targ = target;
                    break;

                  case 2:
                    /* Step three (optional): "CWD []" to restore server
                       VMS-ness.
                    */
                    targ = "[]";
                    break;

                  default:
                    logprintf (LOG_ALWAYS, _("Logically impossible section reached in getftp()"));
                    logprintf (LOG_ALWAYS, _("cwd_count: %d\ncwd_start: %d\ncwd_end: %d\n"),
                                             cwd_count, cwd_start, cwd_end);
                    abort ();
                }

              if (!opt.server_response)
                logprintf (LOG_VERBOSE, "==> CWD (%d) %s ... ", cwd_count,
                           quotearg_style (escape_quoting_style, target));

              err = ftp_cwd (csock, targ);

              /* FTPRERR, WRITEFAILED, FTPNSFOD */
              switch (err)
                {
                  case FTPRERR:
                    logputs (LOG_VERBOSE, "\n");
                    logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
                    fd_close (csock);
                    con->csock = -1;
                    return err;
                  case WRITEFAILED:
                    logputs (LOG_VERBOSE, "\n");
                    logputs (LOG_NOTQUIET,
                             _("Write failed, closing control connection.\n"));
                    fd_close (csock);
                    con->csock = -1;
                    return err;
                  case FTPNSFOD:
                    logputs (LOG_VERBOSE, "\n");
                    logprintf (LOG_NOTQUIET, _("No such directory %s.\n\n"),
                               quote (u->dir));
                    fd_close (csock);
                    con->csock = -1;
                    return err;
                  case FTPOK:
                    break;
                  default:
                    abort ();
                }

              if (!opt.server_response)
                logputs (LOG_VERBOSE, _("done.\n"));

            } /* for */

          /* 2004-09-20 SMS. */

        } /* else */
    }
  else /* do not CWD */
    logputs (LOG_VERBOSE, _("==> CWD not required.\n"));

  if ((cmd & DO_RETR) && passed_expected_bytes == 0)
    {
      if (opt.verbose)
        {
          if (!opt.server_response)
            logprintf (LOG_VERBOSE, "==> SIZE %s ... ",
                       quotearg_style (escape_quoting_style, u->file));
        }

      err = ftp_size (csock, u->file, &expected_bytes);
      /* FTPRERR */
      switch (err)
        {
        case FTPRERR:
        case FTPSRVERR:
          logputs (LOG_VERBOSE, "\n");
          logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
          fd_close (csock);
          con->csock = -1;
          return err;
        case FTPOK:
          got_expected_bytes = true;
          /* Everything is OK.  */
          break;
        default:
          abort ();
        }
        if (!opt.server_response)
          {
            logprintf (LOG_VERBOSE, "%s\n",
                    expected_bytes ?
                    number_to_static_string (expected_bytes) :
                    _("done.\n"));
          }
    }

  if (cmd & DO_RETR && restval > 0 && restval == expected_bytes)
    {
      /* Server confirms that file has length restval. We should stop now.
         Some servers (f.e. NcFTPd) return error when receive REST 0 */
      logputs (LOG_VERBOSE, _("File has already been retrieved.\n"));
      fd_close (csock);
      con->csock = -1;
      return RETRFINISHED;
    }

  do
  {
  try_again = false;
  /* If anything is to be retrieved, PORT (or PASV) must be sent.  */
  if (cmd & (DO_LIST | DO_RETR))
    {
      if (opt.ftp_pasv)
        {
          ip_address passive_addr;
          int        passive_port;
          err = ftp_do_pasv (csock, &passive_addr, &passive_port);
          /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
          switch (err)
            {
            case FTPRERR:
              logputs (LOG_VERBOSE, "\n");
              logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
              fd_close (csock);
              con->csock = -1;
              return err;
            case WRITEFAILED:
              logputs (LOG_VERBOSE, "\n");
              logputs (LOG_NOTQUIET,
                       _("Write failed, closing control connection.\n"));
              fd_close (csock);
              con->csock = -1;
              return err;
            case FTPNOPASV:
              logputs (LOG_VERBOSE, "\n");
              logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
              break;
            case FTPINVPASV:
              logputs (LOG_VERBOSE, "\n");
              logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
              break;
            case FTPOK:
              break;
            default:
              abort ();
            }   /* switch (err) */
          if (err==FTPOK)
            {
              DEBUGP (("trying to connect to %s port %d\n",
                      print_address (&passive_addr), passive_port));
              dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
              if (dtsock < 0)
                {
                  int save_errno = errno;
                  fd_close (csock);
                  con->csock = -1;
                  logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
                             print_address (&passive_addr), passive_port,
                             strerror (save_errno));
                  return (retryable_socket_connect_error (save_errno)
                          ? CONERROR : CONIMPOSSIBLE);
                }

              if (!opt.server_response)
                logputs (LOG_VERBOSE, _("done.    "));
            }
          else
            return err;

          /*
           * We do not want to fall back from PASSIVE mode to ACTIVE mode !
           * The reason is the PORT command exposes the client's real IP address
           * to the server. Bad for someone who relies on privacy via a ftp proxy.
           */
        }
      else
        {
          err = ftp_do_port (csock, &local_sock);
          /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
             FTPPORTERR */
          switch (err)
            {
            case FTPRERR:
              logputs (LOG_VERBOSE, "\n");
              logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
              fd_close (csock);
              con->csock = -1;
              fd_close (dtsock);
              fd_close (local_sock);
              return err;
            case WRITEFAILED:
              logputs (LOG_VERBOSE, "\n");
              logputs (LOG_NOTQUIET,
                       _("Write failed, closing control connection.\n"));
              fd_close (csock);
              con->csock = -1;
              fd_close (dtsock);
              fd_close (local_sock);
              return err;
            case CONSOCKERR:
              logputs (LOG_VERBOSE, "\n");
              logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
              fd_close (csock);
              con->csock = -1;
              fd_close (dtsock);
              fd_close (local_sock);
              return err;
            case FTPSYSERR:
              logputs (LOG_VERBOSE, "\n");
              logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
                         strerror (errno));
              fd_close (dtsock);
              return err;
            case FTPPORTERR:
              logputs (LOG_VERBOSE, "\n");
              logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
              fd_close (csock);
              con->csock = -1;
              fd_close (dtsock);
              fd_close (local_sock);
              return err;
            case FTPOK:
              break;
            default:
              abort ();
            } /* port switch */
          if (!opt.server_response)
            logputs (LOG_VERBOSE, _("done.    "));
        } /* dtsock == -1 */
    } /* cmd & (DO_LIST | DO_RETR) */

  /* Restart if needed.  */
  if (restval && (cmd & DO_RETR))
    {
      if (!opt.server_response)
        logprintf (LOG_VERBOSE, "==> REST %s ... ",
                   number_to_static_string (restval));
      err = ftp_rest (csock, restval);

      /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
      switch (err)
        {
        case FTPRERR:
          logputs (LOG_VERBOSE, "\n");
          logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
          fd_close (csock);
          con->csock = -1;
          fd_close (dtsock);
          fd_close (local_sock);
          return err;
        case WRITEFAILED:
          logputs (LOG_VERBOSE, "\n");
          logputs (LOG_NOTQUIET,
                   _("Write failed, closing control connection.\n"));
          fd_close (csock);
          con->csock = -1;
          fd_close (dtsock);
          fd_close (local_sock);
          return err;
        case FTPRESTFAIL:
          logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
          rest_failed = true;
          break;
        case FTPOK:
          break;
        default:
          abort ();
        }
      if (err != FTPRESTFAIL && !opt.server_response)
        logputs (LOG_VERBOSE, _("done.    "));
    } /* restval && cmd & DO_RETR */

  if (cmd & DO_RETR)
    {
      /* If we're in spider mode, don't really retrieve anything except
         the directory listing and verify whether the given "file" exists.  */
      if (opt.spider)
        {
          bool exists = false;
          bool all_exist = true;
          struct fileinfo *f;
          uerr_t _res = ftp_get_listing (u, original_url, con, &f);
          /* Set the DO_RETR command flag again, because it gets unset when
             calling ftp_get_listing() and would otherwise cause an assertion
             failure earlier on when this function gets repeatedly called
             (e.g., when recursing).  */
          con->cmd |= DO_RETR;
          if (_res == RETROK)
            {
              while (f)
                {
                  if (!strcmp (f->name, u->file))
                    {
                      exists = true;
                      break;
                    } else {
                      all_exist = false;
                    }
                  f = f->next;
                }
              if (exists)
                {
                  logputs (LOG_VERBOSE, "\n");
                  logprintf (LOG_NOTQUIET, _("File %s exists.\n"),
                             quote (u->file));
                }
              else
                {
                  logputs (LOG_VERBOSE, "\n");
                  logprintf (LOG_NOTQUIET, _("No such file %s.\n"),
                             quote (u->file));
                }
            }
          fd_close (csock);
          con->csock = -1;
          fd_close (dtsock);
          fd_close (local_sock);
          if (all_exist) {
              return RETRFINISHED;
          } else {
              return FTPNSFOD;
          }
        }

      if (opt.verbose)
        {
          if (!opt.server_response)
            {
              if (restval)
                logputs (LOG_VERBOSE, "\n");
              logprintf (LOG_VERBOSE, "==> RETR %s ... ",
                         quotearg_style (escape_quoting_style, u->file));
            }
        }

      err = ftp_retr (csock, u->file);
      /* FTPRERR, WRITEFAILED, FTPNSFOD */
      switch (err)
        {
        case FTPRERR:
          logputs (LOG_VERBOSE, "\n");
          logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
          fd_close (csock);
          con->csock = -1;
          fd_close (dtsock);
          fd_close (local_sock);
          return err;
        case WRITEFAILED:
          logputs (LOG_VERBOSE, "\n");
          logputs (LOG_NOTQUIET,
                   _("Write failed, closing control connection.\n"));
          fd_close (csock);
          con->csock = -1;
          fd_close (dtsock);
          fd_close (local_sock);
          return err;
        case FTPNSFOD:
          logputs (LOG_VERBOSE, "\n");
          logprintf (LOG_NOTQUIET, _("No such file %s.\n\n"),
                     quote (u->file));
          fd_close (dtsock);
          fd_close (local_sock);
          return err;
        case FTPOK:
          break;
        default:
          abort ();
        }

      if (!opt.server_response)
        logputs (LOG_VERBOSE, _("done.\n"));

      if (! got_expected_bytes)
        expected_bytes = *last_expected_bytes;
    } /* do retrieve */

  if (cmd & DO_LIST)
    {
      if (!opt.server_response)
        logputs (LOG_VERBOSE, "==> LIST ... ");
      /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
         without arguments is better than `LIST .'; confirmed by
         RFC959.  */
      err = ftp_list (csock, NULL, con->st&AVOID_LIST_A, con->st&AVOID_LIST, &list_a_used);

      /* FTPRERR, WRITEFAILED */
      switch (err)
        {
        case FTPRERR:
          logputs (LOG_VERBOSE, "\n");
          logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
          fd_close (csock);
          con->csock = -1;
          fd_close (dtsock);
          fd_close (local_sock);
          return err;
        case WRITEFAILED:
          logputs (LOG_VERBOSE, "\n");
          logputs (LOG_NOTQUIET,
                   _("Write failed, closing control connection.\n"));
          fd_close (csock);
          con->csock = -1;
          fd_close (dtsock);
          fd_close (local_sock);
          return err;
        case FTPNSFOD:
          logputs (LOG_VERBOSE, "\n");
          logprintf (LOG_NOTQUIET, _("No such file or directory %s.\n\n"),
                     quote ("."));
          fd_close (dtsock);
          fd_close (local_sock);
          return err;
        case FTPOK:
          break;
        default:
          abort ();
        }
      if (!opt.server_response)
        logputs (LOG_VERBOSE, _("done.\n"));

      if (! got_expected_bytes)
        expected_bytes = *last_expected_bytes;
    } /* cmd & DO_LIST */

  if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
    return RETRFINISHED;

  /* Some FTP servers return the total length of file after REST
     command, others just return the remaining size. */
  if (passed_expected_bytes && restval && expected_bytes
      && (expected_bytes == passed_expected_bytes - restval))
    {
      DEBUGP (("Lying FTP server found, adjusting.\n"));
      expected_bytes = passed_expected_bytes;
    }

  /* If no transmission was required, then everything is OK.  */
  if (!opt.ftp_pasv)  /* we are not using passive mode so we need
                         to accept */
    {
      /* Wait for the server to connect to the address we're waiting
         at.  */
      dtsock = accept_connection (local_sock);
      if (dtsock < 0)
        {
          logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
          return CONERROR;
        }
    }

  /* Open the file -- if output_stream is set, use it instead.  */

  /* 2005-04-17 SMS.
     Note that having the output_stream ("-O") file opened in main
     (main.c) rather limits the ability in VMS to open the file
     differently for ASCII versus binary FTP here.  (Of course, doing it
     there allows a open failure to be detected immediately, without first
     connecting to the server.)
  */
  if (!output_stream || con->cmd & DO_LIST)
    {
/* On VMS, alter the name as required. */
#ifdef __VMS
      char *targ;

      targ = ods_conform (con->target);
      if (targ != con->target)
        {
          xfree (con->target);
          con->target = targ;
        }
#endif /* def __VMS */

      mkalldirs (con->target);
      if (opt.backups)
        rotate_backups (con->target);

/* 2005-04-15 SMS.
   For VMS, define common fopen() optional arguments, and a handy macro
   for use as a variable "binary" flag.
   Elsewhere, define a constant "binary" flag.
   Isn't it nice to have distinct text and binary file types?
*/
/* 2011-09-30 SMS.
   Added listing files to the set of non-"binary" (text, Stream_LF)
   files.  (Wget works either way, but other programs, like, say, text
   editors, work better on listing files which have text attributes.)
   Now we use "binary" attributes for a binary ("IMAGE") transfer,
   unless "--ftp-stmlf" was specified, and we always use non-"binary"
   (text, Stream_LF) attributes for a listing file, or for an ASCII
   transfer.
   Tidied the VMS-specific BIN_TYPE_xxx macros, and changed the call to
   fopen_excl() (restored?) to use BIN_TYPE_FILE instead of "true".
*/
#ifdef __VMS
# define BIN_TYPE_TRANSFER (type_char != 'A')
# define BIN_TYPE_FILE \
   ((!(cmd & DO_LIST)) && BIN_TYPE_TRANSFER && (opt.ftp_stmlf == 0))
# define FOPEN_OPT_ARGS "fop=sqo", "acc", acc_cb, &open_id
# define FOPEN_OPT_ARGS_BIN "ctx=bin,stm", "rfm=fix", "mrs=512" FOPEN_OPT_ARGS
#else /* def __VMS */
# define BIN_TYPE_FILE true
#endif /* def __VMS [else] */

      if (restval && !(con->cmd & DO_LIST))
        {
#ifdef __VMS
          int open_id;

          if (BIN_TYPE_FILE)
            {
              open_id = 3;
              fp = fopen (con->target, "ab", FOPEN_OPT_ARGS_BIN);
            }
          else
            {
              open_id = 4;
              fp = fopen (con->target, "a", FOPEN_OPT_ARGS);
            }
#else /* def __VMS */
          fp = fopen (con->target, "ab");
#endif /* def __VMS [else] */
        }
      else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
               || opt.output_document || count > 0)
        {
          if (opt.unlink_requested && file_exists_p (con->target, NULL))
            {
              if (unlink (con->target) < 0)
                {
                  logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
                    strerror (errno));
                    fd_close (csock);
                    con->csock = -1;
                    fd_close (dtsock);
                    fd_close (local_sock);
                    return UNLINKERR;
                }
            }

#ifdef __VMS
          int open_id;

          if (BIN_TYPE_FILE)
            {
              open_id = 5;
              fp = fopen (con->target, "wb", FOPEN_OPT_ARGS_BIN);
            }
          else
            {
              open_id = 6;
              fp = fopen (con->target, "w", FOPEN_OPT_ARGS);
            }
#else /* def __VMS */
          fp = fopen (con->target, "wb");
#endif /* def __VMS [else] */
        }
      else
        {
          fp = fopen_excl (con->target, BIN_TYPE_FILE);
          if (!fp && errno == EEXIST)
            {
              /* We cannot just invent a new name and use it (which is
                 what functions like unique_create typically do)
                 because we told the user we'd use this name.
                 Instead, return and retry the download.  */
              logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
                         con->target);
              fd_close (csock);
              con->csock = -1;
              fd_close (dtsock);
              fd_close (local_sock);
              return FOPEN_EXCL_ERR;
            }
        }
      if (!fp)
        {
          logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
          fd_close (csock);
          con->csock = -1;
          fd_close (dtsock);
          fd_close (local_sock);
          return FOPENERR;
        }
    }
  else
    fp = output_stream;

  if (passed_expected_bytes)
    {
      print_length (passed_expected_bytes, restval, true);
      expected_bytes = passed_expected_bytes;
        /* for fd_read_body's progress bar */
    }
  else if (expected_bytes)
    print_length (expected_bytes, restval, false);

#ifdef HAVE_SSL
  if (u->scheme == SCHEME_FTPS && using_data_security)
    {
      /* We should try to restore the existing SSL session in the data connection
       * and fall back to establishing a new session if the server doesn't want to restore it.
       */
      if (!opt.ftps_resume_ssl || !ssl_connect_wget (dtsock, u->host, &csock))
        {
          if (opt.ftps_resume_ssl)
            logputs (LOG_NOTQUIET, "Server does not want to resume the SSL session. Trying with a new one.\n");
          if (!ssl_connect_wget (dtsock, u->host, NULL))
            {
              fd_close (csock);
              fd_close (dtsock);
              err = CONERROR;
              logputs (LOG_NOTQUIET, "Could not perform SSL handshake.\n");
              goto exit_error;
            }
        }
      else
        logputs (LOG_NOTQUIET, "Resuming SSL session in data connection.\n");

      if (!ssl_check_certificate (dtsock, u->host))
        {
          fd_close (csock);
          fd_close (dtsock);
          err = CONERROR;
          goto exit_error;
        }
    }
#endif

  /* Get the contents of the document.  */
  flags = 0;
  if (restval && rest_failed)
    flags |= rb_skip_startpos;
  rd_size = 0;
  res = fd_read_body (con->target, dtsock, fp,
                      expected_bytes ? expected_bytes - restval : 0,
                      restval, &rd_size, qtyread, &con->dltime, flags, warc_tmp);

  tms = datetime_str (time (NULL));
  tmrate = retr_rate (rd_size, con->dltime);
  total_download_time += con->dltime;

#ifdef ENABLE_XATTR
  if (opt.enable_xattr)
    set_file_metadata (u, NULL, fp);
#endif

  fd_close (local_sock);
  /* Close the local file.  */
  if (!output_stream || con->cmd & DO_LIST)
    fclose (fp);

  /* If fd_read_body couldn't write to fp or warc_tmp, bail out.  */
  if (res == -2 || (warc_tmp != NULL && res == -3))
    {
      logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
                 con->target, strerror (errno));
      fd_close (csock);
      con->csock = -1;
      fd_close (dtsock);
      if (res == -2)
        return FWRITEERR;
      else if (res == -3)
        return WARC_TMP_FWRITEERR;
    }
  else if (res == -1)
    {
      logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
                 tms, tmrate, fd_errstr (dtsock));
      if (opt.server_response)
        logputs (LOG_ALWAYS, "\n");
    }
  fd_close (dtsock);

  /* Get the server to tell us if everything is retrieved.  */
  err = ftp_response (csock, &respline);
  if (err != FTPOK)
    {
      /* The control connection is decidedly closed.  Print the time
         only if it hasn't already been printed.  */
      if (res != -1)
        logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
      logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
      /* If there is an error on the control connection, close it, but
         return FTPRETRINT, since there is a possibility that the
         whole file was retrieved nevertheless (but that is for
         ftp_loop_internal to decide).  */
      fd_close (csock);
      con->csock = -1;
      return FTPRETRINT;
    } /* err != FTPOK */
  *last_expected_bytes = ftp_expected_bytes (respline);
  /* If retrieval failed for any reason, return FTPRETRINT, but do not
     close socket, since the control connection is still alive.  If
     there is something wrong with the control connection, it will
     become apparent later.  */
  if (*respline != '2')
    {
      if (res != -1)
        logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
      logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
#ifdef HAVE_SSL
      if (!c_strncasecmp (respline, "425", 3) && u->scheme == SCHEME_FTPS)
        {
          logputs (LOG_NOTQUIET, "FTPS server rejects new SSL sessions in the data connection.\n");
          xfree (respline);
          return FTPRESTFAIL;
        }
#endif
      xfree (respline);
      return FTPRETRINT;
    }
  xfree (respline);

  if (res == -1)
    {
      /* What now?  The data connection was erroneous, whereas the
         response says everything is OK.  We shall play it safe.  */
      return FTPRETRINT;
    }

  if (!(cmd & LEAVE_PENDING))
    {
      /* Closing the socket is faster than sending 'QUIT' and the
         effect is the same.  */
      fd_close (csock);
      con->csock = -1;
    }
  /* If it was a listing, and opt.server_response is true,
     print it out.  */
  if (con->cmd & DO_LIST)
    {
      if (opt.server_response)
        {
/* 2005-02-25 SMS.
   Much of this work may already have been done, but repeating it should
   do no damage beyond wasting time.
*/
/* On VMS, alter the name as required. */
#ifdef __VMS
      char *targ;

      targ = ods_conform (con->target);
      if (targ != con->target)
        {
          xfree (con->target);
          con->target = targ;
        }
#endif /* def __VMS */

      mkalldirs (con->target);
      fp = fopen (con->target, "r");
      if (!fp)
        logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
      else
        {
          char *line = NULL;
          size_t bufsize = 0;
          ssize_t len;

          /* The lines are being read with getline because of
             no-buffering on opt.lfile.  */
          while ((len = getline (&line, &bufsize, fp)) > 0)
            {
              while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r'))
                line[--len] = '\0';
              logprintf (LOG_ALWAYS, "%s\n",
                         quotearg_style (escape_quoting_style, line));
            }
          xfree (line);
          fclose (fp);
        }
        } /* server_response */

      /* 2013-10-17 Andrea Urbani (matfanjol)
         < __LIST_A_EXPLANATION__ >
          After the SYST command, looks if it knows that system.
          If yes, wget will force the use of "LIST" or "LIST -a".
          If no, wget will try, only the first time of each session, before the
          "LIST -a" command and after the "LIST".
          If "LIST -a" works and returns more or equal data of the "LIST",
          "LIST -a" will be the standard list command for all the session.
          If "LIST -a" fails or returns less data than "LIST" (think on the case
          of an existing file called "-a"), "LIST" will be the standard list
          command for all the session.
          ("LIST -a" is used to get also the hidden files)

          */
      if (!(con->st & LIST_AFTER_LIST_A_CHECK_DONE))
        {
          /* We still have to check "LIST" after the first "LIST -a" to see
             if with "LIST" we get more data than "LIST -a", that means
             "LIST -a" returned files/folders with "-a" name. */
          if (con->st & AVOID_LIST_A)
            {
              /* LIST was used in this cycle.
                 Let's see the result. */
              if (rd_size > previous_rd_size)
                {
                  /* LIST returns more data than "LIST -a".
                     "LIST" is the official command to use. */
                  con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
                  DEBUGP (("LIST returned more data than \"LIST -a\": "
                           "I will use \"LIST\" as standard list command\n"));
                }
              else if (previous_rd_size > rd_size)
                {
                  /* "LIST -a" returned more data then LIST.
                     "LIST -a" is the official command to use. */
                  con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
                  con->st |= AVOID_LIST;
                  con->st &= ~AVOID_LIST_A;
                  /* Sorry, please, download again the "LIST -a"... */
                  try_again = true;
                  DEBUGP (("LIST returned less data than \"LIST -a\": I will "
                           "use \"LIST -a\" as standard list command\n"));
                }
              else
                {
                  /* LIST and "LIST -a" return the same data. */
                  if (rd_size == 0)
                    {
                      /* Same empty data. We will check both again because
                         we cannot check if "LIST -a" has returned an empty
                         folder instead of a folder content. */
                      con->st &= ~AVOID_LIST_A;
                    }
                  else
                    {
                      /* Same data, so, better to take "LIST -a" that
                         shows also hidden files/folders (when present) */
                      con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
                      con->st |= AVOID_LIST;
                      con->st &= ~AVOID_LIST_A;
                      DEBUGP (("LIST returned the same amount of data of "
                               "\"LIST -a\": I will use \"LIST -a\" as standard "
                               "list command\n"));
                    }
                }
            }
          else
            {
              /* In this cycle "LIST -a" should being used. Is it true? */
              if (list_a_used)
                {
                  /* Yes, it is.
                     OK, let's save the amount of data and try again
                     with LIST */
                  previous_rd_size = rd_size;
                  try_again = true;
                  con->st |= AVOID_LIST_A;
                }
              else
                {
                  /* No: something happens and LIST was used.
                     This means "LIST -a" raises an error. */
                  con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
                  con->st |= AVOID_LIST_A;
                  DEBUGP (("\"LIST -a\" failed: I will use \"LIST\" "
                           "as standard list command\n"));
                }
            }
        }
    }
  } while (try_again);
  return RETRFINISHED;

exit_error:

  /* If fp is a regular file, close and try to remove it */
  if (fp && (!output_stream || con->cmd & DO_LIST))
    fclose (fp);
  return err;
}

Project: wget

check_auth (const struct url *u, char *user, char *passwd, struct response *resp,
            struct request *req, bool *ntlm_seen_ref, bool *retry,
            bool *basic_auth_finished_ref, bool *auth_finished_ref)
{
  uerr_t auth_err = RETROK;
  bool basic_auth_finished = *basic_auth_finished_ref;
  bool auth_finished = *auth_finished_ref;
  bool ntlm_seen = *ntlm_seen_ref;
  *retry = false;
  if (!auth_finished && (user && passwd))
    {
      /* IIS sends multiple copies of WWW-Authenticate, one with
         the value "negotiate", and other(s) with data.  Loop over
         all the occurrences and pick the one we recognize.  */
      int wapos;
      char *buf;
      const char *www_authenticate = NULL;
      const char *wabeg, *waend;
      const char *digest = NULL, *basic = NULL, *ntlm = NULL;
      for (wapos = 0; !ntlm
             && (wapos = resp_header_locate (resp, "WWW-Authenticate", wapos,
                                             &wabeg, &waend)) != -1;
           ++wapos)
        {
          param_token name, value;

          BOUNDED_TO_ALLOCA (wabeg, waend, buf);
          www_authenticate = buf;

          for (;!ntlm;)
            {
              /* extract the auth-scheme */
              while (c_isspace (*www_authenticate)) www_authenticate++;
              name.e = name.b = www_authenticate;
              while (*name.e && !c_isspace (*name.e)) name.e++;

              if (name.b == name.e)
                break;

              DEBUGP (("Auth scheme found '%.*s'\n", (int) (name.e - name.b), name.b));

              if (known_authentication_scheme_p (name.b, name.e))
                {
                  if (BEGINS_WITH (name.b, "NTLM"))
                    {
                      ntlm = name.b;
                      break; /* this is the most secure challenge, stop here */
                    }
                  else if (!digest && BEGINS_WITH (name.b, "Digest"))
                    digest = name.b;
                  else if (!basic && BEGINS_WITH (name.b, "Basic"))
                    basic = name.b;
                }

              /* now advance over the auth-params */
              www_authenticate = name.e;
              DEBUGP (("Auth param list '%s'\n", www_authenticate));
              while (extract_param (&www_authenticate, &name, &value, ',', NULL) && name.b && value.b)
                {
                  DEBUGP (("Auth param %.*s=%.*s\n",
                           (int) (name.e - name.b), name.b, (int) (value.e - value.b), value.b));
                }
            }
        }

      if (!basic && !digest && !ntlm)
        {
          /* If the authentication header is missing or
             unrecognized, there's no sense in retrying.  */
          logputs (LOG_NOTQUIET, _("Unknown authentication scheme.\n"));
        }
      else if (!basic_auth_finished
               || !basic)
        {
          char *pth = url_full_path (u);
          const char *value;
          uerr_t *auth_stat;
          auth_stat = xmalloc (sizeof (uerr_t));
          *auth_stat = RETROK;

          if (ntlm)
            www_authenticate = ntlm;
          else if (digest)
            www_authenticate = digest;
          else
            www_authenticate = basic;

          logprintf (LOG_NOTQUIET, _("Authentication selected: %s\n"), www_authenticate);

          value =  create_authorization_line (www_authenticate,
                                              user, passwd,
                                              request_method (req),
                                              pth,
                                              &auth_finished,
                                              auth_stat);

          auth_err = *auth_stat;
          xfree (auth_stat);
          xfree (pth);
          if (auth_err == RETROK)
            {
              request_set_header (req, "Authorization", value, rel_value);

              if (BEGINS_WITH (www_authenticate, "NTLM"))
                ntlm_seen = true;
              else if (!u->user && BEGINS_WITH (www_authenticate, "Basic"))
                {
                  /* Need to register this host as using basic auth,
                   * so we automatically send creds next time. */
                  register_basic_auth_host (u->host);
                }

              *retry = true;
              goto cleanup;
            }
          else
            {
              /* Creating the Authorization header went wrong */
            }
        }
      else
        {
          /* We already did Basic auth, and it failed. Gotta
           * give up. */
        }
    }

 cleanup:
  *ntlm_seen_ref = ntlm_seen;
  *basic_auth_finished_ref = basic_auth_finished;
  *auth_finished_ref = auth_finished;
  return auth_err;
}

Project: wget

ftp_retrieve_dirs (struct url *u, struct url *original_url,
                   struct fileinfo *f, ccon *con)
{
  char *container = NULL;
  int container_size = 0;

  for (; f; f = f->next)
    {
      int size;
      char *odir, *newdir;

      if (opt.quota && total_downloaded_bytes > opt.quota)
        break;
      if (f->type != FT_DIRECTORY)
        continue;

      /* Allocate u->dir off stack, but reallocate only if a larger
         string is needed.  It's a pity there's no "realloca" for an
         item on the bottom of the stack.  */
      size = strlen (u->dir) + 1 + strlen (f->name) + 1;
      if (size > container_size)
        container = (char *)alloca (size);
      newdir = container;

      odir = u->dir;
      if (*odir == '\0'
          || (*odir == '/' && *(odir + 1) == '\0'))
        /* If ODIR is empty or just "/", simply append f->name to
           ODIR.  (In the former case, to preserve u->dir being
           relative; in the latter case, to avoid double slash.)  */
        sprintf (newdir, "%s%s", odir, f->name);
      else
        /* Else, use a separator. */
        sprintf (newdir, "%s/%s", odir, f->name);

      DEBUGP (("Composing new CWD relative to the initial directory.\n"));
      DEBUGP (("  odir = '%s'\n  f->name = '%s'\n  newdir = '%s'\n\n",
               odir, f->name, newdir));
      if (!accdir (newdir))
        {
          logprintf (LOG_VERBOSE, _("\
Not descending to %s as it is excluded/not-included.\n"),
                     quote (newdir));
          continue;
        }

      con->st &= ~DONE_CWD;

      odir = xstrdup (u->dir);  /* because url_set_dir will free
                                   u->dir. */
      url_set_dir (u, newdir);
      ftp_retrieve_glob (u, original_url, con, GLOB_GETALL);
      url_set_dir (u, odir);
      xfree (odir);

      /* Set the time-stamp?  */
    }

  if (opt.quota && total_downloaded_bytes > opt.quota)
    return QUOTEXC;
  else
    return RETROK;
}

Project: wget

set_file_metadata (const struct url *origin_url, const struct url *referrer_url, FILE *fp)
{
  /* Save metadata about where the file came from (requested, final URLs) to
   * user POSIX Extended Attributes of retrieved file.
   *
   * For more details about the user namespace see
   * [http://freedesktop.org/wiki/CommonExtendedAttributes] and
   * [http://0pointer.de/lennart/projects/mod_mime_xattr/].
   */
  int retval = -1;
  char *value;

  if (!origin_url || !fp)
    return retval;

  value = url_string (origin_url, URL_AUTH_HIDE);
  retval = write_xattr_metadata ("user.xdg.origin.url", escnonprint_uri (value), fp);
  xfree (value);

  if (!retval && referrer_url)
    {
	  struct url u;

	  memset(&u, 0, sizeof(u));
      u.scheme = referrer_url->scheme;
      u.host = referrer_url->host;
      u.port = referrer_url->port;

      value = url_string (&u, 0);
      retval = write_xattr_metadata ("user.xdg.referrer.url", escnonprint_uri (value), fp);
      xfree (value);
    }

  return retval;
}

Project: wget

ftp_retrieve_list (struct url *u, struct url *original_url,
                   struct fileinfo *f, ccon *con)
{
  static int depth = 0;
  uerr_t err;
  struct fileinfo *orig;
  wgint local_size;
  time_t tml;
  bool dlthis; /* Download this (file). */
  const char *actual_target = NULL;
  bool force_full_retrieve = false;

  /* Increase the depth.  */
  ++depth;
  if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
    {
      DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
               depth, opt.reclevel));
      --depth;
      return RECLEVELEXC;
    }

  assert (f != NULL);
  orig = f;

  con->st &= ~ON_YOUR_OWN;
  if (!(con->st & DONE_CWD))
    con->cmd |= DO_CWD;
  else
    con->cmd &= ~DO_CWD;
  con->cmd |= (DO_RETR | LEAVE_PENDING);

  if (con->csock < 0)
    con->cmd |= DO_LOGIN;
  else
    con->cmd &= ~DO_LOGIN;

  err = RETROK;                 /* in case it's not used */

  while (f)
    {
      char *old_target, *ofile;

      if (opt.quota && total_downloaded_bytes > opt.quota)
        {
          --depth;
          return QUOTEXC;
        }
      old_target = con->target;

      ofile = xstrdup (u->file);
      url_set_file (u, f->name);

      con->target = url_file_name (u, NULL);
      err = RETROK;

      dlthis = true;
      if (opt.timestamping && f->type == FT_PLAINFILE)
        {
          struct stat st;
          /* If conversion of HTML files retrieved via FTP is ever implemented,
             we'll need to stat() <file>.orig here when -K has been specified.
             I'm not implementing it now since files on an FTP server are much
             more likely than files on an HTTP server to legitimately have a
             .orig suffix. */
          if (!stat (con->target, &st))
            {
              bool eq_size;
              bool cor_val;
              /* Else, get it from the file.  */
              local_size = st.st_size;
              tml = st.st_mtime;
#ifdef WINDOWS
              /* Modification time granularity is 2 seconds for Windows, so
                 increase local time by 1 second for later comparison. */
              tml++;
#endif
              /* Compare file sizes only for servers that tell us correct
                 values. Assume sizes being equal for servers that lie
                 about file size.  */
              cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
              eq_size = cor_val ? (local_size == f->size) : true;
              if (f->tstamp <= tml && eq_size)
                {
                  /* Remote file is older, file sizes can be compared and
                     are both equal. */
                  logprintf (LOG_VERBOSE, _("\
Remote file no newer than local file %s -- not retrieving.\n"), quote (con->target));
                  dlthis = false;
                }
              else if (f->tstamp > tml)
                {
                  /* Remote file is newer */
                  force_full_retrieve = true;
                  logprintf (LOG_VERBOSE, _("\
Remote file is newer than local file %s -- retrieving.\n\n"),
                             quote (con->target));
                }
              else
                {
                  /* Sizes do not match */
                  logprintf (LOG_VERBOSE, _("\
The sizes do not match (local %s) -- retrieving.\n\n"),
                             number_to_static_string (local_size));
                }
            }
        }       /* opt.timestamping && f->type == FT_PLAINFILE */
      switch (f->type)
        {
        case FT_SYMLINK:
          /* If opt.retr_symlinks is defined, we treat symlinks as
             if they were normal files.  There is currently no way
             to distinguish whether they might be directories, and
             follow them.  */
          if (!opt.retr_symlinks)
            {
#ifdef HAVE_SYMLINK
              if (!f->linkto)
                logputs (LOG_NOTQUIET,
                         _("Invalid name of the symlink, skipping.\n"));
              else
                {
                  struct stat st;
                  /* Check whether we already have the correct
                     symbolic link.  */
                  int rc = lstat (con->target, &st);
                  if (rc == 0)
                    {
                      size_t len = strlen (f->linkto) + 1;
                      if (S_ISLNK (st.st_mode))
                        {
                          char *link_target = (char *)alloca (len);
                          size_t n = readlink (con->target, link_target, len);
                          if ((n == len - 1)
                              && (memcmp (link_target, f->linkto, n) == 0))
                            {
                              logprintf (LOG_VERBOSE, _("\
Already have correct symlink %s -> %s\n\n"),
                                         quote (con->target),
                                         quote (f->linkto));
                              dlthis = false;
                              break;
                            }
                        }
                    }
                  logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
                             quote (con->target), quote (f->linkto));
                  /* Unlink before creating symlink!  */
                  unlink (con->target);
                  if (symlink (f->linkto, con->target) == -1)
                    logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
                  logputs (LOG_VERBOSE, "\n");
                } /* have f->linkto */
#else  /* not HAVE_SYMLINK */
              logprintf (LOG_NOTQUIET,
                         _("Symlinks not supported, skipping symlink %s.\n"),
                         quote (con->target));
#endif /* not HAVE_SYMLINK */
            }
          else                /* opt.retr_symlinks */
            {
              if (dlthis)
                {
                  err = ftp_loop_internal (u, original_url, f, con, NULL,
                                           force_full_retrieve);
                }
            } /* opt.retr_symlinks */
          break;
        case FT_DIRECTORY:
          if (!opt.recursive)
            logprintf (LOG_NOTQUIET, _("Skipping directory %s.\n"),
                       quote (f->name));
          break;
        case FT_PLAINFILE:
          /* Call the retrieve loop.  */
          if (dlthis)
            {
              err = ftp_loop_internal (u, original_url, f, con, NULL,
                                       force_full_retrieve);
            }
          break;
        case FT_UNKNOWN:
        default:
          logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
                     quote (f->name));
          break;
        }       /* switch */


      /* 2004-12-15 SMS.
       * Set permissions _before_ setting the times, as setting the
       * permissions changes the modified-time, at least on VMS.
       * Also, use the opt.output_document name here, too, as
       * appropriate.  (Do the test once, and save the result.)
       */

      set_local_file (&actual_target, con->target);

      /* If downloading a plain file, and the user requested it, then
         set valid (non-zero) permissions. */
      if (dlthis && (actual_target != NULL) &&
       (f->type == FT_PLAINFILE) && opt.preserve_perm)
        {
          if (f->perms)
            {
              if (chmod (actual_target, f->perms))
                logprintf (LOG_NOTQUIET,
                           _("Failed to set permissions for %s.\n"),
                           actual_target);
            }
          else
            DEBUGP (("Unrecognized permissions for %s.\n", actual_target));
        }

      /* Set the time-stamp information to the local file.  Symlinks
         are not to be stamped because it sets the stamp on the
         original.  :( */
      if (actual_target != NULL)
        {
          if (opt.useservertimestamps
              && !(f->type == FT_SYMLINK && !opt.retr_symlinks)
              && f->tstamp != -1
              && dlthis
              && file_exists_p (con->target, NULL))
            {
              touch (actual_target, f->tstamp);
            }
          else if (f->tstamp == -1)
            logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"),
                       actual_target);
        }

      xfree (con->target);
      con->target = old_target;

      url_set_file (u, ofile);
      xfree (ofile);

      /* Break on fatals.  */
      if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR
          || err == WARC_ERR || err == WARC_TMP_FOPENERR
          || err == WARC_TMP_FWRITEERR)
        break;
      con->cmd &= ~ (DO_CWD | DO_LOGIN);
      f = f->next;
    }

  /* We do not want to call ftp_retrieve_dirs here */
  if (opt.recursive &&
      !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
    err = ftp_retrieve_dirs (u, original_url, orig, con);
  else if (opt.recursive)
    DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
             depth, opt.reclevel));
  --depth;
  return err;
}

Project: wget

gethttp (const struct url *u, struct url *original_url, struct http_stat *hs,
         int *dt, struct url *proxy, struct iri *iri, int count)
{
  struct request *req = NULL;

  char *type = NULL;
  char *user, *passwd;
  char *proxyauth;
  int statcode;
  int write_error;
  wgint contlen, contrange;
  const struct url *conn;
  FILE *fp;
  int err;
  uerr_t retval;
#ifdef HAVE_HSTS
#ifdef TESTING
  /* we don't link against main.o when we're testing */
  hsts_store_t hsts_store = NULL;
#else
  extern hsts_store_t hsts_store;
#endif
  const char *hsts_params;
  time_t max_age;
  bool include_subdomains;
#endif

  int sock = -1;

  /* Set to 1 when the authorization has already been sent and should
     not be tried again. */
  bool auth_finished = false;

  /* Set to 1 when just globally-set Basic authorization has been sent;
   * should prevent further Basic negotiations, but not other
   * mechanisms. */
  bool basic_auth_finished = false;

  /* Whether NTLM authentication is used for this request. */
  bool ntlm_seen = false;

  /* Whether our connection to the remote host is through SSL.  */
  bool using_ssl = false;

  /* Whether a HEAD request will be issued (as opposed to GET or
     POST). */
  bool head_only = !!(*dt & HEAD_ONLY);

  /* Whether conditional get request will be issued.  */
  bool cond_get = !!(*dt & IF_MODIFIED_SINCE);

#ifdef HAVE_METALINK
  /* Are we looking for metalink info in HTTP headers?  */
  bool metalink = !!(*dt & METALINK_METADATA);
#endif

  char *head = NULL;
  struct response *resp = NULL;
  char hdrval[512];
  char *message = NULL;

  /* Declare WARC variables. */
  bool warc_enabled = (opt.warc_filename != NULL);
  FILE *warc_tmp = NULL;
  char warc_timestamp_str [21];
  char warc_request_uuid [48];
  ip_address *warc_ip = NULL;
  off_t warc_payload_offset = -1;

  /* Whether this connection will be kept alive after the HTTP request
     is done. */
  bool keep_alive;

  /* Is the server using the chunked transfer encoding?  */
  bool chunked_transfer_encoding = false;

  /* Whether keep-alive should be inhibited.  */
  bool inhibit_keep_alive =
    !opt.http_keep_alive || opt.ignore_length;

  /* Headers sent when using POST. */
  wgint body_data_size = 0;

#ifdef HAVE_SSL
  if (u->scheme == SCHEME_HTTPS)
    {
      /* Initialize the SSL context.  After this has once been done,
         it becomes a no-op.  */
      if (!ssl_init ())
        {
          scheme_disable (SCHEME_HTTPS);
          logprintf (LOG_NOTQUIET,
                     _("Disabling SSL due to encountered errors.\n"));
          retval = SSLINITFAILED;
          goto cleanup;
        }
    }
#endif /* HAVE_SSL */

  /* Initialize certain elements of struct http_stat.  */
  hs->len = 0;
  hs->contlen = -1;
  hs->res = -1;
  hs->rderrmsg = NULL;
  hs->newloc = NULL;
  xfree (hs->remote_time);
  hs->error = NULL;
  hs->message = NULL;
  hs->local_encoding = ENC_NONE;
  hs->remote_encoding = ENC_NONE;

  conn = u;

  {
    uerr_t ret;
    req = initialize_request (u, hs, dt, proxy, inhibit_keep_alive,
                              &basic_auth_finished, &body_data_size,
                              &user, &passwd, &ret);
    if (req == NULL)
      {
        retval = ret;
        goto cleanup;
      }
  }
 retry_with_auth:
  /* We need to come back here when the initial attempt to retrieve
     without authorization header fails.  (Expected to happen at least
     for the Digest authorization scheme.)  */

  if (opt.cookies)
    request_set_header (req, "Cookie",
                        cookie_header (wget_cookie_jar,
                                       u->host, u->port, u->path,
#ifdef HAVE_SSL
                                       u->scheme == SCHEME_HTTPS
#else
                                       0
#endif
                                       ),
                        rel_value);

  /* Add the user headers. */
  if (opt.user_headers)
    {
      int i;
      for (i = 0; opt.user_headers[i]; i++)
        request_set_user_header (req, opt.user_headers[i]);
    }

  proxyauth = NULL;
  if (proxy)
    {
      conn = proxy;
      initialize_proxy_configuration (u, req, proxy, &proxyauth);
    }
  keep_alive = true;

  /* Establish the connection.  */
  if (inhibit_keep_alive)
    keep_alive = false;

  {
    uerr_t conn_err = establish_connection (u, &conn, hs, proxy, &proxyauth, &req,
                                            &using_ssl, inhibit_keep_alive, &sock);
    if (conn_err != RETROK)
      {
        retval = conn_err;
        goto cleanup;
      }
  }

  /* Open the temporary file where we will write the request. */
  if (warc_enabled)
    {
      warc_tmp = warc_tempfile ();
      if (warc_tmp == NULL)
        {
          CLOSE_INVALIDATE (sock);
          retval = WARC_TMP_FOPENERR;
          goto cleanup;
        }

      if (! proxy)
        {
          warc_ip = (ip_address *) alloca (sizeof (ip_address));
          socket_ip_address (sock, warc_ip, ENDPOINT_PEER);
        }
    }

  /* Send the request to server.  */
  write_error = request_send (req, sock, warc_tmp);

  if (write_error >= 0)
    {
      if (opt.body_data)
        {
          DEBUGP (("[BODY data: %s]\n", opt.body_data));
          write_error = fd_write (sock, opt.body_data, body_data_size, -1);
          if (write_error >= 0 && warc_tmp != NULL)
            {
              int warc_tmp_written;

              /* Remember end of headers / start of payload. */
              warc_payload_offset = ftello (warc_tmp);

              /* Write a copy of the data to the WARC record. */
              warc_tmp_written = fwrite (opt.body_data, 1, body_data_size, warc_tmp);
              if (warc_tmp_written != body_data_size)
                write_error = -2;
            }
         }
      else if (opt.body_file && body_data_size != 0)
        {
          if (warc_tmp != NULL)
            /* Remember end of headers / start of payload */
            warc_payload_offset = ftello (warc_tmp);

          write_error = body_file_send (sock, opt.body_file, body_data_size, warc_tmp);
        }
    }

  if (write_error < 0)
    {
      CLOSE_INVALIDATE (sock);

      if (warc_tmp != NULL)
        fclose (warc_tmp);

      if (write_error == -2)
        retval = WARC_TMP_FWRITEERR;
      else
        retval = WRITEFAILED;
      goto cleanup;
    }
  logprintf (LOG_VERBOSE, _("%s request sent, awaiting response... "),
             proxy ? "Proxy" : "HTTP");
  contlen = -1;
  contrange = 0;
  *dt &= ~RETROKF;


  if (warc_enabled)
    {
      bool warc_result;

      /* Generate a timestamp and uuid for this request. */
      warc_timestamp (warc_timestamp_str, sizeof (warc_timestamp_str));
      warc_uuid_str (warc_request_uuid);

      /* Create a request record and store it in the WARC file. */
      warc_result = warc_write_request_record (u->url, warc_timestamp_str,
                                               warc_request_uuid, warc_ip,
                                               warc_tmp, warc_payload_offset);
      if (! warc_result)
        {
          CLOSE_INVALIDATE (sock);
          retval = WARC_ERR;
          goto cleanup;
        }

      /* warc_write_request_record has also closed warc_tmp. */
    }

  /* Repeat while we receive a 10x response code.  */
  {
    bool _repeat;

    do
      {
        head = read_http_response_head (sock);
        if (!head)
          {
            if (errno == 0)
              {
                logputs (LOG_NOTQUIET, _("No data received.\n"));
                CLOSE_INVALIDATE (sock);
                retval = HEOF;
              }
            else
              {
                logprintf (LOG_NOTQUIET, _("Read error (%s) in headers.\n"),
                           fd_errstr (sock));
                CLOSE_INVALIDATE (sock);
                retval = HERR;
              }
            goto cleanup;
          }
        DEBUGP (("\n---response begin---\n%s---response end---\n", head));

        resp = resp_new (head);

        /* Check for status line.  */
        xfree (message);
        statcode = resp_status (resp, &message);
        if (statcode < 0)
          {
            char *tms = datetime_str (time (NULL));
            logprintf (LOG_VERBOSE, "%d\n", statcode);
            logprintf (LOG_NOTQUIET, _("%s ERROR %d: %s.\n"), tms, statcode,
                       quotearg_style (escape_quoting_style,
                                       _("Malformed status line")));
            CLOSE_INVALIDATE (sock);
            retval = HERR;
            goto cleanup;
          }

        if (H_10X (statcode))
          {
            xfree (head);
            resp_free (&resp);
            _repeat = true;
            DEBUGP (("Ignoring response\n"));
          }
        else
          {
            _repeat = false;
          }
      }
    while (_repeat);
  }

  xfree (hs->message);
  hs->message = xstrdup (message);
  if (!opt.server_response)
    logprintf (LOG_VERBOSE, "%2d %s\n", statcode,
               message ? quotearg_style (escape_quoting_style, message) : "");
  else
    {
      logprintf (LOG_VERBOSE, "\n");
      print_server_response (resp, "  ");
    }

  if (!opt.ignore_length
      && resp_header_copy (resp, "Content-Length", hdrval, sizeof (hdrval)))
    {
      wgint parsed;
      errno = 0;
      parsed = str_to_wgint (hdrval, NULL, 10);
      if (parsed == WGINT_MAX && errno == ERANGE)
        {
          /* Out of range.
             #### If Content-Length is out of range, it most likely
             means that the file is larger than 2G and that we're
             compiled without LFS.  In that case we should probably
             refuse to even attempt to download the file.  */
          contlen = -1;
        }
      else if (parsed < 0)
        {
          /* Negative Content-Length; nonsensical, so we can't
             assume any information about the content to receive. */
          contlen = -1;
        }
      else
        contlen = parsed;
    }

  /* Check for keep-alive related responses. */
  if (!inhibit_keep_alive)
    {
      if (resp_header_copy (resp, "Connection", hdrval, sizeof (hdrval)))
        {
          if (0 == c_strcasecmp (hdrval, "Close"))
            keep_alive = false;
        }
    }

  chunked_transfer_encoding = false;
  if (resp_header_copy (resp, "Transfer-Encoding", hdrval, sizeof (hdrval))
      && 0 == c_strcasecmp (hdrval, "chunked"))
    chunked_transfer_encoding = true;

  /* Handle (possibly multiple instances of) the Set-Cookie header. */
  if (opt.cookies)
    {
      int scpos;
      const char *scbeg, *scend;
      /* The jar should have been created by now. */
      assert (wget_cookie_jar != NULL);
      for (scpos = 0;
           (scpos = resp_header_locate (resp, "Set-Cookie", scpos,
                                        &scbeg, &scend)) != -1;
           ++scpos)
        {
          char *set_cookie; BOUNDED_TO_ALLOCA (scbeg, scend, set_cookie);
          cookie_handle_set_cookie (wget_cookie_jar, u->host, u->port,
                                    u->path, set_cookie);
        }
    }

  if (keep_alive)
    /* The server has promised that it will not close the connection
       when we're done.  This means that we can register it.  */
    register_persistent (conn->host, conn->port, sock, using_ssl);

#ifdef HAVE_METALINK
  /* We need to check for the Metalink data in the very first response
     we get from the server (before redirections, authorization, etc.).  */
  if (metalink)
    {
      hs->metalink = metalink_from_http (resp, hs, u);
      /* Bugfix: hs->local_file is NULL (opt.content_disposition).  */
      if (!hs->local_file && hs->metalink && hs->metalink->origin)
        hs->local_file = xstrdup (hs->metalink->origin);
      xfree (hs->message);
      retval = RETR_WITH_METALINK;
      CLOSE_FINISH (sock);
      goto cleanup;
    }
#endif

  if (statcode == HTTP_STATUS_UNAUTHORIZED)
    {
      /* Authorization is required.  */
      uerr_t auth_err = RETROK;
      bool retry;
      /* Normally we are not interested in the response body.
         But if we are writing a WARC file we are: we like to keep everything.  */
      if (warc_enabled)
        {
          int _err;
          type = resp_header_strdup (resp, "Content-Type");
          _err = read_response_body (hs, sock, NULL, contlen, 0,
                                    chunked_transfer_encoding,
                                    u->url, warc_timestamp_str,
                                    warc_request_uuid, warc_ip, type,
                                    statcode, head);
          xfree (type);

          if (_err != RETRFINISHED || hs->res < 0)
            {
              CLOSE_INVALIDATE (sock);
              retval = _err;
              goto cleanup;
            }
          else
            CLOSE_FINISH (sock);
        }
      else
        {
          /* Since WARC is disabled, we are not interested in the response body.  */
          if (keep_alive && !head_only
              && skip_short_body (sock, contlen, chunked_transfer_encoding))
            CLOSE_FINISH (sock);
          else
            CLOSE_INVALIDATE (sock);
        }

      pconn.authorized = false;

      {
        auth_err = check_auth (u, user, passwd, resp, req,
                               &ntlm_seen, &retry,
                               &basic_auth_finished,
                               &auth_finished);
        if (auth_err == RETROK && retry)
          {
            xfree (hs->message);
            resp_free (&resp);
            xfree (message);
            xfree (head);
            goto retry_with_auth;
          }
      }
      if (auth_err == RETROK)
        retval = AUTHFAILED;
      else
        retval = auth_err;
      goto cleanup;
    }
  else /* statcode != HTTP_STATUS_UNAUTHORIZED */
    {
      /* Kludge: if NTLM is used, mark the TCP connection as authorized. */
      if (ntlm_seen)
        pconn.authorized = true;
    }

  {
    uerr_t ret = check_file_output (u, hs, resp, hdrval, sizeof hdrval);
    if (ret != RETROK)
      {
        retval = ret;
        goto cleanup;
      }
  }

  hs->statcode = statcode;
  if (statcode == -1)
    hs->error = xstrdup (_("Malformed status line"));
  else if (!*message)
    hs->error = xstrdup (_("(no description)"));
  else
    hs->error = xstrdup (message);

#ifdef HAVE_HSTS
  if (opt.hsts && hsts_store)
    {
      hsts_params = resp_header_strdup (resp, "Strict-Transport-Security");
      if (parse_strict_transport_security (hsts_params, &max_age, &include_subdomains))
        {
          /* process strict transport security */
          if (hsts_store_entry (hsts_store, u->scheme, u->host, u->port, max_age, include_subdomains))
            DEBUGP(("Added new HSTS host: %s:%u (max-age: %lu, includeSubdomains: %s)\n",
                   u->host,
                   (unsigned) u->port,
                   (unsigned long) max_age,
                   (include_subdomains ? "true" : "false")));
          else
            DEBUGP(("Updated HSTS host: %s:%u (max-age: %lu, includeSubdomains: %s)\n",
                   u->host,
                   (unsigned) u->port,
                   (unsigned long) max_age,
                   (include_subdomains ? "true" : "false")));
        }
    }
#endif

  type = resp_header_strdup (resp, "Content-Type");
  if (type)
    {
      char *tmp = strchr (type, ';');
      if (tmp)
        {
#ifdef ENABLE_IRI
          /* sXXXav: only needed if IRI support is enabled */
          char *tmp2 = tmp + 1;
#endif

          while (tmp > type && c_isspace (tmp[-1]))
            --tmp;
          *tmp = '\0';

#ifdef ENABLE_IRI
          /* Try to get remote encoding if needed */
          if (opt.enable_iri && !opt.encoding_remote)
            {
              tmp = parse_charset (tmp2);
              if (tmp)
                set_content_encoding (iri, tmp);
              xfree (tmp);
            }
#endif
        }
    }
  hs->newloc = resp_header_strdup (resp, "Location");
  hs->remote_time = resp_header_strdup (resp, "Last-Modified");
  if (!hs->remote_time) // now look for the Wayback Machine's timestamp
    hs->remote_time = resp_header_strdup (resp, "X-Archive-Orig-last-modified");

  if (resp_header_copy (resp, "Content-Range", hdrval, sizeof (hdrval)))
    {
      wgint first_byte_pos, last_byte_pos, entity_length;
      if (parse_content_range (hdrval, &first_byte_pos, &last_byte_pos,
                               &entity_length))
        {
          contrange = first_byte_pos;
          contlen = last_byte_pos - first_byte_pos + 1;
        }
    }

  if (resp_header_copy (resp, "Content-Encoding", hdrval, sizeof (hdrval)))
    {
      hs->local_encoding = ENC_INVALID;

      switch (hdrval[0])
        {
        case 'b': case 'B':
          if (0 == c_strcasecmp(hdrval, "br"))
            hs->local_encoding = ENC_BROTLI;
          break;
        case 'c': case 'C':
          if (0 == c_strcasecmp(hdrval, "compress"))
            hs->local_encoding = ENC_COMPRESS;
          break;
        case 'd': case 'D':
          if (0 == c_strcasecmp(hdrval, "deflate"))
            hs->local_encoding = ENC_DEFLATE;
          break;
        case 'g': case 'G':
          if (0 == c_strcasecmp(hdrval, "gzip"))
            hs->local_encoding = ENC_GZIP;
          break;
        case 'i': case 'I':
          if (0 == c_strcasecmp(hdrval, "identity"))
            hs->local_encoding = ENC_NONE;
          break;
        case 'x': case 'X':
          if (0 == c_strcasecmp(hdrval, "x-compress"))
            hs->local_encoding = ENC_COMPRESS;
          else if (0 == c_strcasecmp(hdrval, "x-gzip"))
            hs->local_encoding = ENC_GZIP;
          break;
        case '\0':
          hs->local_encoding = ENC_NONE;
        }

      if (hs->local_encoding == ENC_INVALID)
        {
          DEBUGP (("Unrecognized Content-Encoding: %s\n", hdrval));
          hs->local_encoding = ENC_NONE;
        }
#ifdef HAVE_LIBZ
      else if (hs->local_encoding == ENC_GZIP
               && opt.compression != compression_none)
        {
          const char *p;

          /* Make sure the Content-Type is not gzip before decompressing */
          if (type)
            {
              p = strchr (type, '/');
              if (p == NULL)
                {
                  hs->remote_encoding = ENC_GZIP;
                  hs->local_encoding = ENC_NONE;
                }
              else
                {
                  p++;
                  if (c_tolower(p[0]) == 'x' && p[1] == '-')
                    p += 2;
                  if (0 != c_strcasecmp (p, "gzip"))
                    {
                      hs->remote_encoding = ENC_GZIP;
                      hs->local_encoding = ENC_NONE;
                    }
                }
            }
          else
            {
               hs->remote_encoding = ENC_GZIP;
               hs->local_encoding = ENC_NONE;
            }

          /* don't uncompress if a file ends with '.gz' or '.tgz' */
          if (hs->remote_encoding == ENC_GZIP
              && (p = strrchr(u->file, '.'))
              && (c_strcasecmp(p, ".gz") == 0 || c_strcasecmp(p, ".tgz") == 0))
            {
               DEBUGP (("Enabling broken server workaround. Will not decompress this GZip file.\n"));
               hs->remote_encoding = ENC_NONE;
            }
        }
#endif
    }

  /* 20x responses are counted among successful by default.  */
  if (H_20X (statcode))
    *dt |= RETROKF;

  if (statcode == HTTP_STATUS_NO_CONTENT)
    {
      /* 204 response has no body (RFC 2616, 4.3) */

      /* In case the caller cares to look...  */
      hs->len = 0;
      hs->res = 0;
      hs->restval = 0;

      CLOSE_FINISH (sock);

      retval = RETRFINISHED;
      goto cleanup;
    }

  /* Return if redirected.  */
  if (H_REDIRECTED (statcode) || statcode == HTTP_STATUS_MULTIPLE_CHOICES)
    {
      /* RFC2068 says that in case of the 300 (multiple choices)
         response, the server can output a preferred URL through
         `Location' header; otherwise, the request should be treated
         like GET.  So, if the location is set, it will be a
         redirection; otherwise, just proceed normally.  */
      if (statcode == HTTP_STATUS_MULTIPLE_CHOICES && !hs->newloc)
        *dt |= RETROKF;
      else
        {
          logprintf (LOG_VERBOSE,
                     _("Location: %s%s\n"),
                     hs->newloc ? escnonprint_uri (hs->newloc) : _("unspecified"),
                     hs->newloc ? _(" [following]") : "");

          /* In case the caller cares to look...  */
          hs->len = 0;
          hs->res = 0;
          hs->restval = 0;

          /* Normally we are not interested in the response body of a redirect.
             But if we are writing a WARC file we are: we like to keep everything.  */
          if (warc_enabled)
            {
              int _err = read_response_body (hs, sock, NULL, contlen, 0,
                                            chunked_transfer_encoding,
                                            u->url, warc_timestamp_str,
                                            warc_request_uuid, warc_ip, type,
                                            statcode, head);

              if (_err != RETRFINISHED || hs->res < 0)
                {
                  CLOSE_INVALIDATE (sock);
                  retval = _err;
                  goto cleanup;
                }
              else
                CLOSE_FINISH (sock);
            }
          else
            {
              /* Since WARC is disabled, we are not interested in the response body.  */
              if (keep_alive && !head_only
                  && skip_short_body (sock, contlen, chunked_transfer_encoding))
                CLOSE_FINISH (sock);
              else
                CLOSE_INVALIDATE (sock);
            }

          /* From RFC2616: The status codes 303 and 307 have
             been added for servers that wish to make unambiguously
             clear which kind of reaction is expected of the client.

             A 307 should be redirected using the same method,
             in other words, a POST should be preserved and not
             converted to a GET in that case.

             With strict adherence to RFC2616, POST requests are not
             converted to a GET request on 301 Permanent Redirect
             or 302 Temporary Redirect.

             A switch may be provided later based on the HTTPbis draft
             that allows clients to convert POST requests to GET
             requests on 301 and 302 response codes. */
          switch (statcode)
            {
            case HTTP_STATUS_TEMPORARY_REDIRECT:
            case HTTP_STATUS_PERMANENT_REDIRECT:
              retval = NEWLOCATION_KEEP_POST;
              goto cleanup;
            case HTTP_STATUS_MOVED_PERMANENTLY:
              if (opt.method && c_strcasecmp (opt.method, "post") != 0)
                {
                  retval = NEWLOCATION_KEEP_POST;
                  goto cleanup;
                }
              break;
            case HTTP_STATUS_MOVED_TEMPORARILY:
              if (opt.method && c_strcasecmp (opt.method, "post") != 0)
                {
                  retval = NEWLOCATION_KEEP_POST;
                  goto cleanup;
                }
              break;
            }
          retval = NEWLOCATION;
          goto cleanup;
        }
    }

  if (cond_get)
    {
      if (statcode == HTTP_STATUS_NOT_MODIFIED)
        {
          logprintf (LOG_VERBOSE,
                     _ ("File %s not modified on server. Omitting download.\n\n"),
                     quote (hs->local_file));
          *dt |= RETROKF;
          CLOSE_FINISH (sock);
          retval = RETRUNNEEDED;
          goto cleanup;
        }
    }

  set_content_type (dt, type);

  if (opt.adjust_extension)
    {
      const char *encoding_ext = NULL;
      switch (hs->local_encoding)
        {
        case ENC_INVALID:
        case ENC_NONE:
          break;
        case ENC_BROTLI:
          encoding_ext = ".br";
          break;
        case ENC_COMPRESS:
          encoding_ext = ".Z";
          break;
        case ENC_DEFLATE:
          encoding_ext = ".zlib";
          break;
        case ENC_GZIP:
          encoding_ext = ".gz";
          break;
        default:
          DEBUGP (("No extension found for encoding %d\n",
                   hs->local_encoding));
      }
      if (encoding_ext != NULL)
        {
          char *file_ext = strrchr (hs->local_file, '.');
          /* strip Content-Encoding extension (it will be re-added later) */
          if (file_ext != NULL && 0 == strcasecmp (file_ext, encoding_ext))
            *file_ext = '\0';
        }
      if (*dt & TEXTHTML)
        /* -E / --adjust-extension / adjust_extension = on was specified,
           and this is a text/html file.  If some case-insensitive
           variation on ".htm[l]" isn't already the file's suffix,
           tack on ".html". */
        {
          ensure_extension (hs, ".html", dt);
        }
      else if (*dt & TEXTCSS)
        {
          ensure_extension (hs, ".css", dt);
        }
      if (encoding_ext != NULL)
        {
          ensure_extension (hs, encoding_ext, dt);
        }
    }

  if (cond_get)
    {
      /* Handle the case when server ignores If-Modified-Since header.  */
      if (statcode == HTTP_STATUS_OK && hs->remote_time)
        {
          time_t tmr = http_atotm (hs->remote_time);

          /* Check if the local file is up-to-date based on Last-Modified header
             and content length.  */
          if (tmr != (time_t) - 1 && tmr <= hs->orig_file_tstamp
              && (contlen == -1 || contlen == hs->orig_file_size))
            {
              logprintf (LOG_VERBOSE,
                         _("Server ignored If-Modified-Since header for file %s.\n"
                           "You might want to add --no-if-modified-since option."
                           "\n\n"),
                         quote (hs->local_file));
              *dt |= RETROKF;
              CLOSE_INVALIDATE (sock);
              retval = RETRUNNEEDED;
              goto cleanup;
            }
        }
    }

  if (statcode == HTTP_STATUS_RANGE_NOT_SATISFIABLE
      || (!opt.timestamping && hs->restval > 0 && statcode == HTTP_STATUS_OK
          && contrange == 0 && contlen >= 0 && hs->restval >= contlen))
    {
      /* If `-c' is in use and the file has been fully downloaded (or
         the remote file has shrunk), Wget effectively requests bytes
         after the end of file and the server response with 416
         (or 200 with a <= Content-Length.  */
      logputs (LOG_VERBOSE, _("\
\n    The file is already fully retrieved; nothing to do.\n\n"));
      /* In case the caller inspects. */
      hs->len = contlen;
      hs->res = 0;
      /* Mark as successfully retrieved. */
      *dt |= RETROKF;

      /* Try to maintain the keep-alive connection. It is often cheaper to
       * consume some bytes which have already been sent than to negotiate
       * a new connection. However, if the body is too large, or we don't
       * care about keep-alive, then simply terminate the connection */
      if (keep_alive &&
          skip_short_body (sock, contlen, chunked_transfer_encoding))
        CLOSE_FINISH (sock);
      else
        CLOSE_INVALIDATE (sock);
      retval = RETRUNNEEDED;
      goto cleanup;
    }
  if ((contrange != 0 && contrange != hs->restval)
      || (H_PARTIAL (statcode) && !contrange && hs->restval))
    {
      /* The Range request was somehow misunderstood by the server.
         Bail out.  */
      CLOSE_INVALIDATE (sock);
      retval = RANGEERR;
      goto cleanup;
    }
  if (contlen == -1)
    hs->contlen = -1;
  /* If the response is gzipped, the uncompressed size is unknown. */
  else if (hs->remote_encoding == ENC_GZIP)
    hs->contlen = -1;
  else
    hs->contlen = contlen + contrange;

  if (opt.verbose)
    {
      if (*dt & RETROKF)
        {
          /* No need to print this output if the body won't be
             downloaded at all, or if the original server response is
             printed.  */
          logputs (LOG_VERBOSE, _("Length: "));
          if (contlen != -1)
            {
              logputs (LOG_VERBOSE, number_to_static_string (contlen + contrange));
              if (contlen + contrange >= 1024)
                logprintf (LOG_VERBOSE, " (%s)",
                           human_readable (contlen + contrange, 10, 1));
              if (contrange)
                {
                  if (contlen >= 1024)
                    logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
                               number_to_static_string (contlen),
                               human_readable (contlen, 10, 1));
                  else
                    logprintf (LOG_VERBOSE, _(", %s remaining"),
                               number_to_static_string (contlen));
                }
            }
          else
            logputs (LOG_VERBOSE,
                     opt.ignore_length ? _("ignored") : _("unspecified"));
          if (type)
            logprintf (LOG_VERBOSE, " [%s]\n", quotearg_style (escape_quoting_style, type));
          else
            logputs (LOG_VERBOSE, "\n");
        }
    }

  /* Return if we have no intention of further downloading.  */
  if ((!(*dt & RETROKF) && !opt.content_on_error) || head_only || (opt.spider && !opt.recursive))
    {
      /* In case the caller cares to look...  */
      hs->len = 0;
      hs->res = 0;
      hs->restval = 0;

      /* Normally we are not interested in the response body of a error responses.
         But if we are writing a WARC file we are: we like to keep everything.  */
      if (warc_enabled)
        {
          int _err = read_response_body (hs, sock, NULL, contlen, 0,
                                        chunked_transfer_encoding,
                                        u->url, warc_timestamp_str,
                                        warc_request_uuid, warc_ip, type,
                                        statcode, head);

          if (_err != RETRFINISHED || hs->res < 0)
            {
              CLOSE_INVALIDATE (sock);
              retval = _err;
              goto cleanup;
            }

          CLOSE_FINISH (sock);
        }
      else
        {
          /* Since WARC is disabled, we are not interested in the response body.  */
          if (head_only)
            /* Pre-1.10 Wget used CLOSE_INVALIDATE here.  Now we trust the
               servers not to send body in response to a HEAD request, and
               those that do will likely be caught by test_socket_open.
               If not, they can be worked around using
               `--no-http-keep-alive'.  */
            CLOSE_FINISH (sock);
          else if (opt.spider && !opt.recursive)
            /* we just want to see if the page exists - no downloading required */
            CLOSE_INVALIDATE (sock);
          else if (keep_alive
                   && skip_short_body (sock, contlen, chunked_transfer_encoding))
            /* Successfully skipped the body; also keep using the socket. */
            CLOSE_FINISH (sock);
          else
            CLOSE_INVALIDATE (sock);
        }

      if (statcode == HTTP_STATUS_GATEWAY_TIMEOUT)
        retval = GATEWAYTIMEOUT;
      else
        retval = RETRFINISHED;

      goto cleanup;
    }

  err = open_output_stream (hs, count, &fp);
  if (err != RETROK)
    {
      CLOSE_INVALIDATE (sock);
      retval = err;
      goto cleanup;
    }

#ifdef ENABLE_XATTR
  if (opt.enable_xattr)
    {
      if (original_url != u)
        set_file_metadata (u, original_url, fp);
      else
        set_file_metadata (u, NULL, fp);
    }
#endif

  err = read_response_body (hs, sock, fp, contlen, contrange,
                            chunked_transfer_encoding,
                            u->url, warc_timestamp_str,
                            warc_request_uuid, warc_ip, type,
                            statcode, head);

  if (hs->res >= 0)
    CLOSE_FINISH (sock);
  else
    CLOSE_INVALIDATE (sock);

  if (!output_stream)
    fclose (fp);

  retval = err;

  cleanup:
  xfree (head);
  xfree (type);
  xfree (message);
  resp_free (&resp);
  request_free (&req);

  return retval;
}