rails/rails — Issue #3307: Exception Encoding::UndefinedConversionError -> "\xC5" from ASCII-8BIT to UTF-8
Author: Fallen-Zen
Created: 2011-10-12 08:28:46+00:00
State: open

I'm getting annoying bug when handling e-mails with Polish characters in them. I'll start with the code.

Here's the script that fetches emails from mailboxes in an endless loop (this script is being handled by bluepill):

```
#!/usr/bin/env ruby

require 'net/imap'
require 'yaml'
require 'stalker'
require 'logger'

# Useful functions
def clean_message_id(message_id)
message_id[/<([^<>]+)>/, 1]
end

# Configuration
config_file = File.expand_path("../../config/mailboxes.yml", __FILE__)
begin
config = YAML.load_file(config_file)
rescue
puts "Error while loading configuration file. Please ensure #{config_file} exists."
end

# Additional configuration
config[:source_mailbox] = 'INBOX'
config[:target_mailbox] = "ARCHIVE.#{Time.now.strftime('%Y-%m')}"

beginning_time = last_action_time = Time.now

loop do

# Loop on the mailboxes
config[:mailboxes].each do |mailbox|

puts "Child process for mailbox #{mailbox[:name]} started."

log = Logger.new(File.expand_path("../../log/emailfetch/#{mailbox[:username]}.log", __FILE__), 10, 1024000)

begin

log.debug "Connecting to IMAP server #{config[:mail_server]}..."
imap = Net::IMAP.new(config[:mail_server])
log.debug ' done.'

log.debug "Authenticating to mailbox #{mailbox[:name]}..."
imap.authenticate('PLAIN', mailbox[:username], mailbox[:password])
log.debug ' done.'

log.debug "Selecting mailbox #{config[:source_mailbox]}..."
imap.select(config[:source_mailbox])
log.debug ' done.'

# Create the target mailbox if it does not exist
if imap.list('', config[:target_mailbox])
log.debug "#{config[:target_mailbox]} mailbox exists."
else
log.debug "#{config[:target_mailbox]} mailbox does not exist, let's create it..."
imap.create(config[:target_mailbox])
log.debug ' done.'
end

count = 0
email = nil

# Let's fetch the e-mails
imap.search(["SINCE", "1-Jan-2011"]).each do |uid|

count = count + 1

log.debug "Fetching e-mail ##{uid}..."

email = imap.fetch(uid, ['RFC822', 'ENVELOPE']).first
message_id = clean_message_id(email.attr['ENVELOPE'].message_id)
body = email.attr['RFC822']
log.debug " fetched: #{message_id}"

# Store the email in a temporary file
filename = File.expand_path("../../tmp/mail_queue/#{message_id}.eml", __FILE__)
file = File.open( filename, 'w+')
file.write(body)
file.close

raise 'Error while saving the message' if !File.exists?(filename) or File.zero?(filename)

# Put the email in the queue for later processing
Stalker.enqueue('email.process', :file => filename)

log.debug ' queued.'

# Move the email to the target mailbox
imap.copy(uid, config[:target_mailbox])
imap.store(uid, "+FLAGS", [:Deleted])
end
# Remove e-mails flagged as Deleted
imap.expunge

rescue Exception => e
puts "#{ e } (#{ e.class })!"
log.error "#{ e } (#{ e.class })!"
end

imap.logout
log.debug 'Logged out.'

imap.disconnect
log.debug 'Disconnected.'

end_time = Time.now

log.debug "Fetched #{count} e-mails in #{(end_time - last_action_time)} seconds"

log.close

puts "Child process for mailbox #{mailbox[:name]} finished (#{(end_time - last_action_time)} seconds)."
last_action_time = Time.now

end

end
```

If I set in line 76 'w+', I get this error while saving to an eml file. If I set this to 'wb', the file is saved but error is reproduced in a class to which Stalker directs the saved eml message file (around line 50).

Here's the class:

```
class IncomingMailHandler
def self.process_email(file)

email = Mail.read(file)

from = Mail::Address.new email.from.first
to = Mail::Address.new email.to.first

# Ignore the message if it has been sent by the support-app
mailbox = Mailbox.find_by_email_address(self.strip_guid_from_email_address(from.address))
return nil if self.guid_in_email_address?(from.address) and !mailbox.nil?

# Find the related mailbox
mailbox = Mailbox.find_by_email_address(self.strip_guid_from_email_address(to.address))

# Find the related game, if any
game = Game.find_title_in_string(email.subject)

# Is this an answer to an existing ticket?
existing_ticket = self.guid_in_email_address?(to.address)

if existing_ticket
# This is an answer to an existing ticket
guid = self.get_guid_from_email_address(to.address)
ticket = Ticket.find_by_guid(guid)
ticket.game = game unless game.nil?
ticket.closed = false
ticket.save

# If we can't find the ticket, let's assume it's a new one
existing_ticket = false if ticket.nil?
end

unless existing_ticket
# This is a new ticket
ticket = Ticket.create(
:title => email.subject,
:first_message_id => email.message_id,
:mailbox => mailbox,
:game => game.nil? ? nil : game,
:created_at => email.date
)

# Put the user in the queue for asynchronous fetching
Stalker.enqueue('user.fetch', :email_address => from.address, :ticket_id => ticket.id, :delay => 10)
end

# Get the attachments, if any
MMS2R::Media.tmp_dir = "#{::Rails.root.to_s}/tmp/mail_attachments"
emailmedia = MMS2R::Media.new(email)

# Create the message linked to the ticket
message = Message.create(
:message_id => email.message_id,
:title => email.subject,
:from => from.address,
:to => to.address,
:ticket => ticket,
:created_at => email.date
)
emailmedia.process do |media_type, files|
next if files.nil?

case media_type

when 'text/plain'

text_count = 0
files.each do |file|

text_count = text_count + 1

if text_count == 1
# It's the text part of the email
message.text_content = IO.read(file)
else
# It's an attached text file
mail_attachment = MailAttachment.new(:message => message)
File.open(file) { |opened_file| mail_attachment.attachment = opened_file }
mail_attachment.save!
end

end

when 'text/html'

html_count = 0
files.each do |file|

html_count = html_count + 1

if html_count == 1
# It's the HTML part of the email
message.html_content = IO.read(file)
else
# It's an attached HTML file
mail_attachment = MailAttachment.new(:message => message)
File.open(file) { |opened_file| mail_attachment.attachment = opened_file }
mail_attachment.save!
end

end

else

# It's another type of attached file
files.each do |file|

mail_attachment = MailAttachment.new(:message => message)
File.open(file) { |opened_file| mail_attachment.attachment = opened_file }
mail_attachment.save!

end

end

end

# Find the URL of the attached files in the mail body
# and replace them with their URL on the server
if message.html_content
message.html_content = self.replace_cid_by_urls(message.html_content, message.mail_attachments)
end

message.save!

# Delete the temporary files
emailmedia.purge

end

private

def self.get_guid_from_email_address(email_address)
matches = Mail::Address.new(email_address).local.match(/.+\+([a-zA-Z0-9]+)/)
matches[1] unless matches.nil?
end

def self.strip_guid_from_email_address(email_address)
return email_address unless self.guid_in_email_address?(email_address)
email_address.gsub("+" + self.get_guid_from_email_address(email_address), '')
end

def self.guid_in_email_address?(email_address)
!self.get_guid_from_email_address(email_address).nil?
end

def self.replace_cid_by_urls(html,attachments)
html.gsub(/cid:.+\/(.+\.[a-zA-Z]{1,5})/) { |filename|
attachments.select{|att| att.attachment_file_name == $1}.first.url
}
end

end
```

Here's the error:

```
Working email.process (file=/Users/Fallen/Projects/support-app/trunk/tmp/mail_queue/6244AFEF-5052-4A4F-8223-2F94A84927A8@gog.com.eml)
Exception Encoding::UndefinedConversionError -> "\xC5" from ASCII-8BIT to UTF-8
/usr/local/rvm/gems/ruby-1.9.2-p290/gems/mms2r-3.3.1/lib/mms2r/media.rb:424:in `write'
/usr/local/rvm/gems/ruby-1.9.2-p290/gems/mms2r-3.3.1/lib/mms2r/media.rb:424:in `block in process_media'
/usr/local/rvm/gems/ruby-1.9.2-p290/gems/mms2r-3.3.1/lib/mms2r/media.rb:424:in `open'
/usr/local/rvm/gems/ruby-1.9.2-p290/gems/mms2r-3.3.1/lib/mms2r/media.rb:424:in `process_media'
/usr/local/rvm/gems/ruby-1.9.2-p290/gems/mms2r-3.3.1/lib/mms2r/media.rb:360:in `block in process'
/usr/local/rvm/gems/ruby-1.9.2-p290/gems/mms2r-3.3.1/lib/mms2r/media.rb:357:in `each'
/usr/local/rvm/gems/ruby-1.9.2-p290/gems/mms2r-3.3.1/lib/mms2r/media.rb:357:in `process'
/usr/local/rvm/gems/ruby-1.9.2-p290/gems/mms2r-3.3.1/lib/mms2r/media.rb:218:in `initialize'
app/mailers/incoming_mail_handler.rb:50:in `new'
app/mailers/incoming_mail_handler.rb:50:in `process_email'
config/jobs.rb:9:in `block in '
/usr/local/rvm/gems/ruby-1.9.2-p290/gems/stalker-0.9.0/lib/stalker.rb:83:in `call'
/usr/local/rvm/gems/ruby-1.9.2-p290/gems/stalker-0.9.0/lib/stalker.rb:83:in `block in work_one_job'
/usr/local/rvm/rubies/ruby-1.9.2-p290/lib/ruby/1.9.1/timeout.rb:58:in `timeout'
/usr/local/rvm/gems/ruby-1.9.2-p290/gems/stalker-0.9.0/lib/stalker.rb:77:in `work_one_job'
/usr/local/rvm/gems/ruby-1.9.2-p290/gems/stalker-0.9.0/lib/stalker.rb:64:in `block in work'
/usr/local/rvm/gems/ruby-1.9.2-p290/gems/stalker-0.9.0/lib/stalker.rb:64:in `loop'
/usr/local/rvm/gems/ruby-1.9.2-p290/gems/stalker-0.9.0/lib/stalker.rb:64:in `work'
/usr/local/rvm/gems/ruby-1.9.2-p290/gems/stalker-0.9.0/bin/stalk:21:in `'
/usr/local/rvm/gems/ruby-1.9.2-p290/bin/stalk:19:in `load'
/usr/local/rvm/gems/ruby-1.9.2-p290/bin/stalk:19:in `

'
Finished email.process in 128ms (failed)
```