Fixes: #5022 - Zammad should not write to the var/ folder

Co-authored-by: Mantas Masalskis <mm@zammad.com>
Co-authored-by: Martin Gruner <mg@zammad.com>

.gitignore

@@ -62,8 +62,6 @@
 /tmp/pids/*
 !/tmp/pids/.keep
 /storage/fs
-/var/*
-!/var/README.md
 
 # doorkeeper (OAuth 2)
 /public/assets/doorkeeper

app/models/channel/email_parser.rb

@@ -5,15 +5,13 @@
 class Channel::EmailParser
   include Channel::EmailHelper
 
-  PROZESS_TIME_MAX = 180
+  PROCESS_TIME_MAX = 180
   EMAIL_REGEX = %r{.+@.+}
   RECIPIENT_FIELDS = %w[to cc delivered-to x-original-to envelope-to].freeze
   SENDER_FIELDS = %w[from reply-to return-path sender].freeze
   EXCESSIVE_LINKS_MSG = __('This message cannot be displayed because it contains over 5,000 links. Download the raw message below and open it via an Email client if you still wish to view it.').freeze
   MESSAGE_STRUCT = Struct.new(:from_display_name, :subject, :msg_size).freeze
 
-  UNPROCESSABLE_MAIL_DIRECTORY = Rails.root.join('var/spool/unprocessable_mail')
-
 =begin
 
   parser = Channel::EmailParser.new
@@ -120,24 +118,29 @@ returns
 =end
 
   def process(channel, msg, exception = true)
-
-    Timeout.timeout(PROZESS_TIME_MAX) do
-      _process(channel, msg)
-    end
+    process_with_timeout(channel, msg)
   rescue => e
-    # store unprocessable email for bug reporting
-    filename = archive_mail(UNPROCESSABLE_MAIL_DIRECTORY, msg)
+    failed_email = ::FailedEmail.create(data: msg, parsing_error: e)
 
-    message = "Can't process email, you will find it for bug reporting under #{filename}, please create an issue at https://github.com/zammad/zammad/issues"
+    message = <<~MESSAGE.chomp
+      Can't process email. Run the following command to get the message for issue report at https://github.com/zammad/zammad/issues:
+        zammad run rails r "puts FailedEmail.find(#{failed_email.id}).data"
+    MESSAGE
 
-    p "ERROR: #{message}" # rubocop:disable Rails/Output
-    p "ERROR: #{e.inspect}" # rubocop:disable Rails/Output
+    puts "ERROR: #{message}" # rubocop:disable Rails/Output
+    puts "ERROR: #{e.inspect}" # rubocop:disable Rails/Output
     Rails.logger.error message
     Rails.logger.error e
 
     return false if exception == false
 
-    raise %(#{e.inspect}\n#{e.backtrace.join("\n")})
+    raise failed_email.parsing_error
+  end
+
+  def process_with_timeout(channel, msg)
+    Timeout.timeout(PROCESS_TIME_MAX) do
+      _process(channel, msg)
+    end
   end
 
   def _process(channel, msg)
@@ -497,35 +500,7 @@ returns
     end
   end
 
-=begin
-
-process unprocessable_mails (var/spool/unprocessable_mail/*.eml) again
-
-  Channel::EmailParser.process_unprocessable_mails
-
-=end
-
-  def self.process_unprocessable_mails(params = {})
-    files = []
-    Dir.glob("#{UNPROCESSABLE_MAIL_DIRECTORY}/*.eml") do |entry|
-      ticket, _article, _user, _mail = Channel::EmailParser.new.process(params, File.binread(entry))
-      next if ticket.blank?
-
-      files.push entry
-      File.delete(entry)
-    end
-    files
-  end
-
-=begin
-
-process unprocessable articles provided by the HTMLSanitizer.
-
-  Channel::EmailParser.process_unprocessable_articles
-
-=end
-
-  def self.process_unprocessable_articles(_params = {})
+  def self.reprocess_failed_articles
     articles = Ticket::Article.where(body: ::HtmlSanitizer::UNPROCESSABLE_HTML_MSG)
     articles.reorder(id: :desc).as_batches do |article|
       if !article.as_raw&.content
@@ -940,15 +915,6 @@ process unprocessable articles provided by the HTMLSanitizer.
     [attach]
   end
 
-  # Archive the given message as tmp/folder/md5.eml
-  def archive_mail(path, msg)
-    FileUtils.mkpath path
-
-    path.join("#{Digest::MD5.hexdigest(msg)}.eml").tap do |file_path|
-      File.binwrite(file_path, msg)
-    end
-  end
-
   # Auto reply as the postmaster to oversized emails with:
   # [undeliverable] Message too large
   def postmaster_response(channel, msg)

app/models/failed_email.rb

@@ -0,0 +1,104 @@
+# Copyright (C) 2012-2024 Zammad Foundation, https://zammad-foundation.org/
+
+class FailedEmail < ApplicationModel
+  def reprocess(params = {})
+    ticket, _article, _user, _mail = Channel::EmailParser.new.process_with_timeout(params, data)
+
+    raise __('Unknown error: Could not create a ticket from this email.') if !ticket
+
+    destroy
+
+    ticket
+  rescue => e
+    self.retries += 1
+    self.parsing_error = e
+    save!
+
+    message = "Can't reprocess failed email '#{id}'. This was attempt # #{retries}."
+
+    puts "ERROR: #{message}" # rubocop:disable Rails/Output
+    puts "ERROR: #{e.inspect}" # rubocop:disable Rails/Output
+    Rails.logger.error message
+    Rails.logger.error e
+
+    false
+  end
+
+  def parsing_error=(input)
+    message = case input
+              when StandardError
+                "#{input.inspect}\n#{input.backtrace&.join("\n")}"
+              else
+                input
+              end
+
+    write_attribute :parsing_error, message
+  end
+
+  def self.by_filepath(filepath)
+    id = Pathname
+      .new(filepath)
+      .basename
+      .to_s
+      .delete_suffix('.eml')
+
+    return if id.include?('.')
+
+    find_by(id:)
+  end
+
+  def self.reprocess_all(params = {})
+    reorder(id: :desc)
+      .in_batches
+      .each_record
+      .select { |elem| elem.reprocess(params) }
+      .map { |elem| "#{elem.id}.eml" }
+  end
+
+  def self.generate_path
+    Rails.root.join('tmp', "failed-email-#{SecureRandom.uuid}")
+  end
+
+  def self.export_all(path = generate_path)
+    in_batches
+      .each_record
+      .map { |elem| elem.export(path) }
+  end
+
+  def export(path = self.class.generate_path)
+    FileUtils.mkdir_p(path)
+
+    full_path = path.join("#{id}.eml")
+
+    File.binwrite full_path, data
+
+    full_path
+  end
+
+  def self.import_all(path)
+    Dir
+      .each_child(path)
+      .filter_map do |filename|
+        next if !filename.ends_with?('.eml')
+
+        import(path.join(filename))
+      end
+  end
+
+  def self.import(filepath)
+    failed_email = FailedEmail.by_filepath(filepath.basename)
+
+    new_data = File.binread filepath
+
+    return if new_data == failed_email.data
+
+    failed_email.data = new_data
+    failed_email.parsing_error = nil
+    failed_email.save!
+
+    filepath
+  rescue => e
+    Rails.logger.error e
+    nil
+  end
+end

contrib/backup/functions

@@ -174,7 +174,7 @@ function backup_files () {
       ZAMMAD_STORAGE_DIR="${ZAMMAD_DIR#/}/storage/"
     fi
 
-    tar -C / -czf ${BACKUP_DIR}/${TIMESTAMP}_zammad_files.tar.gz ${ZAMMAD_STORAGE_DIR} ${ZAMMAD_DIR#/}/var/
+    tar -C / -czf ${BACKUP_DIR}/${TIMESTAMP}_zammad_files.tar.gz ${ZAMMAD_STORAGE_DIR}
 
     state=$?
   fi

db/migrate/20120101000001_create_base.rb

@@ -924,5 +924,13 @@ class CreateBase < ActiveRecord::Migration[4.2]
 
       t.timestamps limit: 3, null: false
     end
+
+    create_table :failed_emails do |t|
+      t.binary  :data,         null: false
+      t.integer :retries,      null: false, default: 1
+      t.text    :parsing_error
+
+      t.timestamps limit: 3, null: false
+    end
   end
 end

db/migrate/20230330122449_relocate_unprocessable_mails.rb

@@ -14,7 +14,13 @@ class RelocateUnprocessableMails < ActiveRecord::Migration[6.1]
     return if !old_dir.exist? || old_dir.children.empty?
 
     new_dir = Rails.root.join('var/spool', type)
-    FileUtils.mkdir_p(new_dir)
+    begin
+      # In case of readonly file systems (like in k8s), skip this migration.
+      FileUtils.mkdir_p(new_dir)
+    rescue
+      return
+    end
+
     FileUtils.cp_r(old_dir.children, new_dir)
     FileUtils.rm_r(old_dir)
   end

db/migrate/20240312110258_create_failed_emails.rb

@@ -0,0 +1,50 @@
+# Copyright (C) 2012-2024 Zammad Foundation, https://zammad-foundation.org/
+
+class CreateFailedEmails < ActiveRecord::Migration[7.0]
+  OLD_FAILED_EMAIL_DIRECTORY = Rails.root.join('var/spool/unprocessable_mail')
+
+  def up
+    # return if it's a new setup
+    return if !Setting.exists?(name: 'system_init_done')
+
+    create_table :failed_emails do |t|
+      t.binary  :data,         null: false
+      t.integer :retries,      null: false, default: 1
+      t.text    :parsing_error
+
+      t.timestamps limit: 3, null: false
+    end
+
+    return if !Dir.exist?(OLD_FAILED_EMAIL_DIRECTORY)
+
+    import_emails
+    remove_old_unprocessable_emails
+  end
+
+  def down
+    drop_table :failed_emails
+  end
+
+  private
+
+  def remove_old_unprocessable_emails
+    FileUtils.rm_rf OLD_FAILED_EMAIL_DIRECTORY
+  rescue # handle read-only file systems gracefully
+    nil
+  end
+
+  def import_emails
+    Dir.each_child(OLD_FAILED_EMAIL_DIRECTORY) do |filename|
+      next if !filename.ends_with? '.eml'
+
+      import_single_email(filename)
+    end
+  end
+
+  def import_single_email(filename)
+    path = OLD_FAILED_EMAIL_DIRECTORY.join(filename)
+    data = File.binread(path)
+
+    FailedEmail.create(data:)
+  end
+end

i18n/zammad.pot

@@ -14235,6 +14235,10 @@ msgstr ""
 msgid "Unknown error"
 msgstr ""
 
+#: app/models/failed_email.rb
+msgid "Unknown error: Could not create a ticket from this email."
+msgstr ""
+
 #: app/assets/javascripts/app/views/widget/two_factor_configuration/authenticator_app.jst.eco
 msgid "Unless you already have it, install one of the following authenticator apps on your mobile device:"
 msgstr ""

lib/monitoring_helper/health_checker/failed_email.rb

@@ -0,0 +1,16 @@
+# Copyright (C) 2012-2024 Zammad Foundation, https://zammad-foundation.org/
+
+module MonitoringHelper
+  class HealthChecker
+    class FailedEmail < Backend
+
+      def run_health_check
+        count = ::FailedEmail.count
+
+        return if count.zero?
+
+        response.issues.push "emails that could not be processed: #{count}"
+      end
+    end
+  end
+end

lib/monitoring_helper/health_checker/unprocessable_mail.rb

@@ -1,18 +0,0 @@
-# Copyright (C) 2012-2024 Zammad Foundation, https://zammad-foundation.org/
-
-module MonitoringHelper
-  class HealthChecker
-    class UnprocessableMail < Backend
-
-      def run_health_check
-        return if !File.exist?(Channel::EmailParser::UNPROCESSABLE_MAIL_DIRECTORY)
-
-        count = Dir.glob("#{Channel::EmailParser::UNPROCESSABLE_MAIL_DIRECTORY}/*.eml").count
-
-        return if count.zero?
-
-        response.issues.push "unprocessable mails: #{count}"
-      end
-    end
-  end
-end

lib/tasks/zammad/command.rb

@@ -52,6 +52,17 @@ module Tasks
         # Rake will try to run additional arguments as tasks, so make sure nothing happens for these.
         args[1..].each { |a| Rake::Task.define_task(a.to_sym => :environment) {} } # rubocop:disable Lint/EmptyBlock
       end
+
+      # Rake switches the current working directory to the Rails root.
+      # Make sure that relative pathnames still get resolved correctly.
+      # Note: This works only when invoked via 'rake', not 'rails'!
+      def self.resolve_filepath(path)
+        given_path = Pathname.new(path)
+
+        return given_path if given_path.absolute?
+
+        Pathname.new(Rake.original_dir).join(path)
+      end
     end
   end
 end

lib/tasks/zammad/email_parser/failed_article/reprocess.rake

@@ -0,0 +1,15 @@
+# Copyright (C) 2012-2024 Zammad Foundation, https://zammad-foundation.org/
+
+namespace :zammad do
+  namespace :email_parser do
+    namespace :failed_article do
+
+      desc 'Reprocess articles which failed to parse.'
+      task reprocess_all: :environment do |_task, _args|
+        Channel::EmailParser.reprocess_failed_articles
+        puts 'done.'
+      end
+    end
+
+  end
+end

lib/tasks/zammad/email_parser/failed_email/delete.rake

@@ -0,0 +1,4 @@
+# Copyright (C) 2012-2024 Zammad Foundation, https://zammad-foundation.org/
+
+require_dependency 'tasks/zammad/email_parser/failed_email/delete.rb'
+Tasks::Zammad::EmailParser::FailedEmail::Delete.register_rake_task

lib/tasks/zammad/email_parser/failed_email/delete.rb

@@ -0,0 +1,51 @@
+# Copyright (C) 2012-2024 Zammad Foundation, https://zammad-foundation.org/
+
+require_dependency 'tasks/zammad/command.rb'
+
+module Tasks
+  module Zammad
+    module EmailParser
+      module FailedEmail
+        class Delete < Tasks::Zammad::Command
+
+          def self.usage
+            "#{super} /folder_with_downloaded_emails/spam_email.eml"
+          end
+
+          def self.description
+            'Remove a failed email from to the database.'
+          end
+
+          ARGUMENT_COUNT = 1
+
+          def self.task_handler
+            email_file = resolve_filepath(ArgvHelper.argv[1])
+            failed_email = ::FailedEmail.by_filepath(email_file)
+            raise "No database record could be found for #{email_file}.\n" if !failed_email
+
+            failed_email.destroy
+            puts "Deleting failed email record #{failed_email.id}."
+
+            if email_file.exist?
+              puts "Deleting file #{email_file}."
+              email_file.unlink
+            end
+
+            puts 'Done.'
+          end
+
+          def self.target_path
+            given_path = Pathname.new(ArgvHelper.argv[1])
+
+            return given_path if given_path.absolute?
+
+            Pathname
+              .new(Rake.original_dir)
+              .join(ArgvHelper.argv[1])
+          end
+
+        end
+      end
+    end
+  end
+end

lib/tasks/zammad/email_parser/failed_email/export_all.rake

@@ -0,0 +1,22 @@
+# Copyright (C) 2012-2024 Zammad Foundation, https://zammad-foundation.org/
+
+namespace :zammad do
+  namespace :email_parser do
+    namespace :failed_email do
+
+      desc 'Export all failed emails to a local folder.'
+      task export_all: :environment do |_task, _args|
+        files = FailedEmail.export_all
+        if files.present?
+          puts "#{files.count} failed email(s) exported:"
+          files.each do |f|
+            puts "  #{f}"
+          end
+        else
+          puts 'No failed emails were found.'
+        end
+        puts 'Done.'
+      end
+    end
+  end
+end

lib/tasks/zammad/email_parser/failed_email/import.rake

@@ -0,0 +1,4 @@
+# Copyright (C) 2012-2024 Zammad Foundation, https://zammad-foundation.org/
+
+require_dependency 'tasks/zammad/email_parser/failed_email/import.rb'
+Tasks::Zammad::EmailParser::FailedEmail::Import.register_rake_task

lib/tasks/zammad/email_parser/failed_email/import.rb

@@ -0,0 +1,63 @@
+# Copyright (C) 2012-2024 Zammad Foundation, https://zammad-foundation.org/
+
+require_dependency 'tasks/zammad/command.rb'
+
+module Tasks
+  module Zammad
+    module EmailParser
+      module FailedEmail
+        class Import < Tasks::Zammad::Command
+
+          def self.usage
+            <<~USAGE
+              Usage: bundle exec rails #{task_name} /folder_with_downloaded_emails  # Imports all found .eml files
+                 or: bundle exec rails #{task_name} /folder_with_downloaded_emails/single_file.eml
+            USAGE
+          end
+
+          def self.description
+            'Import locally modified failed emails back to the database.'
+          end
+
+          ARGUMENT_COUNT = 1
+
+          def self.task_handler
+            file_or_folder = resolve_filepath(Pathname.new(ArgvHelper.argv[1]))
+
+            if file_or_folder.directory?
+              import_dir(file_or_folder)
+            else
+              import_file(file_or_folder)
+            end
+
+            puts 'Done.'
+          end
+
+          def self.import_dir(path)
+            imported = ::FailedEmail.import_all(path)
+
+            if imported.blank?
+              puts 'No changed email files could be imported.'
+              return
+            end
+
+            puts "#{imported.count} file(s) imported:"
+            imported.each { |f| puts "  #{f}" }
+          end
+
+          def self.import_file(path)
+            if !path.exist? || path.extname != '.eml'
+              raise "#{path} is not a valid .eml file."
+            end
+
+            if ::FailedEmail.import(path)
+              puts "#{path} was imported."
+            else
+              puts "#{path} was not changed."
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/tasks/zammad/email_parser/failed_email/reprocess.rake

@@ -0,0 +1,21 @@
+# Copyright (C) 2012-2024 Zammad Foundation, https://zammad-foundation.org/
+
+namespace :zammad do
+  namespace :email_parser do
+    namespace :failed_email do
+
+      desc 'Reprocess mails which failed to parse.'
+      task reprocess_all: :environment do |_task, _args|
+        successfully_reprocessed_files = FailedEmail.reprocess_all
+        if successfully_reprocessed_files.present?
+          puts "#{successfully_reprocessed_files.count} email(s) successfully reprocessed:\n"
+          successfully_reprocessed_files.each { |f| puts "  #{f}" }
+        else
+          puts 'No emails were successfully reprocessed.'
+        end
+        puts 'Done.'
+      end
+
+    end
+  end
+end

lib/tasks/zammad/email_parser/reprocess_articles.rake

@@ -1,13 +0,0 @@
-# Copyright (C) 2012-2024 Zammad Foundation, https://zammad-foundation.org/
-
-namespace :zammad do
-
-  namespace :email_parser do
-
-    desc 'Reprocess articles which failed to parse and were saved as unprocessable.'
-    task reprocess_articles: :environment do |_task, _args|
-      Channel::EmailParser.process_unprocessable_articles
-      puts 'done.'
-    end
-  end
-end

lib/tasks/zammad/email_parser/reprocess_mails.rake

@@ -1,13 +0,0 @@
-# Copyright (C) 2012-2024 Zammad Foundation, https://zammad-foundation.org/
-
-namespace :zammad do
-
-  namespace :email_parser do
-
-    desc 'Reprocess mails which failed to parse and were saved as unprocessable.'
-    task reprocess_mails: :environment do |_task, _args|
-      Channel::EmailParser.process_unprocessable_mails
-      puts 'done.'
-    end
-  end
-end

spec/db/migrate/create_failed_emails_spec.rb

@@ -0,0 +1,43 @@
+# Copyright (C) 2012-2024 Zammad Foundation, https://zammad-foundation.org/
+
+require 'rails_helper'
+
+RSpec.describe CreateFailedEmails, db_strategy: :reset, type: :db_migration do
+  before do
+    ActiveRecord::Migration[7.0].drop_table :failed_emails
+    # make sure folder does not exist
+    FileUtils.rm_rf(dir)
+  end
+
+  describe '#up', system_init_done: true do
+    let(:dir) { described_class::OLD_FAILED_EMAIL_DIRECTORY }
+
+    context 'when unprocessable email files exist' do
+      let(:content) { attributes_for(:failed_email)[:data] }
+
+      before do
+        FileUtils.mkdir_p dir
+
+        File.binwrite dir.join('test.eml'), content
+      end
+
+      it 'imports unprocessable email form files' do
+        migrate
+
+        expect(FailedEmail.first).to have_attributes(data: content)
+      end
+
+      it 'removes unprocessable email directory' do
+        expect { migrate }
+          .to change { Dir.exist? dir }
+          .to false
+      end
+    end
+
+    context 'when unprocessable emails directory does not exist' do
+      it 'does not crash' do
+        expect { migrate }.not_to raise_error
+      end
+    end
+  end
+end

spec/db/migrate/relocate_unprocessable_mails_spec.rb

@@ -9,11 +9,12 @@ OVERSIZED_DIR_NEW = Rails.root.join('var/spool/oversized_mail')
 OLD_DIRS = [UNPROCESSABLE_DIR_OLD, OVERSIZED_DIR_OLD].freeze
 NEW_DIRS = [UNPROCESSABLE_DIR_NEW, OVERSIZED_DIR_NEW].freeze
 DIRS = [UNPROCESSABLE_DIR_OLD, UNPROCESSABLE_DIR_NEW, OVERSIZED_DIR_OLD, OVERSIZED_DIR_NEW].freeze
+VAR_DIR = Rails.root.join('var')
 
 RSpec.describe RelocateUnprocessableMails, :aggregate_failures, type: :db_migration do
 
   def remove_all_directories
-    DIRS.each { |dir| FileUtils.rm_r(dir) if File.exist?(dir) }
+    [*DIRS, VAR_DIR].each { |dir| FileUtils.rm_r(dir) if File.exist?(dir) }
   end
 
   def create_old_directories
@@ -52,34 +53,78 @@ RSpec.describe RelocateUnprocessableMails, :aggregate_failures, type: :db_migrat
       end
     end
 
-    context 'with unprocessable mails' do
+    context 'with unprocessable mails present' do
       before do
-        create_all_directories
+        create_old_directories
         %w[unprocessable_mail oversized_mail].each do |type|
           files = [
             "tmp/#{type}/1.eml",
             "tmp/#{type}/2.eml",
-            "var/spool/#{type}/2.eml",
-            "var/spool/#{type}/3.eml",
           ]
           FileUtils.touch(files.map { |f| Rails.root.join(f) })
         end
       end
 
-      it 'removes source directories' do
-        migrate
-        [UNPROCESSABLE_DIR_OLD, OVERSIZED_DIR_OLD].each { |dir| expect(dir).not_to exist }
+      context 'with var folder being not creatable' do
+        before do
+          # Fake a situation with a readonly FS where the var folder cannot be created by
+          #   placing a reguar 'var' file instead. Migration must tolerate this and skip.
+          FileUtils.touch(VAR_DIR)
+        end
+
+        after do
+          # Remove the regular 'var' file again.
+          FileUtils.rm(VAR_DIR)
+        end
+
+        it 'silently skips the migration' do
+          migrate
+          expect([UNPROCESSABLE_DIR_OLD, OVERSIZED_DIR_OLD]).to all(exist)
+        end
       end
 
-      it 'migrates files and keeps existing' do
-        migrate
-        %w[unprocessable_mail oversized_mail].each do |type|
-          files = [
-            "var/spool/#{type}/1.eml",
-            "var/spool/#{type}/2.eml",
-            "var/spool/#{type}/3.eml",
-          ]
-          files.each { |f| expect(Pathname.new(f)).to exist }
+      context 'with var folder being creatable' do
+
+        it 'removes source directories' do
+          migrate
+          [UNPROCESSABLE_DIR_OLD, OVERSIZED_DIR_OLD].each { |dir| expect(dir).not_to exist }
+        end
+
+        it 'migrates files' do
+          migrate
+          %w[unprocessable_mail oversized_mail].each do |type|
+            files = [
+              "var/spool/#{type}/1.eml",
+              "var/spool/#{type}/2.eml",
+            ]
+            files.each { |f| expect(Pathname.new(f)).to exist }
+          end
+        end
+      end
+
+      context 'with var folder being already present and having entries' do
+
+        before do
+          create_all_directories
+          %w[unprocessable_mail oversized_mail].each do |type|
+            files = [
+              "var/spool/#{type}/2.eml",
+              "var/spool/#{type}/3.eml",
+            ]
+            FileUtils.touch(files.map { |f| Rails.root.join(f) })
+          end
+        end
+
+        it 'migrates files and keeps existing' do
+          migrate
+          %w[unprocessable_mail oversized_mail].each do |type|
+            files = [
+              "var/spool/#{type}/1.eml",
+              "var/spool/#{type}/2.eml",
+              "var/spool/#{type}/3.eml",
+            ]
+            files.each { |f| expect(Pathname.new(f)).to exist }
+          end
         end
       end
     end

spec/factories/failed_emails.rb

@@ -0,0 +1,13 @@
+# Copyright (C) 2012-2024 Zammad Foundation, https://zammad-foundation.org/
+
+FactoryBot.define do
+  factory :failed_email do
+    data { <<~MAIL.chomp }
+      From: ME Bob <me@example.com>
+      To: customer@example.com
+      Subject: some subject
+
+      Some Text
+    MAIL
+  end
+end

spec/lib/auto_wizard_spec.rb

@@ -26,14 +26,14 @@ RSpec.describe AutoWizard do
 
     context 'with "auto_wizard.json" file in custom directory' do
       before do
-        allow(ENV).to receive(:[]).with('AUTOWIZARD_RELATIVE_PATH').and_return('var/auto_wizard.json')
+        allow(ENV).to receive(:[]).with('AUTOWIZARD_RELATIVE_PATH').and_return('tmp/auto_wizard.json')
       end
 
       context 'with file present' do
         around do |example|
-          FileUtils.touch(Rails.root.join('var/auto_wizard.json'))
+          FileUtils.touch(Rails.root.join('tmp/auto_wizard.json'))
           example.run
-          FileUtils.rm(Rails.root.join('var/auto_wizard.json'))
+          FileUtils.rm(Rails.root.join('tmp/auto_wizard.json'))
         end
 
         it 'returns true' do

spec/lib/monitoring_helper/health_checker/failed_email_spec.rb

@@ -0,0 +1,18 @@
+# Copyright (C) 2012-2024 Zammad Foundation, https://zammad-foundation.org/
+
+require 'rails_helper'
+
+RSpec.describe MonitoringHelper::HealthChecker::FailedEmail do
+  let(:instance) { described_class.new }
+
+  describe '#check_health' do
+    it 'does nothing if directory missing' do
+      expect(instance.check_health.issues).to be_blank
+    end
+
+    it 'adds issue if unprocessable mails found' do
+      create(:failed_email)
+      expect(instance.check_health.issues.first).to eq 'emails that could not be processed: 1'
+    end
+  end
+end

spec/lib/monitoring_helper/health_checker/unprocessable_mail_spec.rb

@@ -1,33 +0,0 @@
-# Copyright (C) 2012-2024 Zammad Foundation, https://zammad-foundation.org/
-
-require 'rails_helper'
-
-RSpec.describe MonitoringHelper::HealthChecker::UnprocessableMail do
-  let(:instance)  { described_class.new }
-  let(:folder)    { SecureRandom.hex }
-  let(:directory) { Rails.root.join('tmp', folder) }
-
-  before { stub_const('Channel::EmailParser::UNPROCESSABLE_MAIL_DIRECTORY', directory) }
-  after { FileUtils.rm_r(directory) if File.exist?(directory) }
-
-  describe '#check_health' do
-    it 'does nothing if directory missing' do
-      expect(instance.check_health.issues).to be_blank
-    end
-
-    it 'does nothing if no matching files' do
-      FileUtils.mkdir_p directory
-      FileUtils.touch("#{directory}/test.not.email")
-
-      expect(instance.check_health.issues).to be_blank
-    end
-
-    it 'adds issue if unprocessable mails found' do
-      FileUtils.mkdir_p directory
-      FileUtils.touch("#{directory}/test.not.email")
-      FileUtils.touch("#{directory}/test.eml")
-
-      expect(instance.check_health.issues.first).to eq 'unprocessable mails: 1'
-    end
-  end
-end

spec/models/channel/email_parser/process_unprocessable_articles_spec.rb

@@ -2,7 +2,7 @@
 
 require 'rails_helper'
 
-RSpec.describe 'Channel::EmailParser#process_unprocessable_articles', aggregate_failures: true, type: :model do
+RSpec.describe 'Channel::EmailParser#reprocess_failed_articles', aggregate_failures: true, type: :model do
   context 'when receiving unprocessable article' do
     before do
       allow_any_instance_of(HtmlSanitizer::Strict).to receive(:run_sanitization).and_raise(Timeout::Error, HtmlSanitizer::UNPROCESSABLE_HTML_MSG)
@@ -17,7 +17,7 @@ RSpec.describe 'Channel::EmailParser#process_unprocessable_articles', aggregate_
     it 'does reprocess the unprocessable article' do
       expect(Ticket::Article.last.body).to eq(HtmlSanitizer::UNPROCESSABLE_HTML_MSG)
       allow_any_instance_of(HtmlSanitizer::Strict).to receive(:run_sanitization).and_call_original
-      Channel::EmailParser.process_unprocessable_articles
+      Channel::EmailParser.reprocess_failed_articles
       expect(Ticket::Article.last.body).not_to eq(HtmlSanitizer::UNPROCESSABLE_HTML_MSG)
     end
   end

spec/models/channel/email_parser/process_unprocessable_mail_spec.rb

@@ -1,43 +0,0 @@
-# Copyright (C) 2012-2024 Zammad Foundation, https://zammad-foundation.org/
-
-require 'rails_helper'
-
-RSpec.describe 'Channel::EmailParser#process_unprocessable_mail', aggregate_failures: true, type: :model do
-
-  context 'when receiving unprocessable mail' do
-    let(:mail) do
-      <<~MAIL
-        From: ME Bob <me@example.com>
-        To: customer@example.com
-        Subject: some subject
-
-        Some Text
-      MAIL
-    end
-    let(:dir) { Channel::EmailParser::UNPROCESSABLE_MAIL_DIRECTORY }
-
-    before do
-      FileUtils.rm_r(dir) if dir.exist?
-      parser = Channel::EmailParser.new
-      allow(parser).to receive(:_process).and_raise(Timeout::Error)
-      begin
-        parser.process({}, mail)
-      rescue RuntimeError
-        # expected
-      end
-    end
-
-    after do
-      FileUtils.rm_r(dir) if dir.exist?
-    end
-
-    it 'saves the unprocessable email into a file' do
-      expect(dir.join('ce61e7319bcc4297c1d7dfea2fbc87dd.eml')).to exist
-    end
-
-    it 'allows reprocessing of the stored email' do
-      expect { Channel::EmailParser.process_unprocessable_mails }.to change(Ticket, :count).by(1)
-      expect(dir.join('ce61e7319bcc4297c1d7dfea2fbc87dd.eml')).not_to exist
-    end
-  end
-end

spec/models/channel/email_parser_spec.rb

@@ -1581,6 +1581,25 @@ RSpec.describe Channel::EmailParser, type: :model do
         end
       end
     end
+
+    context 'when an unprocessable mail is received' do
+      let(:parser) { described_class.new }
+      let(:mail)   { attributes_for(:failed_email)[:data] }
+
+      before do
+        allow(parser).to receive(:_process).and_raise(Timeout::Error)
+      end
+
+      it 'saves the unprocessable email' do
+        begin
+          parser.process({}, mail)
+        rescue RuntimeError
+          # expected
+        end
+
+        expect(FailedEmail).to be_exist
+      end
+    end
   end
 
   describe '#compose_postmaster_reply' do
@@ -1598,7 +1617,7 @@ RSpec.describe Channel::EmailParser, type: :model do
 
     context 'for English locale (en)' do
       include_examples 'postmaster reply' do
-        let(:locale) { 'en' }
+        let(:locale)           { 'en' }
         let(:expected_subject) { '[undeliverable] Message too large' }
         let(:expected_body) do
           body = <<~BODY

spec/models/failed_email_spec.rb

@@ -0,0 +1,168 @@
+# Copyright (C) 2012-2024 Zammad Foundation, https://zammad-foundation.org/
+
+require 'rails_helper'
+
+RSpec.describe FailedEmail, type: :model do
+  subject(:instance) { create(:failed_email) }
+
+  describe '#parsing_error' do
+    it 'sets parsing error' do
+      instance.parsing_error = 'text'
+
+      expect(instance).to have_attributes(parsing_error: 'text')
+    end
+
+    it 'sets parsing error off error' do
+      instance.parsing_error = StandardError.new('Sample error')
+
+      expect(instance).to have_attributes(parsing_error: include('Sample error'))
+    end
+  end
+
+  describe '.by_filepath' do
+    let!(:failed_email) { create(:failed_email) }
+
+    it 'finds the email by filename' do
+      expect(described_class.by_filepath("some/folder/#{failed_email.id}.eml")).to eq(failed_email)
+    end
+
+    it 'finds the email by id' do
+      expect(described_class.by_filepath(failed_email.id.to_s)).to eq(failed_email)
+    end
+
+    it 'does not find with another extension' do
+      expect(described_class.by_filepath("some/folder/#{failed_email.id}.yml")).to be_nil
+    end
+
+    it 'does not find if not existant' do
+      expect(described_class.by_filepath('1337.eml')).to be_nil
+    end
+  end
+
+  describe '#reprocess' do
+    context 'when it succeeds' do
+      it 'destroys entry' do
+        instance.reprocess
+
+        expect(instance).to be_destroyed
+      end
+
+      it 'creates a ticket' do
+        ticket = instance.reprocess
+
+        expect(ticket.articles.first).to have_attributes(
+          body: 'Some Text'
+        )
+      end
+    end
+
+    context 'when it fails' do
+      before do
+        allow_any_instance_of(Channel::EmailParser)
+          .to receive(:process_with_timeout)
+          .and_return([])
+      end
+
+      it 'increases retries count on failure' do
+        expect { instance.reprocess }
+          .to change(instance, :retries).by(1)
+      end
+
+      it 'does not create a ticket' do
+        expect { instance.reprocess }.not_to change(Ticket, :count)
+      end
+    end
+  end
+
+  describe '.reprocess_all' do
+    let!(:failed_email) { create(:failed_email, data: 'not a mail') }
+    let!(:failed_but_correct_email) { create(:failed_email, data: "From: me\nTo: you\nSubject: Hi\n\ntest") }
+
+    before do
+      failed_email
+      failed_but_correct_email
+    end
+
+    it 'creates one ticket for the parseable mail and keeps the other' do
+      expect { described_class.reprocess_all }
+        .to change(Ticket, :count).by(1)
+        .and(change(described_class, :count).by(-1))
+    end
+
+    it 'returns a list of processed email files' do
+      expect(described_class.reprocess_all).to eq(["#{failed_but_correct_email.id}.eml"])
+    end
+  end
+
+  describe '.export_all' do
+    it 'calls export with all records' do
+      instance
+
+      allow_any_instance_of(described_class)
+        .to receive(:export)
+        .with('path')
+        .and_return('path/file.eml')
+
+      expect(described_class.export_all('path'))
+        .to contain_exactly('path/file.eml')
+    end
+  end
+
+  describe '#export' do
+    it 'creates a file' do
+      path = instance.export
+
+      expect(File.binread(path)).to eq(instance.data)
+    end
+  end
+
+  describe '.import_all' do
+    it 'calls import with all files' do
+      path = described_class.generate_path
+      instance.export(path)
+
+      allow(described_class)
+        .to receive(:import)
+        .with(path.join("#{instance.id}.eml"))
+        .and_return('imported_path')
+
+      expect(described_class.import_all(path))
+        .to contain_exactly('imported_path')
+    end
+  end
+
+  describe '.import' do
+    let(:path)        { described_class.generate_path }
+    let(:file_path)   { instance.export(path) }
+    let(:sample_text) { Faker::Lorem.sentence }
+
+    context 'with changed content' do
+      before { File.binwrite(file_path, sample_text) }
+
+      it 'returns file path for imported file' do
+        expect(described_class.import(path.join("#{instance.id}.eml")))
+          .to eq(file_path)
+      end
+
+      it 'updates record with content of the file' do
+        described_class.import(path.join("#{instance.id}.eml"))
+
+        expect(instance.reload).to have_attributes(
+          data:          sample_text,
+          parsing_error: be_nil
+        )
+      end
+    end
+
+    it 'returns nil if database row does not exist' do
+      expect(described_class.import('tmp/1337.eml')).to be_nil
+    end
+
+    it 'returns nil if database content matches file content' do
+      file_path
+
+      expect(described_class.import(path.join("#{instance.id}.eml")))
+        .to be_nil
+    end
+  end
+end

var/README.md

@@ -1,8 +0,0 @@
-This folder stores persistent application data like unprocessable emails,
-and must be shared in cluster setups to be available for all nodes.
-
-Possible subfolders (depending on system configuration and usage):
-- `var/spool/unprocessable_mail` - stores emails that could not properly be imported
-
-Just for reference, Zammad 6.1 and earlier did also store here:
-- `var/spool/oversized_mail` - emails that were rejected as too big