integrate outboxer (with e2e tests)

.github/workflows/ci.yml

@@ -65,5 +65,15 @@ jobs:
           bundle exec rails db:create
           bundle exec rails db:schema:load
 
+      # - name: add Outboxer
+      #   run: |
+      #     echo "gem 'outboxer', git: 'https://github.com/fast-programmer/outboxer.git', branch: 'master'" >> Gemfile
+      #     bundle install
+      #     bin/rails g outboxer:schema
+      #     bin/rake db:migrate
+      #     bin/rake outboxer:db:seed
+      #     bin/rails g outboxer:sidekiq_publisher
+      #     echo "$(sed '$d' lib/event.rb)\n\n  # callbacks\n\n  after_create do |event|\n    Outboxer::Message.queue(messageable: event)\n  end\nend" > lib/event.rb
+
       - name: Run tests
         run: bundle exec rspec

Gemfile

@@ -14,6 +14,8 @@ gem "activerecord"
 
 gem "sidekiq"
 
+gem 'outboxer', git: 'https://github.com/fast-programmer/outboxer.git', branch: 'master'
+
 group :development do
 end
 

Gemfile.lock

@@ -1,3 +1,11 @@
+GIT
+  remote: https://github.com/fast-programmer/outboxer.git
+  revision: 70e02414ed68f58e864f738651dfe4a14b947569
+  branch: master
+  specs:
+    outboxer (1.0.0.pre.beta)
+      activerecord (>= 7.0.8.6)
+
 GEM
   remote: https://rubygems.org/
   specs:
@@ -212,6 +220,7 @@ DEPENDENCIES
   activerecord
   database_cleaner-active_record
   factory_bot_rails
+  outboxer!
   pg
   pry-byebug
   puma

app/jobs/outboxer_integration/message/publish_job.rb

@@ -19,37 +19,37 @@ def perform(args)
         when 'Accountify::Invoice::DraftedEvent'
           Accountify::InvoiceStatusSummary::RegenerateJob.perform_async({
             'tenant_id' => messageable.tenant_id,
-            'organisation_id' => messageable.body['organisation']['id'],
+            'organisation_id' => messageable.body['invoice']['organisation_id'],
             'invoice_updated_at' => messageable.created_at.utc.iso8601 })
 
         when 'Accountify::Invoice::UpdatedEvent'
           Accountify::InvoiceStatusSummary::RegenerateJob.perform_async({
             'tenant_id' => messageable.tenant_id,
-            'organisation_id' => messageable.body['organisation']['id'],
+            'organisation_id' => messageable.body['invoice']['organisation_id'],
             'invoice_updated_at' => messageable.created_at.utc.iso8601 })
 
         when 'Accountify::Invoice::IssuedEvent'
           Accountify::InvoiceStatusSummary::RegenerateJob.perform_async({
             'tenant_id' => messageable.tenant_id,
-            'organisation_id' => messageable.body['organisation']['id'],
+            'organisation_id' => messageable.body['invoice']['organisation_id'],
             'invoice_updated_at' => messageable.created_at.utc.iso8601 })
 
         when 'Accountify::Invoice::PaidEvent'
           Accountify::InvoiceStatusSummary::RegenerateJob.perform_async({
             'tenant_id' => messageable.tenant_id,
-            'organisation_id' => messageable.body['organisation']['id'],
+            'organisation_id' => messageable.body['invoice']['organisation_id'],
             'invoice_updated_at' => messageable.created_at.utc.iso8601 })
 
         when 'Accountify::Invoice::VoidedEvent'
           Accountify::InvoiceStatusSummary::RegenerateJob.perform_async({
             'tenant_id' => messageable.tenant_id,
-            'organisation_id' => messageable.body['organisation']['id'],
+            'organisation_id' => messageable.body['invoice']['organisation_id'],
             'invoice_updated_at' => messageable.created_at.utc.iso8601 })
 
         when 'Accountify::Invoice::DeletedEvent'
           Accountify::InvoiceStatusSummary::RegenerateJob.perform_async({
             'tenant_id' => messageable.tenant_id,
-            'organisation_id' => messageable.body['organisation']['id'],
+            'organisation_id' => messageable.body['invoice']['organisation_id'],
             'invoice_updated_at' => messageable.created_at.utc.iso8601 })
         end
       end

bin/outboxer_publisher

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'sidekiq'
+require 'outboxer'
+
+require_relative '../app/jobs/outboxer_integration/message/publish_job'
+
+options = {
+  env: ENV.fetch('OUTBOXER_ENV', 'development'),
+  buffer: ENV.fetch('OUTBOXER_BUFFER', 1000).to_i,
+  concurrency: ENV.fetch('OUTBOXER_CONCURRENCY', 1).to_i,
+  poll: ENV.fetch('OUTBOXER_POLL', 5.0).to_f,
+  tick: ENV.fetch('OUTBOXER_TICK', 0.1).to_f,
+  heartbeat: ENV.fetch('OUTBOXER_HEARTBEAT', 5.0).to_f,
+  log_level: ENV.fetch('OUTBOXER_LOG_LEVEL', 'INFO'),
+  redis_url: ENV.fetch('OUTBOXER_REDIS_URL', 'redis://localhost:6379/0')
+}
+
+Sidekiq.configure_client do |config|
+  config.redis = { url: options[:redis_url], size: options[:concurrency] }
+end
+
+logger = Outboxer::Logger.new($stdout, level: options[:log_level])
+
+Outboxer::Publisher.publish(
+  env: options[:env],
+  buffer: options[:buffer],
+  concurrency: options[:concurrency],
+  poll: options[:poll],
+  tick: options[:tick],
+  heartbeat: options[:heartbeat],
+  logger: logger
+) do |message|
+  OutboxerIntegration::Message::PublishJob.perform_async({
+    'messageable_id' => message[:messageable_id], 'messageable_type' => message[:messageable_type]
+  })
+end

db/migrate/20241214080817_create_outboxer_settings.rb

@@ -0,0 +1,18 @@
+class CreateOutboxerSettings < ActiveRecord::Migration[6.1]
+  def up
+    ActiveRecord::Base.transaction do
+      create_table :outboxer_settings do |t|
+        t.string :name, limit: 255, null: false
+        t.string :value, limit: 255, null: false
+
+        t.timestamps
+      end
+
+      add_index :outboxer_settings, :name, unique: true
+    end
+  end
+
+  def down
+    drop_table :outboxer_settings
+  end
+end

db/migrate/20241214080818_create_outboxer_messages.rb

@@ -0,0 +1,40 @@
+class CreateOutboxerMessages < ActiveRecord::Migration[6.1]
+  def up
+    create_table :outboxer_messages do |t|
+      t.string :status, limit: 255, null: false
+
+      t.string :messageable_id, limit: 255, null: false
+      t.string :messageable_type, limit: 255, null: false
+
+      t.datetime :queued_at, precision: 6, null: false
+
+      t.datetime :buffered_at, precision: 6
+
+      t.datetime :publishing_at, precision: 6
+
+      t.datetime :updated_at, precision: 6, null: false
+
+      t.bigint :publisher_id
+      t.string :publisher_name, limit: 263 # 255 (hostname) + 1 (colon) + 7 (pid)
+    end
+
+    # messages by status count
+    add_index :outboxer_messages, :status, name: 'idx_outboxer_status'
+
+    # messages by status latency
+    add_index :outboxer_messages, [:status, :updated_at],
+      name: 'idx_outboxer_status_updated_at'
+
+    # publisher latency
+    add_index :outboxer_messages, [:publisher_id, :updated_at],
+      name: 'idx_outboxer_pub_id_updated_at'
+
+    # publisher throughput
+    add_index :outboxer_messages, [:status, :publisher_id, :updated_at],
+      name: 'idx_outboxer_status_pub_id_updated_at'
+  end
+
+  def down
+    drop_table :outboxer_messages if table_exists?(:outboxer_messages)
+  end
+end

db/migrate/20241214080819_create_outboxer_exceptions.rb

@@ -0,0 +1,20 @@
+class CreateOutboxerExceptions < ActiveRecord::Migration[6.1]
+  def up
+    ActiveRecord::Base.transaction do
+      create_table :outboxer_exceptions do |t|
+        t.references :message, foreign_key: { to_table: :outboxer_messages }, null: false
+
+        t.string :class_name, limit: 255, null: false
+        t.text :message_text, null: false
+
+        t.timestamps
+      end
+
+      remove_column :outboxer_exceptions, :updated_at
+    end
+  end
+
+  def down
+    drop_table :outboxer_exceptions if table_exists?(:outboxer_exceptions)
+  end
+end

db/migrate/20241214080820_create_outboxer_frames.rb

@@ -0,0 +1,16 @@
+class CreateOutboxerFrames < ActiveRecord::Migration[6.1]
+  def up
+    create_table :outboxer_frames do |t|
+      t.references :exception, foreign_key: { to_table: :outboxer_exceptions }, null: false
+
+      t.integer :index, null: false
+      t.text :text, null: false
+
+      t.index [:exception_id, :index], unique: true
+    end
+  end
+
+  def down
+    drop_table :outboxer_frames if table_exists?(:outboxer_frames)
+  end
+end

db/migrate/20241214080821_create_outboxer_publishers.rb

@@ -0,0 +1,18 @@
+class CreateOutboxerPublishers < ActiveRecord::Migration[6.1]
+  def up
+    ActiveRecord::Base.transaction do
+      create_table :outboxer_publishers do |t|
+        t.string :name, limit: 263, null: false # 255 (hostname) + 1 (colon) + 7 (pid)
+        t.string :status, limit: 255, null: false
+        t.json :settings, null: false
+        t.json :metrics, null: false
+
+        t.timestamps
+      end
+    end
+  end
+
+  def down
+    drop_table :outboxer_publishers
+  end
+end

db/migrate/20241214080822_create_outboxer_signals.rb

@@ -0,0 +1,16 @@
+class CreateOutboxerSignals < ActiveRecord::Migration[6.1]
+  def up
+    ActiveRecord::Base.transaction do
+      create_table :outboxer_signals do |t|
+        t.string :name, limit: 9, null: false
+        t.references :publisher, foreign_key: { to_table: :outboxer_publishers }, null: false
+
+        t.datetime :created_at, null: false
+      end
+    end
+  end
+
+  def down
+    drop_table :outboxer_signals
+  end
+end

lib/event.rb

@@ -9,4 +9,10 @@ class Event < ActiveRecord::Base
   # associations
 
   belongs_to :eventable, polymorphic: true
+
+  # callbacks
+
+  after_create do |event|
+    Outboxer::Message.queue(messageable: event)
+  end
 end