Merge pull request #459 from coderdojo-japan/add_upcoming_events_task

直近のイベント情報を集計するtaskの追加(仕切り直し)

Author: chicaco

app/models/dojo_event_service.rb

@@ -3,6 +3,8 @@ class DojoEventService < ApplicationRecord
   INTERNAL_SERVICES = %i( static_yaml )
 
   belongs_to :dojo
+  has_many :upcoming_events, dependent: :destroy
+
   enum name: EXTERNAL_SERVICES + INTERNAL_SERVICES
 
   validates :name, presence: true

app/models/upcoming_event.rb

@@ -0,0 +1,12 @@
+class UpcomingEvent < ApplicationRecord
+  belongs_to :dojo_event_service
+
+  validates :service_name, presence: true, uniqueness: { scope: :event_id  }
+  validates :event_id,     presence: true
+  validates :event_url,    presence: true
+  validates :event_at,     presence: true
+  validates :participants, presence: true
+
+  scope :for, ->(service) { where(dojo_event_service: DojoEventService.for(service)) }
+  scope :until, ->(date) { where('event_at < ?', date.beginning_of_day) }
+end

db/migrate/20190526151359_mod_columns_to_upcoming_event.rb

@@ -0,0 +1,19 @@
+class ModColumnsToUpcomingEvent < ActiveRecord::Migration[5.1]
+  def up
+    remove_index :upcoming_events, :event_at
+
+    add_column :upcoming_events, :service_name, :string, null: false
+    add_column :upcoming_events, :participants, :integer, null: false
+
+    add_index :upcoming_events, [:service_name, :event_id], :unique => true
+  end
+
+  def down
+    remove_index :upcoming_events, [:service_name, :event_id]
+
+    remove_column :upcoming_events, :service_name, :string, null: false
+    remove_column :upcoming_events, :participants, :integer, null: false
+
+    add_index :upcoming_events, :event_at, name: "index_upcoming_events_on_event_at"
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20190423141200) do
+ActiveRecord::Schema.define(version: 20190526151359) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -84,8 +84,10 @@
     t.string "event_id", null: false
     t.string "event_url", null: false
     t.datetime "event_at", null: false
+    t.string "service_name", null: false
+    t.integer "participants", null: false
     t.index ["dojo_event_service_id"], name: "index_upcoming_events_on_dojo_event_service_id"
-    t.index ["event_at"], name: "index_upcoming_events_on_event_at"
+    t.index ["service_name", "event_id"], name: "index_upcoming_events_on_service_name_and_event_id", unique: true
   end
 
   add_foreign_key "dojo_event_services", "dojos"

docs/upcoming_events_aggregation.md

@@ -0,0 +1,19 @@
+# rake upcoming_events:aggregation[provider]
+
+## 概要
+
+近日開催(2ヶ月分)のイベント情報を収集する
+
+## 引数
+
+|引数名|型|必須|説明|
+|--|--|--|--|
+|provider|string|(省略可)|集計対象プロバイダ|
+
+## 説明
+
+過去(昨日分まで)のイベント情報を削除し、本日から 2 ヶ月後までのイベント情報を収集する。
+
+provider が指定されたとき、指定プロバイダに対してのみ集計を行う。
+
++ provider には、connpass, doorkeeper, facebook が指定可能。ただし、現時点で facebook は収集対象外のため処理を skip する。

lib/tasks/upcoming_events.rake

@@ -0,0 +1,10 @@
+require_relative '../upcoming_events.rb'
+
+namespace :upcoming_events do
+  desc '指定期間/プロバイダのイベント履歴を集計します'
+  task :aggregation, [:provider] => :environment do |tasks, args|
+    UpcomingEvent.transaction do
+      UpcomingEvents::Aggregation.new(args).run
+    end
+  end
+end

lib/upcoming_events.rb

@@ -0,0 +1,5 @@
+module UpcomingEvents; end
+
+require_relative 'upcoming_events/tasks'
+require_relative 'upcoming_events/aggregation'
+require_relative 'event_service'

lib/upcoming_events/aggregation.rb

@@ -0,0 +1,90 @@
+module UpcomingEvents
+  class Aggregation
+    def initialize(args)
+      @from = Time.zone.today
+      @to = @from + 2.months
+      @provider = args[:provider]
+      # NOTE: 対象は一旦収集可能な connpass, doorkeeper のみにする
+      @externals = fetch_dojos(@provider)
+    end
+
+    def run
+      puts "UpcomingEvents aggregate"
+      with_notifying do
+        delete_upcoming_events
+        execute
+      end
+    end
+
+    private
+
+    def fetch_dojos(provider)
+      base_providers = DojoEventService::EXTERNAL_SERVICES - [:facebook]
+      services = if provider.blank?
+                   # 全プロバイダ対象
+                   base_providers
+                 elsif base_providers.include?(provider.to_sym)
+                   [provider.to_sym]
+                 end
+      return [] unless services
+      find_dojos_by(services)
+    end
+
+    def find_dojos_by(services)
+      services.each.with_object({}) do |name, hash|
+        hash[name] = Dojo.eager_load(:dojo_event_services).where(dojo_event_services: { name: name }).to_a
+      end
+    end
+
+    def with_notifying
+      yield
+      Notifier.notify_success(@provider)
+    rescue => e
+      Notifier.notify_failure(@provider, e)
+    end
+
+    def delete_upcoming_events
+      UpcomingEvent.until(@from).delete_all
+    end
+
+    def execute
+      target_period = @from..@to
+      @externals.each do |kind, list|
+        puts "Aggregate of #{kind}"
+        "UpcomingEvents::Tasks::#{kind.to_s.camelize}".constantize.new(list, target_period).run
+      end
+    end
+
+    class Notifier
+      class << self
+        def notify_success(provider)
+          notify("近日開催イベント情報#{provider_info(provider)}を収集しました")
+        end
+
+        def notify_failure(provider, exception)
+          notify("近日開催イベント情報の収集#{provider_info(provider)}でエラーが発生しました\n#{exception.message}\n#{exception.backtrace.join("\n")}")
+        end
+
+        private
+
+        def provider_info(provider)
+          provider ? "(#{provider})" : nil
+        end
+
+        def idobata_hook_url
+          return @idobata_hook_url if defined?(@idobata_hook_url)
+          @idobata_hook_url = ENV['IDOBATA_HOOK_URL']
+        end
+
+        def notifierable?
+          idobata_hook_url.present?
+        end
+
+        def notify(msg)
+          $stdout.puts msg
+          puts `curl --data-urlencode "source=#{msg}" -s #{idobata_hook_url} -o /dev/null -w "idobata: %{http_code}"` if notifierable?
+        end
+      end
+    end
+  end
+end

lib/upcoming_events/tasks.rb

@@ -0,0 +1,7 @@
+module UpcomingEvents
+  module Tasks
+  end
+end
+
+require_relative 'tasks/connpass'
+require_relative 'tasks/doorkeeper'

lib/upcoming_events/tasks/connpass.rb

@@ -0,0 +1,53 @@
+module UpcomingEvents
+  module Tasks
+    class Connpass
+      def initialize(dojos, period)
+        @client = EventService::Providers::Connpass.new
+        @dojos = dojos
+        @params = build_params(period)
+      end
+
+      def run
+        @dojos.each do |dojo|
+          dojo.dojo_event_services.for(:connpass).each do |dojo_event_service|
+            @client.fetch_events(@params.merge(series_id: dojo_event_service.group_id)).each do |e|
+              next unless e.dig('series', 'id').to_s == dojo_event_service.group_id
+
+              record = dojo_event_service.upcoming_events.find_or_initialize_by(event_id: e['event_id'])
+              record.update!(service_name: dojo_event_service.name,
+                             event_url:    e['event_url'],
+                             event_at:     Time.zone.parse(e['started_at']),
+                             participants: e['accepted'])
+            end
+          end
+        end
+      end
+
+      private
+
+      def build_params(period)
+        yyyymmdd = []
+        yyyymm = []
+
+        st_date = period.first
+        ed_date = period.last
+
+        date = period.first
+        while date <= ed_date
+          if date.day == 1 && date.end_of_month <= ed_date
+            yyyymm << date.strftime('%Y%m')
+            date += 1.month
+          else
+            yyyymmdd << date.strftime('%Y%m%d')
+            date += 1.day
+          end
+        end
+
+        {
+          yyyymmdd: yyyymmdd,
+          yyyymm: yyyymm
+        }
+      end
+    end
+  end
+end

lib/upcoming_events/tasks/doorkeeper.rb

@@ -0,0 +1,36 @@
+module UpcomingEvents
+  module Tasks
+    class Doorkeeper
+      def initialize(dojos, period)
+        @client = EventService::Providers::Doorkeeper.new
+        @dojos = dojos
+        @params = build_params(period)
+      end
+
+      def run
+        @dojos.each do |dojo|
+          dojo.dojo_event_services.for(:doorkeeper).each do |dojo_event_service|
+            @client.fetch_events(@params.merge(group_id: dojo_event_service.group_id)).each do |e|
+              next unless e['group'].to_s == dojo_event_service.group_id
+
+              record = dojo_event_service.upcoming_events.find_or_initialize_by(event_id: e['id'])
+              record.update!(service_name: dojo_event_service.name,
+                             event_url: e['public_url'],
+                             participants: e['participants'],
+                             event_at: Time.zone.parse(e['starts_at']))
+            end
+          end
+        end
+      end
+
+      private
+
+      def build_params(period)
+        {
+          since_at: period.first.beginning_of_day,
+          until_at: period.last.end_of_day
+        }
+      end
+    end
+  end
+end

spec/factories/upcoming_events.rb

@@ -0,0 +1,12 @@
+require 'factory_bot'
+
+FactoryBot.define do
+  factory :upcoming_event do
+    service_name { :connpass }
+    event_id     { '1234' }
+    event_url    { 'http:/www.aaa.com/events/1224' }
+    event_at     { '2019-05-01 10:00'.in_time_zone }
+    participants { 1 }
+  end
+end
+

spec/lib/upcoming_events/aggregation_spec.rb

@@ -0,0 +1,38 @@
+require 'rails_helper'
+require 'upcoming_events'
+
+RSpec.describe UpcomingEvents::Aggregation do
+  include_context 'Use stubs UpcomingEvents for Connpass'
+  include_context 'Use stubs UpcomingEvents for Doorkeeper'
+
+  describe '.run' do
+    before do
+      @d1 = create(:dojo, name: 'Dojo1', email: 'info@dojo1.com', description: 'CoderDojo1', tags: %w(CoderDojo1), url: 'https://dojo1.com')
+      @d2 = create(:dojo, name: 'Dojo2', email: 'info@dojo2.com', description: 'CoderDojo2', tags: %w(CoderDojo2), url: 'https://dojo2.com')
+      @es1 = create(:dojo_event_service, dojo_id: @d1.id, name: :connpass,   group_id: 9876)
+      @es2 = create(:dojo_event_service, dojo_id: @d2.id, name: :doorkeeper, group_id: 5555)
+    end
+
+    it 'プロバイダ指定なし' do
+      expect{ UpcomingEvents::Aggregation.new({}).run }.to change{ UpcomingEvent.count }.from(0).to(3)
+    end
+
+    it 'プロバイダ指定(connpass)' do
+      expect{ UpcomingEvents::Aggregation.new(provider: 'connpass').run }.to change{ UpcomingEvent.count }.from(0).to(1)
+    end
+
+    it 'プロバイダ指定(doorkeeper)' do
+      expect{ UpcomingEvents::Aggregation.new(provider: 'doorkeeper').run }.to change{ UpcomingEvent.count }.from(0).to(2)
+    end
+
+    it '昨日分までは削除' do
+      create(:upcoming_event, dojo_event_service_id: @es1.id, service_name: 'connpass', event_id: '1111', event_at: "#{Time.zone.today - 3.days} 13:00:00".in_time_zone)
+      create(:upcoming_event, dojo_event_service_id: @es1.id, service_name: 'connpass', event_id: '2222', event_at: "#{Time.zone.today - 2.days} 14:00:00".in_time_zone)
+      create(:upcoming_event, dojo_event_service_id: @es1.id, service_name: 'connpass', event_id: '3333', event_at: "#{Time.zone.today - 1.days} 15:00:00".in_time_zone)
+      create(:upcoming_event, dojo_event_service_id: @es2.id, service_name: 'doorkeeper', event_id: '4444', event_at: "#{Time.zone.today - 2.days} 10:00:00".in_time_zone)
+      create(:upcoming_event, dojo_event_service_id: @es2.id, service_name: 'doorkeeper', event_id: '5555', event_at: "#{Time.zone.today - 1.days} 11:00:00".in_time_zone)
+
+      expect{ UpcomingEvents::Aggregation.new({}).run }.to change{ UpcomingEvent.count }.from(5).to(3)
+    end
+  end
+end

spec/support/shared_contexts/statistics.rb

@@ -41,3 +41,39 @@
     ]
   end
 end
+
+RSpec.shared_context 'Use stubs UpcomingEvents for Connpass' do
+  include_context 'Use stub connection of Faraday'
+
+  let(:connpass_response) do
+    [
+      200,
+      { 'Content-Type' => 'application/json' },
+      '{"results_returned": 1, "events": [{"event_url": "https://coderdojo-okutama.connpass.com/event/12345/", "event_type": "participation", "owner_nickname": "nalabjp", "series": {"url": "https://coderdojo-okutama.connpass.com/", "id": 9876, "title": "CoderDojo series"}, "updated_at": "' +
+        "#{Time.zone.today}T14:59:30+09:00" + '", "lat": "35.801763000000", "started_at": "' +
+        "#{Time.zone.today + 1.month}T13:00:00+09:00" + '", "hash_tag": "CoderDojo", "title": "CoderDojo title", "event_id": 12345, "lon": "139.087656000000", "waiting": 2, "limit": 10, "owner_id": 2525, "owner_display_name": "nalabjp", "description": "CoderDojo description", "address": "Okutama-cho Tokyo", "catch": "CoderDojo catch", "accepted": 10, "ended_at": "' +
+        "#{Time.zone.today + 1.month}T15:00:00+09:00" + '", "place": "Tokyo"}], "results_start": 200, "results_available": 518}'
+    ]
+  end
+end
+
+RSpec.shared_context 'Use stubs UpcomingEvents for Doorkeeper' do
+  include_context 'Use stub connection of Faraday'
+
+  let(:doorkeeper_response) do
+    [
+      200,
+      { 'Content-Type' => 'application/json' },
+      '[{"event":{"title":"CoderDojo title","id":1234,"starts_at":"' +
+           "#{Time.zone.today + 1.month}T01:00:00.000Z" + '","ends_at":"' +
+           "#{Time.zone.today + 1.month}T04:00:00.000Z" + '","venue_name":"奥多摩町","address":"奥多摩町","lat":"35.801763000000","long":"139.087656000000","ticket_limit":30,"published_at":"' +
+           "#{Time.zone.today - 4.days}T03:43:04.000Z" + '","updated_at":"' +
+           "#{Time.zone.today}T11:31:21.810Z" + '","group":5555,"banner":null,"description":"CoderDojo description","public_url":"https://coderdojo-okutama.doorkeeper.jp/events/8888","participants":12,"waitlisted":0}},' +
+       '{"event":{"title":"CoderDojo title","id":2345,"starts_at":"' +
+           "#{Time.zone.today + 1.month + 1.day}T01:00:00.000Z" + '","ends_at":"' +
+           "#{Time.zone.today + 1.month + 1.day}T04:00:00.000Z" + '","venue_name":"奥多摩町","address":"奥多摩町","lat":"35.801763000000","long":"139.087656000000","ticket_limit":30,"published_at":"' +
+           "#{Time.zone.today - 4.days}T03:43:04.000Z" + '","updated_at":"' +
+           "#{Time.zone.today}T11:31:21.810Z" + '","group":5555,"banner":null,"description":"CoderDojo description","public_url":"https://coderdojo-okutama.doorkeeper.jp/events/8888","participants":12,"waitlisted":0}}]'
+    ]
+  end
+end