diff --git a/app/models/dojo_event_service.rb b/app/models/dojo_event_service.rb index bcde8ceab..f21cd9b65 100644 --- a/app/models/dojo_event_service.rb +++ b/app/models/dojo_event_service.rb @@ -3,6 +3,8 @@ class DojoEventService < ApplicationRecord INTERNAL_SERVICES = %i( static_yaml ) belongs_to :dojo + has_many :upcoming_events, dependent: :destroy + enum name: EXTERNAL_SERVICES + INTERNAL_SERVICES validates :name, presence: true diff --git a/app/models/upcoming_event.rb b/app/models/upcoming_event.rb new file mode 100644 index 000000000..4ee58436c --- /dev/null +++ b/app/models/upcoming_event.rb @@ -0,0 +1,12 @@ +class UpcomingEvent < ApplicationRecord + belongs_to :dojo_event_service + + validates :service_name, presence: true, uniqueness: { scope: :event_id } + validates :event_id, presence: true + validates :event_url, presence: true + validates :event_at, presence: true + validates :participants, presence: true + + scope :for, ->(service) { where(dojo_event_service: DojoEventService.for(service)) } + scope :until, ->(date) { where('event_at < ?', date.beginning_of_day) } +end diff --git a/db/migrate/20190526151359_mod_columns_to_upcoming_event.rb b/db/migrate/20190526151359_mod_columns_to_upcoming_event.rb new file mode 100644 index 000000000..9a1c8c3d2 --- /dev/null +++ b/db/migrate/20190526151359_mod_columns_to_upcoming_event.rb @@ -0,0 +1,19 @@ +class ModColumnsToUpcomingEvent < ActiveRecord::Migration[5.1] + def up + remove_index :upcoming_events, :event_at + + add_column :upcoming_events, :service_name, :string, null: false + add_column :upcoming_events, :participants, :integer, null: false + + add_index :upcoming_events, [:service_name, :event_id], :unique => true + end + + def down + remove_index :upcoming_events, [:service_name, :event_id] + + remove_column :upcoming_events, :service_name, :string, null: false + remove_column :upcoming_events, :participants, :integer, null: false + + add_index :upcoming_events, :event_at, name: "index_upcoming_events_on_event_at" + end +end diff --git a/db/schema.rb b/db/schema.rb index a88992d37..f37f9f29d 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20190423141200) do +ActiveRecord::Schema.define(version: 20190526151359) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -84,8 +84,10 @@ t.string "event_id", null: false t.string "event_url", null: false t.datetime "event_at", null: false + t.string "service_name", null: false + t.integer "participants", null: false t.index ["dojo_event_service_id"], name: "index_upcoming_events_on_dojo_event_service_id" - t.index ["event_at"], name: "index_upcoming_events_on_event_at" + t.index ["service_name", "event_id"], name: "index_upcoming_events_on_service_name_and_event_id", unique: true end add_foreign_key "dojo_event_services", "dojos" diff --git a/docs/upcoming_events_aggregation.md b/docs/upcoming_events_aggregation.md new file mode 100644 index 000000000..80cbf1d4c --- /dev/null +++ b/docs/upcoming_events_aggregation.md @@ -0,0 +1,19 @@ +# rake upcoming_events:aggregation[provider] + +## 概要 + +近日開催(2ヶ月分)のイベント情報を収集する + +## 引数 + +|引数名|型|必須|説明| +|--|--|--|--| +|provider|string|(省略可)|集計対象プロバイダ| + +## 説明 + +過去(昨日分まで)のイベント情報を削除し、本日から 2 ヶ月後までのイベント情報を収集する。 + +provider が指定されたとき、指定プロバイダに対してのみ集計を行う。 + ++ provider には、connpass, doorkeeper, facebook が指定可能。ただし、現時点で facebook は収集対象外のため処理を skip する。 diff --git a/lib/tasks/upcoming_events.rake b/lib/tasks/upcoming_events.rake new file mode 100644 index 000000000..2ea4d80cf --- /dev/null +++ b/lib/tasks/upcoming_events.rake @@ -0,0 +1,10 @@ +require_relative '../upcoming_events.rb' + +namespace :upcoming_events do + desc '指定期間/プロバイダのイベント履歴を集計します' + task :aggregation, [:provider] => :environment do |tasks, args| + UpcomingEvent.transaction do + UpcomingEvents::Aggregation.new(args).run + end + end +end diff --git a/lib/upcoming_events.rb b/lib/upcoming_events.rb new file mode 100644 index 000000000..b712fc996 --- /dev/null +++ b/lib/upcoming_events.rb @@ -0,0 +1,5 @@ +module UpcomingEvents; end + +require_relative 'upcoming_events/tasks' +require_relative 'upcoming_events/aggregation' +require_relative 'event_service' diff --git a/lib/upcoming_events/aggregation.rb b/lib/upcoming_events/aggregation.rb new file mode 100644 index 000000000..cbf9fc867 --- /dev/null +++ b/lib/upcoming_events/aggregation.rb @@ -0,0 +1,90 @@ +module UpcomingEvents + class Aggregation + def initialize(args) + @from = Time.zone.today + @to = @from + 2.months + @provider = args[:provider] + # NOTE: 対象は一旦収集可能な connpass, doorkeeper のみにする + @externals = fetch_dojos(@provider) + end + + def run + puts "UpcomingEvents aggregate" + with_notifying do + delete_upcoming_events + execute + end + end + + private + + def fetch_dojos(provider) + base_providers = DojoEventService::EXTERNAL_SERVICES - [:facebook] + services = if provider.blank? + # 全プロバイダ対象 + base_providers + elsif base_providers.include?(provider.to_sym) + [provider.to_sym] + end + return [] unless services + find_dojos_by(services) + end + + def find_dojos_by(services) + services.each.with_object({}) do |name, hash| + hash[name] = Dojo.eager_load(:dojo_event_services).where(dojo_event_services: { name: name }).to_a + end + end + + def with_notifying + yield + Notifier.notify_success(@provider) + rescue => e + Notifier.notify_failure(@provider, e) + end + + def delete_upcoming_events + UpcomingEvent.until(@from).delete_all + end + + def execute + target_period = @from..@to + @externals.each do |kind, list| + puts "Aggregate of #{kind}" + "UpcomingEvents::Tasks::#{kind.to_s.camelize}".constantize.new(list, target_period).run + end + end + + class Notifier + class << self + def notify_success(provider) + notify("近日開催イベント情報#{provider_info(provider)}を収集しました") + end + + def notify_failure(provider, exception) + notify("近日開催イベント情報の収集#{provider_info(provider)}でエラーが発生しました\n#{exception.message}\n#{exception.backtrace.join("\n")}") + end + + private + + def provider_info(provider) + provider ? "(#{provider})" : nil + end + + def idobata_hook_url + return @idobata_hook_url if defined?(@idobata_hook_url) + @idobata_hook_url = ENV['IDOBATA_HOOK_URL'] + end + + def notifierable? + idobata_hook_url.present? + end + + def notify(msg) + $stdout.puts msg + puts `curl --data-urlencode "source=#{msg}" -s #{idobata_hook_url} -o /dev/null -w "idobata: %{http_code}"` if notifierable? + end + end + end + end +end diff --git a/lib/upcoming_events/tasks.rb b/lib/upcoming_events/tasks.rb new file mode 100644 index 000000000..8b2a801b8 --- /dev/null +++ b/lib/upcoming_events/tasks.rb @@ -0,0 +1,7 @@ +module UpcomingEvents + module Tasks + end +end + +require_relative 'tasks/connpass' +require_relative 'tasks/doorkeeper' diff --git a/lib/upcoming_events/tasks/connpass.rb b/lib/upcoming_events/tasks/connpass.rb new file mode 100644 index 000000000..590262519 --- /dev/null +++ b/lib/upcoming_events/tasks/connpass.rb @@ -0,0 +1,53 @@ +module UpcomingEvents + module Tasks + class Connpass + def initialize(dojos, period) + @client = EventService::Providers::Connpass.new + @dojos = dojos + @params = build_params(period) + end + + def run + @dojos.each do |dojo| + dojo.dojo_event_services.for(:connpass).each do |dojo_event_service| + @client.fetch_events(@params.merge(series_id: dojo_event_service.group_id)).each do |e| + next unless e.dig('series', 'id').to_s == dojo_event_service.group_id + + record = dojo_event_service.upcoming_events.find_or_initialize_by(event_id: e['event_id']) + record.update!(service_name: dojo_event_service.name, + event_url: e['event_url'], + event_at: Time.zone.parse(e['started_at']), + participants: e['accepted']) + end + end + end + end + + private + + def build_params(period) + yyyymmdd = [] + yyyymm = [] + + st_date = period.first + ed_date = period.last + + date = period.first + while date <= ed_date + if date.day == 1 && date.end_of_month <= ed_date + yyyymm << date.strftime('%Y%m') + date += 1.month + else + yyyymmdd << date.strftime('%Y%m%d') + date += 1.day + end + end + + { + yyyymmdd: yyyymmdd, + yyyymm: yyyymm + } + end + end + end +end diff --git a/lib/upcoming_events/tasks/doorkeeper.rb b/lib/upcoming_events/tasks/doorkeeper.rb new file mode 100644 index 000000000..f9a6a5201 --- /dev/null +++ b/lib/upcoming_events/tasks/doorkeeper.rb @@ -0,0 +1,36 @@ +module UpcomingEvents + module Tasks + class Doorkeeper + def initialize(dojos, period) + @client = EventService::Providers::Doorkeeper.new + @dojos = dojos + @params = build_params(period) + end + + def run + @dojos.each do |dojo| + dojo.dojo_event_services.for(:doorkeeper).each do |dojo_event_service| + @client.fetch_events(@params.merge(group_id: dojo_event_service.group_id)).each do |e| + next unless e['group'].to_s == dojo_event_service.group_id + + record = dojo_event_service.upcoming_events.find_or_initialize_by(event_id: e['id']) + record.update!(service_name: dojo_event_service.name, + event_url: e['public_url'], + participants: e['participants'], + event_at: Time.zone.parse(e['starts_at'])) + end + end + end + end + + private + + def build_params(period) + { + since_at: period.first.beginning_of_day, + until_at: period.last.end_of_day + } + end + end + end +end diff --git a/spec/factories/upcoming_events.rb b/spec/factories/upcoming_events.rb new file mode 100644 index 000000000..fdf91df3b --- /dev/null +++ b/spec/factories/upcoming_events.rb @@ -0,0 +1,12 @@ +require 'factory_bot' + +FactoryBot.define do + factory :upcoming_event do + service_name { :connpass } + event_id { '1234' } + event_url { 'http:/www.aaa.com/events/1224' } + event_at { '2019-05-01 10:00'.in_time_zone } + participants { 1 } + end +end + diff --git a/spec/lib/upcoming_events/aggregation_spec.rb b/spec/lib/upcoming_events/aggregation_spec.rb new file mode 100644 index 000000000..f72257e45 --- /dev/null +++ b/spec/lib/upcoming_events/aggregation_spec.rb @@ -0,0 +1,38 @@ +require 'rails_helper' +require 'upcoming_events' + +RSpec.describe UpcomingEvents::Aggregation do + include_context 'Use stubs UpcomingEvents for Connpass' + include_context 'Use stubs UpcomingEvents for Doorkeeper' + + describe '.run' do + before do + @d1 = create(:dojo, name: 'Dojo1', email: 'info@dojo1.com', description: 'CoderDojo1', tags: %w(CoderDojo1), url: 'https://dojo1.com') + @d2 = create(:dojo, name: 'Dojo2', email: 'info@dojo2.com', description: 'CoderDojo2', tags: %w(CoderDojo2), url: 'https://dojo2.com') + @es1 = create(:dojo_event_service, dojo_id: @d1.id, name: :connpass, group_id: 9876) + @es2 = create(:dojo_event_service, dojo_id: @d2.id, name: :doorkeeper, group_id: 5555) + end + + it 'プロバイダ指定なし' do + expect{ UpcomingEvents::Aggregation.new({}).run }.to change{ UpcomingEvent.count }.from(0).to(3) + end + + it 'プロバイダ指定(connpass)' do + expect{ UpcomingEvents::Aggregation.new(provider: 'connpass').run }.to change{ UpcomingEvent.count }.from(0).to(1) + end + + it 'プロバイダ指定(doorkeeper)' do + expect{ UpcomingEvents::Aggregation.new(provider: 'doorkeeper').run }.to change{ UpcomingEvent.count }.from(0).to(2) + end + + it '昨日分までは削除' do + create(:upcoming_event, dojo_event_service_id: @es1.id, service_name: 'connpass', event_id: '1111', event_at: "#{Time.zone.today - 3.days} 13:00:00".in_time_zone) + create(:upcoming_event, dojo_event_service_id: @es1.id, service_name: 'connpass', event_id: '2222', event_at: "#{Time.zone.today - 2.days} 14:00:00".in_time_zone) + create(:upcoming_event, dojo_event_service_id: @es1.id, service_name: 'connpass', event_id: '3333', event_at: "#{Time.zone.today - 1.days} 15:00:00".in_time_zone) + create(:upcoming_event, dojo_event_service_id: @es2.id, service_name: 'doorkeeper', event_id: '4444', event_at: "#{Time.zone.today - 2.days} 10:00:00".in_time_zone) + create(:upcoming_event, dojo_event_service_id: @es2.id, service_name: 'doorkeeper', event_id: '5555', event_at: "#{Time.zone.today - 1.days} 11:00:00".in_time_zone) + + expect{ UpcomingEvents::Aggregation.new({}).run }.to change{ UpcomingEvent.count }.from(5).to(3) + end + end +end diff --git a/spec/support/shared_contexts/statistics.rb b/spec/support/shared_contexts/statistics.rb index 855e87cbf..b6176066a 100644 --- a/spec/support/shared_contexts/statistics.rb +++ b/spec/support/shared_contexts/statistics.rb @@ -41,3 +41,39 @@ ] end end + +RSpec.shared_context 'Use stubs UpcomingEvents for Connpass' do + include_context 'Use stub connection of Faraday' + + let(:connpass_response) do + [ + 200, + { 'Content-Type' => 'application/json' }, + '{"results_returned": 1, "events": [{"event_url": "https://coderdojo-okutama.connpass.com/event/12345/", "event_type": "participation", "owner_nickname": "nalabjp", "series": {"url": "https://coderdojo-okutama.connpass.com/", "id": 9876, "title": "CoderDojo series"}, "updated_at": "' + + "#{Time.zone.today}T14:59:30+09:00" + '", "lat": "35.801763000000", "started_at": "' + + "#{Time.zone.today + 1.month}T13:00:00+09:00" + '", "hash_tag": "CoderDojo", "title": "CoderDojo title", "event_id": 12345, "lon": "139.087656000000", "waiting": 2, "limit": 10, "owner_id": 2525, "owner_display_name": "nalabjp", "description": "CoderDojo description", "address": "Okutama-cho Tokyo", "catch": "CoderDojo catch", "accepted": 10, "ended_at": "' + + "#{Time.zone.today + 1.month}T15:00:00+09:00" + '", "place": "Tokyo"}], "results_start": 200, "results_available": 518}' + ] + end +end + +RSpec.shared_context 'Use stubs UpcomingEvents for Doorkeeper' do + include_context 'Use stub connection of Faraday' + + let(:doorkeeper_response) do + [ + 200, + { 'Content-Type' => 'application/json' }, + '[{"event":{"title":"CoderDojo title","id":1234,"starts_at":"' + + "#{Time.zone.today + 1.month}T01:00:00.000Z" + '","ends_at":"' + + "#{Time.zone.today + 1.month}T04:00:00.000Z" + '","venue_name":"奥多摩町","address":"奥多摩町","lat":"35.801763000000","long":"139.087656000000","ticket_limit":30,"published_at":"' + + "#{Time.zone.today - 4.days}T03:43:04.000Z" + '","updated_at":"' + + "#{Time.zone.today}T11:31:21.810Z" + '","group":5555,"banner":null,"description":"CoderDojo description","public_url":"https://coderdojo-okutama.doorkeeper.jp/events/8888","participants":12,"waitlisted":0}},' + + '{"event":{"title":"CoderDojo title","id":2345,"starts_at":"' + + "#{Time.zone.today + 1.month + 1.day}T01:00:00.000Z" + '","ends_at":"' + + "#{Time.zone.today + 1.month + 1.day}T04:00:00.000Z" + '","venue_name":"奥多摩町","address":"奥多摩町","lat":"35.801763000000","long":"139.087656000000","ticket_limit":30,"published_at":"' + + "#{Time.zone.today - 4.days}T03:43:04.000Z" + '","updated_at":"' + + "#{Time.zone.today}T11:31:21.810Z" + '","group":5555,"banner":null,"description":"CoderDojo description","public_url":"https://coderdojo-okutama.doorkeeper.jp/events/8888","participants":12,"waitlisted":0}}]' + ] + end +end