From 33a16a61cd2c142a486123c3e25f45a2097ded1a Mon Sep 17 00:00:00 2001 From: Chikako OHNO Date: Mon, 27 May 2019 00:25:32 +0900 Subject: [PATCH 1/6] =?UTF-8?q?=E8=BF=91=E6=97=A5=E9=96=8B=E5=82=AC?= =?UTF-8?q?=E3=82=A4=E3=83=99=E3=83=B3=E3=83=88=E6=83=85=E5=A0=B1=E3=82=92?= =?UTF-8?q?=E5=8F=8E=E9=9B=86=E3=81=99=E3=82=8B=20upcoming=5Fevents:aggreg?= =?UTF-8?q?ation=20=E3=81=AE=E3=82=BF=E3=82=B9=E3=82=AF=E3=82=92=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/dojo.rb | 1 + app/models/dojo_event_service.rb | 1 + app/models/upcoming_event.rb | 13 +++ ...526151359_mod_columns_to_upcoming_event.rb | 23 ++++ db/schema.rb | 11 +- lib/tasks/upcoming_events.rake | 10 ++ lib/upcoming_events.rb | 5 + lib/upcoming_events/aggregation.rb | 106 ++++++++++++++++++ lib/upcoming_events/tasks.rb | 9 ++ lib/upcoming_events/tasks/connpass.rb | 55 +++++++++ lib/upcoming_events/tasks/doorkeeper.rb | 38 +++++++ spec/factories/upcoming_events.rb | 14 +++ spec/lib/upcoming_events/aggregation_spec.rb | 38 +++++++ spec/support/shared_contexts/statistics.rb | 36 ++++++ 14 files changed, 356 insertions(+), 4 deletions(-) create mode 100644 app/models/upcoming_event.rb create mode 100644 db/migrate/20190526151359_mod_columns_to_upcoming_event.rb create mode 100644 lib/tasks/upcoming_events.rake create mode 100644 lib/upcoming_events.rb create mode 100644 lib/upcoming_events/aggregation.rb create mode 100644 lib/upcoming_events/tasks.rb create mode 100644 lib/upcoming_events/tasks/connpass.rb create mode 100644 lib/upcoming_events/tasks/doorkeeper.rb create mode 100644 spec/factories/upcoming_events.rb create mode 100644 spec/lib/upcoming_events/aggregation_spec.rb diff --git a/app/models/dojo.rb b/app/models/dojo.rb index 10185faf8..7f75eea8f 100644 --- a/app/models/dojo.rb +++ b/app/models/dojo.rb @@ -6,6 +6,7 @@ class Dojo < ApplicationRecord belongs_to :prefecture has_many :dojo_event_services, dependent: :destroy has_many :event_histories, dependent: :destroy + has_many :upcoming_events, dependent: :destroy serialize :tags before_save { self.email = self.email.downcase } diff --git a/app/models/dojo_event_service.rb b/app/models/dojo_event_service.rb index bcde8ceab..b6994e01b 100644 --- a/app/models/dojo_event_service.rb +++ b/app/models/dojo_event_service.rb @@ -3,6 +3,7 @@ class DojoEventService < ApplicationRecord INTERNAL_SERVICES = %i( static_yaml ) belongs_to :dojo + enum name: EXTERNAL_SERVICES + INTERNAL_SERVICES validates :name, presence: true diff --git a/app/models/upcoming_event.rb b/app/models/upcoming_event.rb new file mode 100644 index 000000000..803d81b16 --- /dev/null +++ b/app/models/upcoming_event.rb @@ -0,0 +1,13 @@ +class UpcomingEvent < ApplicationRecord + belongs_to :dojo + + validates :dojo_name, presence: true + validates :service_name, presence: true, uniqueness: { scope: :event_id } + validates :event_id, presence: true + validates :event_url, presence: true + validates :event_at, presence: true + validates :participants, presence: true + + scope :for, ->(service) { where(dojo_event_service: DojoEventService.for(service)) } + scope :until, ->(date) { where('event_at < ?', date.beginning_of_day) } +end diff --git a/db/migrate/20190526151359_mod_columns_to_upcoming_event.rb b/db/migrate/20190526151359_mod_columns_to_upcoming_event.rb new file mode 100644 index 000000000..e256a8dfa --- /dev/null +++ b/db/migrate/20190526151359_mod_columns_to_upcoming_event.rb @@ -0,0 +1,23 @@ +class ModColumnsToUpcomingEvent < ActiveRecord::Migration[5.1] + def up + remove_index :upcoming_events, :dojo_event_service_id + remove_index :upcoming_events, :event_at + remove_column :upcoming_events, :dojo_event_service_id + + add_reference :upcoming_events, :dojo, foreign_key: true, index: true, null: false + add_column :upcoming_events, :dojo_name, :string, null: false + add_column :upcoming_events, :service_name, :string, null: false + add_column :upcoming_events, :participants, :integer, null: false + end + + def down + remove_reference :upcoming_events, :dojo, findex: true + remove_column :upcoming_events, :dojo_name, :string, null: false + remove_column :upcoming_events, :service_name, :string, null: false + remove_column :upcoming_events, :participants, :integer, null: false + + add_column :upcoming_events, :dojo_event_service_id, :integer, null: false, default: 1 + add_index :upcoming_events, :dojo_event_service_id, name: "index_upcoming_events_on_dojo_event_service_id" + add_index :upcoming_events, :event_at, name: "index_upcoming_events_on_event_at" + end +end diff --git a/db/schema.rb b/db/schema.rb index a88992d37..78d82f519 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20190423141200) do +ActiveRecord::Schema.define(version: 20190526151359) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -80,14 +80,17 @@ end create_table "upcoming_events", force: :cascade do |t| - t.integer "dojo_event_service_id", null: false t.string "event_id", null: false t.string "event_url", null: false t.datetime "event_at", null: false - t.index ["dojo_event_service_id"], name: "index_upcoming_events_on_dojo_event_service_id" - t.index ["event_at"], name: "index_upcoming_events_on_event_at" + t.bigint "dojo_id", null: false + t.string "dojo_name", null: false + t.string "service_name", null: false + t.integer "participants", null: false + t.index ["dojo_id"], name: "index_upcoming_events_on_dojo_id" end add_foreign_key "dojo_event_services", "dojos" add_foreign_key "event_histories", "dojos" + add_foreign_key "upcoming_events", "dojos" end diff --git a/lib/tasks/upcoming_events.rake b/lib/tasks/upcoming_events.rake new file mode 100644 index 000000000..2ea4d80cf --- /dev/null +++ b/lib/tasks/upcoming_events.rake @@ -0,0 +1,10 @@ +require_relative '../upcoming_events.rb' + +namespace :upcoming_events do + desc '指定期間/プロバイダのイベント履歴を集計します' + task :aggregation, [:provider] => :environment do |tasks, args| + UpcomingEvent.transaction do + UpcomingEvents::Aggregation.new(args).run + end + end +end diff --git a/lib/upcoming_events.rb b/lib/upcoming_events.rb new file mode 100644 index 000000000..b712fc996 --- /dev/null +++ b/lib/upcoming_events.rb @@ -0,0 +1,5 @@ +module UpcomingEvents; end + +require_relative 'upcoming_events/tasks' +require_relative 'upcoming_events/aggregation' +require_relative 'event_service' diff --git a/lib/upcoming_events/aggregation.rb b/lib/upcoming_events/aggregation.rb new file mode 100644 index 000000000..8e4bb38b0 --- /dev/null +++ b/lib/upcoming_events/aggregation.rb @@ -0,0 +1,106 @@ +module UpcomingEvents + class Aggregation + def initialize(args) + @from = Time.zone.today + @to = @from + 2.months + @provider = args[:provider] + dojos = fetch_dojos(@provider) + # NOTE: 対象は一旦収集可能な connpass, doorkeeper のみにする + @externals = dojos[:externals] + # @internals = dojos[:internals] + end + + def run + puts "UpcomingEvents aggregate" + with_notifying do + delete_upcoming_events + execute + end + end + + private + + def fetch_dojos(provider) + if provider.blank? + # 全プロバイダ対象 + external_services = DojoEventService::EXTERNAL_SERVICES + internal_services = DojoEventService::INTERNAL_SERVICES + else + external_services = [] + internal_services = [] + case provider + when 'connpass', 'doorkeeper', 'facebook' + external_services = [provider] + when 'static_yaml' + internal_services = [provider] + end + end + + { + externals: find_dojos_by(external_services), + internals: find_dojos_by(internal_services) + } + end + + def find_dojos_by(services) + services.each.with_object({}) do |name, hash| + hash[name] = Dojo.eager_load(:dojo_event_services).where(dojo_event_services: { name: name }).to_a + end + end + + def with_notifying + yield + Notifier.notify_success(@provider) + rescue => e + Notifier.notify_failure(@provider, e) + end + + def delete_upcoming_events + UpcomingEvent.until(@from).delete_all + end + + def execute + target_period = @from..@to + @externals.each do |kind, list| + if kind == :facebook + puts "Aggregate of #{kind} --> skip" + next + end + puts "Aggregate of #{kind}" + "UpcomingEvents::Tasks::#{kind.to_s.camelize}".constantize.new(list, target_period).run + end + end + + class Notifier + class << self + def notify_success(provider) + notify("近日開催イベント情報#{provider_info(provider)}を収集しました") + end + + def notify_failure(provider, exception) + notify("近日開催イベント情報の収集#{provider_info(provider)}でエラーが発生しました\n#{exception.message}\n#{exception.backtrace.join("\n")}") + end + + private + + def provider_info(provider) + provider ? "(#{provider})" : nil + end + + def idobata_hook_url + return @idobata_hook_url if defined?(@idobata_hook_url) + @idobata_hook_url = ENV['IDOBATA_HOOK_URL'] + end + + def notifierable? + idobata_hook_url.present? + end + + def notify(msg) + $stdout.puts msg + puts `curl --data-urlencode "source=#{msg}" -s #{idobata_hook_url} -o /dev/null -w "idobata: %{http_code}"` if notifierable? + end + end + end + end +end diff --git a/lib/upcoming_events/tasks.rb b/lib/upcoming_events/tasks.rb new file mode 100644 index 000000000..058808feb --- /dev/null +++ b/lib/upcoming_events/tasks.rb @@ -0,0 +1,9 @@ +module UpcomingEvents + module Tasks + end +end + +require_relative 'tasks/connpass' +require_relative 'tasks/doorkeeper' +# require_relative 'tasks/facebook' +# require_relative 'tasks/static_yaml' diff --git a/lib/upcoming_events/tasks/connpass.rb b/lib/upcoming_events/tasks/connpass.rb new file mode 100644 index 000000000..2400b9b8b --- /dev/null +++ b/lib/upcoming_events/tasks/connpass.rb @@ -0,0 +1,55 @@ +module UpcomingEvents + module Tasks + class Connpass + def initialize(dojos, period) + @client = EventService::Providers::Connpass.new + @dojos = dojos + @params = build_params(period) + end + + def run + @dojos.each do |dojo| + dojo.dojo_event_services.for(:connpass).each do |dojo_event_service| + @client.fetch_events(@params.merge(series_id: dojo_event_service.group_id)).each do |e| + next unless e.dig('series', 'id').to_s == dojo_event_service.group_id + + record = UpcomingEvent.find_or_initialize_by(dojo_id: dojo.id, + service_name: dojo_event_service.name, + event_id: e['event_id']) + record.update!(dojo_name: dojo.name, + event_url: e['event_url'], + event_at: Time.zone.parse(e['started_at']), + participants: e['accepted']) + end + end + end + end + + private + + def build_params(period) + yyyymmdd = [] + yyyymm = [] + + st_date = period.first + ed_date = period.last + + date = period.first + while date <= ed_date + if date.day == 1 && date.end_of_month <= ed_date + yyyymm << date.strftime('%Y%m') + date += 1.month + else + yyyymmdd << date.strftime('%Y%m%d') + date += 1.day + end + end + + { + yyyymmdd: yyyymmdd, + yyyymm: yyyymm + } + end + end + end +end diff --git a/lib/upcoming_events/tasks/doorkeeper.rb b/lib/upcoming_events/tasks/doorkeeper.rb new file mode 100644 index 000000000..c596e29c4 --- /dev/null +++ b/lib/upcoming_events/tasks/doorkeeper.rb @@ -0,0 +1,38 @@ +module UpcomingEvents + module Tasks + class Doorkeeper + def initialize(dojos, period) + @client = EventService::Providers::Doorkeeper.new + @dojos = dojos + @params = build_params(period) + end + + def run + @dojos.each do |dojo| + dojo.dojo_event_services.for(:doorkeeper).each do |dojo_event_service| + @client.fetch_events(@params.merge(group_id: dojo_event_service.group_id)).each do |e| + next unless e['group'].to_s == dojo_event_service.group_id + + record = UpcomingEvent.find_or_initialize_by(dojo_id: dojo.id, + service_name: dojo_event_service.name, + event_id: e['id']) + record.update!(dojo_name: dojo.name, + event_url: e['public_url'], + participants: e['participants'], + event_at: Time.zone.parse(e['starts_at'])) + end + end + end + end + + private + + def build_params(period) + { + since_at: period.first.beginning_of_day, + until_at: period.last.end_of_day + } + end + end + end +end diff --git a/spec/factories/upcoming_events.rb b/spec/factories/upcoming_events.rb new file mode 100644 index 000000000..3c49ba829 --- /dev/null +++ b/spec/factories/upcoming_events.rb @@ -0,0 +1,14 @@ +require 'factory_bot' + +FactoryBot.define do + factory :upcoming_event do + # dojo_id + dojo_name { 'Dojo Name' } + service_name { :connpass } + event_id { '1234' } + event_url { 'http:/www.aaa.com/events/1224' } + event_at { '2019-05-01 10:00'.in_time_zone } + participants { 1 } + end +end + diff --git a/spec/lib/upcoming_events/aggregation_spec.rb b/spec/lib/upcoming_events/aggregation_spec.rb new file mode 100644 index 000000000..788b22ce0 --- /dev/null +++ b/spec/lib/upcoming_events/aggregation_spec.rb @@ -0,0 +1,38 @@ +require 'rails_helper' +require 'upcoming_events' + +RSpec.describe UpcomingEvents::Aggregation do + include_context 'Use stubs UpcomingEvents for Connpass' + include_context 'Use stubs UpcomingEvents for Doorkeeper' + + describe '.run' do + before do + @d1 = create(:dojo, name: 'Dojo1', email: 'info@dojo1.com', description: 'CoderDojo1', tags: %w(CoderDojo1), url: 'https://dojo1.com') + @d2 = create(:dojo, name: 'Dojo2', email: 'info@dojo2.com', description: 'CoderDojo2', tags: %w(CoderDojo2), url: 'https://dojo2.com') + create(:dojo_event_service, dojo_id: @d1.id, name: :connpass, group_id: 9876) + create(:dojo_event_service, dojo_id: @d2.id, name: :doorkeeper, group_id: 5555) + end + + it 'プロバイダ指定なし' do + expect{ UpcomingEvents::Aggregation.new({}).run }.to change{ UpcomingEvent.count }.from(0).to(3) + end + + it 'プロバイダ指定(connpass)' do + expect{ UpcomingEvents::Aggregation.new(provider: 'connpass').run }.to change{ UpcomingEvent.count }.from(0).to(1) + end + + it 'プロバイダ指定(doorkeeper)' do + expect{ UpcomingEvents::Aggregation.new(provider: 'doorkeeper').run }.to change{ UpcomingEvent.count }.from(0).to(2) + end + + it '昨日分までは削除' do + create(:upcoming_event, dojo_id: @d1.id, service_name: 'connpass', event_id: '1111', event_at: "#{Time.zone.today - 3.days} 13:00:00".in_time_zone) + create(:upcoming_event, dojo_id: @d1.id, service_name: 'connpass', event_id: '2222', event_at: "#{Time.zone.today - 2.days} 14:00:00".in_time_zone) + create(:upcoming_event, dojo_id: @d1.id, service_name: 'connpass', event_id: '3333', event_at: "#{Time.zone.today - 1.days} 15:00:00".in_time_zone) + create(:upcoming_event, dojo_id: @d2.id, service_name: 'doorkeeper', event_id: '4444', event_at: "#{Time.zone.today - 2.days} 10:00:00".in_time_zone) + create(:upcoming_event, dojo_id: @d2.id, service_name: 'doorkeeper', event_id: '5555', event_at: "#{Time.zone.today - 1.days} 11:00:00".in_time_zone) + + expect{ UpcomingEvents::Aggregation.new({}).run }.to change{ UpcomingEvent.count }.from(5).to(3) + end + end +end diff --git a/spec/support/shared_contexts/statistics.rb b/spec/support/shared_contexts/statistics.rb index 855e87cbf..b6176066a 100644 --- a/spec/support/shared_contexts/statistics.rb +++ b/spec/support/shared_contexts/statistics.rb @@ -41,3 +41,39 @@ ] end end + +RSpec.shared_context 'Use stubs UpcomingEvents for Connpass' do + include_context 'Use stub connection of Faraday' + + let(:connpass_response) do + [ + 200, + { 'Content-Type' => 'application/json' }, + '{"results_returned": 1, "events": [{"event_url": "https://coderdojo-okutama.connpass.com/event/12345/", "event_type": "participation", "owner_nickname": "nalabjp", "series": {"url": "https://coderdojo-okutama.connpass.com/", "id": 9876, "title": "CoderDojo series"}, "updated_at": "' + + "#{Time.zone.today}T14:59:30+09:00" + '", "lat": "35.801763000000", "started_at": "' + + "#{Time.zone.today + 1.month}T13:00:00+09:00" + '", "hash_tag": "CoderDojo", "title": "CoderDojo title", "event_id": 12345, "lon": "139.087656000000", "waiting": 2, "limit": 10, "owner_id": 2525, "owner_display_name": "nalabjp", "description": "CoderDojo description", "address": "Okutama-cho Tokyo", "catch": "CoderDojo catch", "accepted": 10, "ended_at": "' + + "#{Time.zone.today + 1.month}T15:00:00+09:00" + '", "place": "Tokyo"}], "results_start": 200, "results_available": 518}' + ] + end +end + +RSpec.shared_context 'Use stubs UpcomingEvents for Doorkeeper' do + include_context 'Use stub connection of Faraday' + + let(:doorkeeper_response) do + [ + 200, + { 'Content-Type' => 'application/json' }, + '[{"event":{"title":"CoderDojo title","id":1234,"starts_at":"' + + "#{Time.zone.today + 1.month}T01:00:00.000Z" + '","ends_at":"' + + "#{Time.zone.today + 1.month}T04:00:00.000Z" + '","venue_name":"奥多摩町","address":"奥多摩町","lat":"35.801763000000","long":"139.087656000000","ticket_limit":30,"published_at":"' + + "#{Time.zone.today - 4.days}T03:43:04.000Z" + '","updated_at":"' + + "#{Time.zone.today}T11:31:21.810Z" + '","group":5555,"banner":null,"description":"CoderDojo description","public_url":"https://coderdojo-okutama.doorkeeper.jp/events/8888","participants":12,"waitlisted":0}},' + + '{"event":{"title":"CoderDojo title","id":2345,"starts_at":"' + + "#{Time.zone.today + 1.month + 1.day}T01:00:00.000Z" + '","ends_at":"' + + "#{Time.zone.today + 1.month + 1.day}T04:00:00.000Z" + '","venue_name":"奥多摩町","address":"奥多摩町","lat":"35.801763000000","long":"139.087656000000","ticket_limit":30,"published_at":"' + + "#{Time.zone.today - 4.days}T03:43:04.000Z" + '","updated_at":"' + + "#{Time.zone.today}T11:31:21.810Z" + '","group":5555,"banner":null,"description":"CoderDojo description","public_url":"https://coderdojo-okutama.doorkeeper.jp/events/8888","participants":12,"waitlisted":0}}]' + ] + end +end From 825070878372f59e75f2655f4b992d3f42a023fb Mon Sep 17 00:00:00 2001 From: Chikako OHNO Date: Mon, 27 May 2019 00:35:06 +0900 Subject: [PATCH 2/6] =?UTF-8?q?rake=20upcoming=5Fevents:aggregation=20?= =?UTF-8?q?=E3=81=AE=E8=AA=AC=E6=98=8E=E3=83=89=E3=82=AD=E3=83=A5=E3=83=A1?= =?UTF-8?q?=E3=83=B3=E3=83=88=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/upcoming_events_aggregation.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 docs/upcoming_events_aggregation.md diff --git a/docs/upcoming_events_aggregation.md b/docs/upcoming_events_aggregation.md new file mode 100644 index 000000000..80cbf1d4c --- /dev/null +++ b/docs/upcoming_events_aggregation.md @@ -0,0 +1,19 @@ +# rake upcoming_events:aggregation[provider] + +## 概要 + +近日開催(2ヶ月分)のイベント情報を収集する + +## 引数 + +|引数名|型|必須|説明| +|--|--|--|--| +|provider|string|(省略可)|集計対象プロバイダ| + +## 説明 + +過去(昨日分まで)のイベント情報を削除し、本日から 2 ヶ月後までのイベント情報を収集する。 + +provider が指定されたとき、指定プロバイダに対してのみ集計を行う。 + ++ provider には、connpass, doorkeeper, facebook が指定可能。ただし、現時点で facebook は収集対象外のため処理を skip する。 From 41cc1cbd0eeb9aa4c9614a84adaee2c3d802f2a1 Mon Sep 17 00:00:00 2001 From: Chikako OHNO Date: Tue, 28 May 2019 00:18:51 +0900 Subject: [PATCH 3/6] =?UTF-8?q?upcoming=5Fevents:aggregation=20=E3=81=AE?= =?UTF-8?q?=E3=82=BF=E3=82=B9=E3=82=AF=E3=81=AE=E3=83=AA=E3=83=95=E3=82=A1?= =?UTF-8?q?=E3=82=AF=E3=82=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/upcoming_events/aggregation.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/upcoming_events/aggregation.rb b/lib/upcoming_events/aggregation.rb index 8e4bb38b0..56efbd094 100644 --- a/lib/upcoming_events/aggregation.rb +++ b/lib/upcoming_events/aggregation.rb @@ -30,9 +30,9 @@ def fetch_dojos(provider) internal_services = [] case provider when 'connpass', 'doorkeeper', 'facebook' - external_services = [provider] + external_services = [provider.to_sym] when 'static_yaml' - internal_services = [provider] + internal_services = [provider.to_sym] end end @@ -62,7 +62,7 @@ def delete_upcoming_events def execute target_period = @from..@to @externals.each do |kind, list| - if kind == :facebook + unless [:connpass, :doorkeeper].include?(kind) puts "Aggregate of #{kind} --> skip" next end From 79c3150817e3d940827119d9b8ccfdec1d1b6724 Mon Sep 17 00:00:00 2001 From: Chikako OHNO Date: Wed, 29 May 2019 00:59:37 +0900 Subject: [PATCH 4/6] =?UTF-8?q?=E3=83=AC=E3=83=93=E3=83=A5=E3=83=BC?= =?UTF-8?q?=E6=8C=87=E6=91=98=E4=BA=8B=E9=A0=85=E5=AF=BE=E5=BF=9C=20?= =?UTF-8?q?=E3=83=BB=E9=96=A2=E9=80=A3=E8=A6=8B=E7=9B=B4=E3=81=97=20?= =?UTF-8?q?=E3=83=BB=E3=83=AA=E3=83=95=E3=82=A1=E3=82=AF=E3=82=BF=20?= =?UTF-8?q?=E3=83=BB=E4=B8=8D=E8=A6=81=E3=81=AA=E3=82=B3=E3=83=A1=E3=83=B3?= =?UTF-8?q?=E3=83=88=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/dojo.rb | 1 - app/models/dojo_event_service.rb | 1 + app/models/upcoming_event.rb | 2 +- ...526151359_mod_columns_to_upcoming_event.rb | 6 ---- db/schema.rb | 5 ++- lib/upcoming_events/aggregation.rb | 36 ++++++------------- lib/upcoming_events/tasks.rb | 2 -- lib/upcoming_events/tasks/connpass.rb | 12 +++---- lib/upcoming_events/tasks/doorkeeper.rb | 6 ++-- spec/factories/upcoming_events.rb | 1 - spec/lib/upcoming_events/aggregation_spec.rb | 14 ++++---- 11 files changed, 30 insertions(+), 56 deletions(-) diff --git a/app/models/dojo.rb b/app/models/dojo.rb index 7f75eea8f..10185faf8 100644 --- a/app/models/dojo.rb +++ b/app/models/dojo.rb @@ -6,7 +6,6 @@ class Dojo < ApplicationRecord belongs_to :prefecture has_many :dojo_event_services, dependent: :destroy has_many :event_histories, dependent: :destroy - has_many :upcoming_events, dependent: :destroy serialize :tags before_save { self.email = self.email.downcase } diff --git a/app/models/dojo_event_service.rb b/app/models/dojo_event_service.rb index b6994e01b..f21cd9b65 100644 --- a/app/models/dojo_event_service.rb +++ b/app/models/dojo_event_service.rb @@ -3,6 +3,7 @@ class DojoEventService < ApplicationRecord INTERNAL_SERVICES = %i( static_yaml ) belongs_to :dojo + has_many :upcoming_events, dependent: :destroy enum name: EXTERNAL_SERVICES + INTERNAL_SERVICES diff --git a/app/models/upcoming_event.rb b/app/models/upcoming_event.rb index 803d81b16..41c123df2 100644 --- a/app/models/upcoming_event.rb +++ b/app/models/upcoming_event.rb @@ -1,5 +1,5 @@ class UpcomingEvent < ApplicationRecord - belongs_to :dojo + belongs_to :dojo_event_service validates :dojo_name, presence: true validates :service_name, presence: true, uniqueness: { scope: :event_id } diff --git a/db/migrate/20190526151359_mod_columns_to_upcoming_event.rb b/db/migrate/20190526151359_mod_columns_to_upcoming_event.rb index e256a8dfa..ca4f3d4ad 100644 --- a/db/migrate/20190526151359_mod_columns_to_upcoming_event.rb +++ b/db/migrate/20190526151359_mod_columns_to_upcoming_event.rb @@ -1,23 +1,17 @@ class ModColumnsToUpcomingEvent < ActiveRecord::Migration[5.1] def up - remove_index :upcoming_events, :dojo_event_service_id remove_index :upcoming_events, :event_at - remove_column :upcoming_events, :dojo_event_service_id - add_reference :upcoming_events, :dojo, foreign_key: true, index: true, null: false add_column :upcoming_events, :dojo_name, :string, null: false add_column :upcoming_events, :service_name, :string, null: false add_column :upcoming_events, :participants, :integer, null: false end def down - remove_reference :upcoming_events, :dojo, findex: true remove_column :upcoming_events, :dojo_name, :string, null: false remove_column :upcoming_events, :service_name, :string, null: false remove_column :upcoming_events, :participants, :integer, null: false - add_column :upcoming_events, :dojo_event_service_id, :integer, null: false, default: 1 - add_index :upcoming_events, :dojo_event_service_id, name: "index_upcoming_events_on_dojo_event_service_id" add_index :upcoming_events, :event_at, name: "index_upcoming_events_on_event_at" end end diff --git a/db/schema.rb b/db/schema.rb index 78d82f519..94c32014a 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -83,14 +83,13 @@ t.string "event_id", null: false t.string "event_url", null: false t.datetime "event_at", null: false - t.bigint "dojo_id", null: false + t.integer "dojo_event_service_id", default: 1, null: false t.string "dojo_name", null: false t.string "service_name", null: false t.integer "participants", null: false - t.index ["dojo_id"], name: "index_upcoming_events_on_dojo_id" + t.index ["dojo_event_service_id"], name: "index_upcoming_events_on_dojo_event_service_id" end add_foreign_key "dojo_event_services", "dojos" add_foreign_key "event_histories", "dojos" - add_foreign_key "upcoming_events", "dojos" end diff --git a/lib/upcoming_events/aggregation.rb b/lib/upcoming_events/aggregation.rb index 56efbd094..cbf9fc867 100644 --- a/lib/upcoming_events/aggregation.rb +++ b/lib/upcoming_events/aggregation.rb @@ -4,10 +4,8 @@ def initialize(args) @from = Time.zone.today @to = @from + 2.months @provider = args[:provider] - dojos = fetch_dojos(@provider) # NOTE: 対象は一旦収集可能な connpass, doorkeeper のみにする - @externals = dojos[:externals] - # @internals = dojos[:internals] + @externals = fetch_dojos(@provider) end def run @@ -21,25 +19,15 @@ def run private def fetch_dojos(provider) - if provider.blank? - # 全プロバイダ対象 - external_services = DojoEventService::EXTERNAL_SERVICES - internal_services = DojoEventService::INTERNAL_SERVICES - else - external_services = [] - internal_services = [] - case provider - when 'connpass', 'doorkeeper', 'facebook' - external_services = [provider.to_sym] - when 'static_yaml' - internal_services = [provider.to_sym] - end - end - - { - externals: find_dojos_by(external_services), - internals: find_dojos_by(internal_services) - } + base_providers = DojoEventService::EXTERNAL_SERVICES - [:facebook] + services = if provider.blank? + # 全プロバイダ対象 + base_providers + elsif base_providers.include?(provider.to_sym) + [provider.to_sym] + end + return [] unless services + find_dojos_by(services) end def find_dojos_by(services) @@ -62,10 +50,6 @@ def delete_upcoming_events def execute target_period = @from..@to @externals.each do |kind, list| - unless [:connpass, :doorkeeper].include?(kind) - puts "Aggregate of #{kind} --> skip" - next - end puts "Aggregate of #{kind}" "UpcomingEvents::Tasks::#{kind.to_s.camelize}".constantize.new(list, target_period).run end diff --git a/lib/upcoming_events/tasks.rb b/lib/upcoming_events/tasks.rb index 058808feb..8b2a801b8 100644 --- a/lib/upcoming_events/tasks.rb +++ b/lib/upcoming_events/tasks.rb @@ -5,5 +5,3 @@ module Tasks require_relative 'tasks/connpass' require_relative 'tasks/doorkeeper' -# require_relative 'tasks/facebook' -# require_relative 'tasks/static_yaml' diff --git a/lib/upcoming_events/tasks/connpass.rb b/lib/upcoming_events/tasks/connpass.rb index 2400b9b8b..423c07c95 100644 --- a/lib/upcoming_events/tasks/connpass.rb +++ b/lib/upcoming_events/tasks/connpass.rb @@ -13,12 +13,12 @@ def run @client.fetch_events(@params.merge(series_id: dojo_event_service.group_id)).each do |e| next unless e.dig('series', 'id').to_s == dojo_event_service.group_id - record = UpcomingEvent.find_or_initialize_by(dojo_id: dojo.id, - service_name: dojo_event_service.name, - event_id: e['event_id']) - record.update!(dojo_name: dojo.name, - event_url: e['event_url'], - event_at: Time.zone.parse(e['started_at']), + record = dojo_event_service.upcoming_events.find_by(event_id: e['event_id']) + record ||= dojo_event_service.upcoming_events.build(event_id: e['event_id']) + record.update!(dojo_name: dojo.name, + service_name: dojo_event_service.name, + event_url: e['event_url'], + event_at: Time.zone.parse(e['started_at']), participants: e['accepted']) end end diff --git a/lib/upcoming_events/tasks/doorkeeper.rb b/lib/upcoming_events/tasks/doorkeeper.rb index c596e29c4..3ee6809b2 100644 --- a/lib/upcoming_events/tasks/doorkeeper.rb +++ b/lib/upcoming_events/tasks/doorkeeper.rb @@ -13,10 +13,10 @@ def run @client.fetch_events(@params.merge(group_id: dojo_event_service.group_id)).each do |e| next unless e['group'].to_s == dojo_event_service.group_id - record = UpcomingEvent.find_or_initialize_by(dojo_id: dojo.id, - service_name: dojo_event_service.name, - event_id: e['id']) + record = dojo_event_service.upcoming_events.find_by(event_id: e['id']) + record ||= dojo_event_service.upcoming_events.build(event_id: e['id']) record.update!(dojo_name: dojo.name, + service_name: dojo_event_service.name, event_url: e['public_url'], participants: e['participants'], event_at: Time.zone.parse(e['starts_at'])) diff --git a/spec/factories/upcoming_events.rb b/spec/factories/upcoming_events.rb index 3c49ba829..f2c0fc5d4 100644 --- a/spec/factories/upcoming_events.rb +++ b/spec/factories/upcoming_events.rb @@ -2,7 +2,6 @@ FactoryBot.define do factory :upcoming_event do - # dojo_id dojo_name { 'Dojo Name' } service_name { :connpass } event_id { '1234' } diff --git a/spec/lib/upcoming_events/aggregation_spec.rb b/spec/lib/upcoming_events/aggregation_spec.rb index 788b22ce0..f72257e45 100644 --- a/spec/lib/upcoming_events/aggregation_spec.rb +++ b/spec/lib/upcoming_events/aggregation_spec.rb @@ -9,8 +9,8 @@ before do @d1 = create(:dojo, name: 'Dojo1', email: 'info@dojo1.com', description: 'CoderDojo1', tags: %w(CoderDojo1), url: 'https://dojo1.com') @d2 = create(:dojo, name: 'Dojo2', email: 'info@dojo2.com', description: 'CoderDojo2', tags: %w(CoderDojo2), url: 'https://dojo2.com') - create(:dojo_event_service, dojo_id: @d1.id, name: :connpass, group_id: 9876) - create(:dojo_event_service, dojo_id: @d2.id, name: :doorkeeper, group_id: 5555) + @es1 = create(:dojo_event_service, dojo_id: @d1.id, name: :connpass, group_id: 9876) + @es2 = create(:dojo_event_service, dojo_id: @d2.id, name: :doorkeeper, group_id: 5555) end it 'プロバイダ指定なし' do @@ -26,11 +26,11 @@ end it '昨日分までは削除' do - create(:upcoming_event, dojo_id: @d1.id, service_name: 'connpass', event_id: '1111', event_at: "#{Time.zone.today - 3.days} 13:00:00".in_time_zone) - create(:upcoming_event, dojo_id: @d1.id, service_name: 'connpass', event_id: '2222', event_at: "#{Time.zone.today - 2.days} 14:00:00".in_time_zone) - create(:upcoming_event, dojo_id: @d1.id, service_name: 'connpass', event_id: '3333', event_at: "#{Time.zone.today - 1.days} 15:00:00".in_time_zone) - create(:upcoming_event, dojo_id: @d2.id, service_name: 'doorkeeper', event_id: '4444', event_at: "#{Time.zone.today - 2.days} 10:00:00".in_time_zone) - create(:upcoming_event, dojo_id: @d2.id, service_name: 'doorkeeper', event_id: '5555', event_at: "#{Time.zone.today - 1.days} 11:00:00".in_time_zone) + create(:upcoming_event, dojo_event_service_id: @es1.id, service_name: 'connpass', event_id: '1111', event_at: "#{Time.zone.today - 3.days} 13:00:00".in_time_zone) + create(:upcoming_event, dojo_event_service_id: @es1.id, service_name: 'connpass', event_id: '2222', event_at: "#{Time.zone.today - 2.days} 14:00:00".in_time_zone) + create(:upcoming_event, dojo_event_service_id: @es1.id, service_name: 'connpass', event_id: '3333', event_at: "#{Time.zone.today - 1.days} 15:00:00".in_time_zone) + create(:upcoming_event, dojo_event_service_id: @es2.id, service_name: 'doorkeeper', event_id: '4444', event_at: "#{Time.zone.today - 2.days} 10:00:00".in_time_zone) + create(:upcoming_event, dojo_event_service_id: @es2.id, service_name: 'doorkeeper', event_id: '5555', event_at: "#{Time.zone.today - 1.days} 11:00:00".in_time_zone) expect{ UpcomingEvents::Aggregation.new({}).run }.to change{ UpcomingEvent.count }.from(5).to(3) end From b7e0ec4b7f1e1ff36596226225c74d9b16ffe313 Mon Sep 17 00:00:00 2001 From: Chikako OHNO Date: Wed, 29 May 2019 23:50:28 +0900 Subject: [PATCH 5/6] =?UTF-8?q?=E3=83=AC=E3=83=93=E3=83=A5=E3=83=BC?= =?UTF-8?q?=E6=8C=87=E6=91=98=E4=BA=8B=E9=A0=85=E5=AF=BE=E5=BF=9C(2)=20?= =?UTF-8?q?=E3=83=BBschema.rb=20=E3=81=AE=E5=B7=AE=E3=81=97=E6=9B=BF?= =?UTF-8?q?=E3=81=88=20=E3=83=BB=E9=96=A2=E9=80=A3=E3=81=AB=E5=AF=BE?= =?UTF-8?q?=E3=81=97=E3=81=A6=20find=5For=5Finitialize=5Fby=20=E3=81=8C?= =?UTF-8?q?=E4=BD=BF=E3=81=88=E3=81=9F=E3=81=AE=E3=81=A7=E3=83=AA=E3=83=95?= =?UTF-8?q?=E3=82=A1=E3=82=AF=E3=82=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/schema.rb | 2 +- lib/upcoming_events/tasks/connpass.rb | 3 +-- lib/upcoming_events/tasks/doorkeeper.rb | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/db/schema.rb b/db/schema.rb index 94c32014a..258de6fbf 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -80,10 +80,10 @@ end create_table "upcoming_events", force: :cascade do |t| + t.integer "dojo_event_service_id", null: false t.string "event_id", null: false t.string "event_url", null: false t.datetime "event_at", null: false - t.integer "dojo_event_service_id", default: 1, null: false t.string "dojo_name", null: false t.string "service_name", null: false t.integer "participants", null: false diff --git a/lib/upcoming_events/tasks/connpass.rb b/lib/upcoming_events/tasks/connpass.rb index 423c07c95..581089932 100644 --- a/lib/upcoming_events/tasks/connpass.rb +++ b/lib/upcoming_events/tasks/connpass.rb @@ -13,8 +13,7 @@ def run @client.fetch_events(@params.merge(series_id: dojo_event_service.group_id)).each do |e| next unless e.dig('series', 'id').to_s == dojo_event_service.group_id - record = dojo_event_service.upcoming_events.find_by(event_id: e['event_id']) - record ||= dojo_event_service.upcoming_events.build(event_id: e['event_id']) + record = dojo_event_service.upcoming_events.find_or_initialize_by(event_id: e['event_id']) record.update!(dojo_name: dojo.name, service_name: dojo_event_service.name, event_url: e['event_url'], diff --git a/lib/upcoming_events/tasks/doorkeeper.rb b/lib/upcoming_events/tasks/doorkeeper.rb index 3ee6809b2..44cfdb703 100644 --- a/lib/upcoming_events/tasks/doorkeeper.rb +++ b/lib/upcoming_events/tasks/doorkeeper.rb @@ -13,8 +13,7 @@ def run @client.fetch_events(@params.merge(group_id: dojo_event_service.group_id)).each do |e| next unless e['group'].to_s == dojo_event_service.group_id - record = dojo_event_service.upcoming_events.find_by(event_id: e['id']) - record ||= dojo_event_service.upcoming_events.build(event_id: e['id']) + record = dojo_event_service.upcoming_events.find_or_initialize_by(event_id: e['id']) record.update!(dojo_name: dojo.name, service_name: dojo_event_service.name, event_url: e['public_url'], From b3bc0885bcc51f00d407a0345f667b64f238585e Mon Sep 17 00:00:00 2001 From: Chikako OHNO Date: Thu, 30 May 2019 23:59:37 +0900 Subject: [PATCH 6/6] =?UTF-8?q?=E3=83=AC=E3=83=93=E3=83=A5=E3=83=BC?= =?UTF-8?q?=E6=8C=87=E6=91=98=E4=BA=8B=E9=A0=85=E5=AF=BE=E5=BF=9C(3)=20?= =?UTF-8?q?=E3=83=BBUpcomingEvent=20=E3=81=8B=E3=82=89=20dojo=5Fname=20?= =?UTF-8?q?=E3=82=AB=E3=83=A9=E3=83=A0=E3=82=92=E5=89=8A=E9=99=A4=20?= =?UTF-8?q?=E3=83=BBUpcomingEvent=20=E3=81=AB=20service=5Fname=20+=20event?= =?UTF-8?q?=5Fid=20=E3=81=A7=E3=83=A6=E3=83=8B=E3=83=BC=E3=82=AF=E5=88=B6?= =?UTF-8?q?=E7=B4=84=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/upcoming_event.rb | 1 - db/migrate/20190526151359_mod_columns_to_upcoming_event.rb | 6 ++++-- db/schema.rb | 2 +- lib/upcoming_events/tasks/connpass.rb | 3 +-- lib/upcoming_events/tasks/doorkeeper.rb | 3 +-- spec/factories/upcoming_events.rb | 1 - 6 files changed, 7 insertions(+), 9 deletions(-) diff --git a/app/models/upcoming_event.rb b/app/models/upcoming_event.rb index 41c123df2..4ee58436c 100644 --- a/app/models/upcoming_event.rb +++ b/app/models/upcoming_event.rb @@ -1,7 +1,6 @@ class UpcomingEvent < ApplicationRecord belongs_to :dojo_event_service - validates :dojo_name, presence: true validates :service_name, presence: true, uniqueness: { scope: :event_id } validates :event_id, presence: true validates :event_url, presence: true diff --git a/db/migrate/20190526151359_mod_columns_to_upcoming_event.rb b/db/migrate/20190526151359_mod_columns_to_upcoming_event.rb index ca4f3d4ad..9a1c8c3d2 100644 --- a/db/migrate/20190526151359_mod_columns_to_upcoming_event.rb +++ b/db/migrate/20190526151359_mod_columns_to_upcoming_event.rb @@ -2,13 +2,15 @@ class ModColumnsToUpcomingEvent < ActiveRecord::Migration[5.1] def up remove_index :upcoming_events, :event_at - add_column :upcoming_events, :dojo_name, :string, null: false add_column :upcoming_events, :service_name, :string, null: false add_column :upcoming_events, :participants, :integer, null: false + + add_index :upcoming_events, [:service_name, :event_id], :unique => true end def down - remove_column :upcoming_events, :dojo_name, :string, null: false + remove_index :upcoming_events, [:service_name, :event_id] + remove_column :upcoming_events, :service_name, :string, null: false remove_column :upcoming_events, :participants, :integer, null: false diff --git a/db/schema.rb b/db/schema.rb index 258de6fbf..f37f9f29d 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -84,10 +84,10 @@ t.string "event_id", null: false t.string "event_url", null: false t.datetime "event_at", null: false - t.string "dojo_name", null: false t.string "service_name", null: false t.integer "participants", null: false t.index ["dojo_event_service_id"], name: "index_upcoming_events_on_dojo_event_service_id" + t.index ["service_name", "event_id"], name: "index_upcoming_events_on_service_name_and_event_id", unique: true end add_foreign_key "dojo_event_services", "dojos" diff --git a/lib/upcoming_events/tasks/connpass.rb b/lib/upcoming_events/tasks/connpass.rb index 581089932..590262519 100644 --- a/lib/upcoming_events/tasks/connpass.rb +++ b/lib/upcoming_events/tasks/connpass.rb @@ -14,8 +14,7 @@ def run next unless e.dig('series', 'id').to_s == dojo_event_service.group_id record = dojo_event_service.upcoming_events.find_or_initialize_by(event_id: e['event_id']) - record.update!(dojo_name: dojo.name, - service_name: dojo_event_service.name, + record.update!(service_name: dojo_event_service.name, event_url: e['event_url'], event_at: Time.zone.parse(e['started_at']), participants: e['accepted']) diff --git a/lib/upcoming_events/tasks/doorkeeper.rb b/lib/upcoming_events/tasks/doorkeeper.rb index 44cfdb703..f9a6a5201 100644 --- a/lib/upcoming_events/tasks/doorkeeper.rb +++ b/lib/upcoming_events/tasks/doorkeeper.rb @@ -14,8 +14,7 @@ def run next unless e['group'].to_s == dojo_event_service.group_id record = dojo_event_service.upcoming_events.find_or_initialize_by(event_id: e['id']) - record.update!(dojo_name: dojo.name, - service_name: dojo_event_service.name, + record.update!(service_name: dojo_event_service.name, event_url: e['public_url'], participants: e['participants'], event_at: Time.zone.parse(e['starts_at'])) diff --git a/spec/factories/upcoming_events.rb b/spec/factories/upcoming_events.rb index f2c0fc5d4..fdf91df3b 100644 --- a/spec/factories/upcoming_events.rb +++ b/spec/factories/upcoming_events.rb @@ -2,7 +2,6 @@ FactoryBot.define do factory :upcoming_event do - dojo_name { 'Dojo Name' } service_name { :connpass } event_id { '1234' } event_url { 'http:/www.aaa.com/events/1224' }