diff --git a/lib/event_service/client.rb b/lib/event_service/client.rb index 4f5898013..98260be0f 100644 --- a/lib/event_service/client.rb +++ b/lib/event_service/client.rb @@ -23,9 +23,6 @@ def connection_for(endpoint) f.adapter Faraday.default_adapter end - # TODO: According to the report by users, the following code fails to aggregate data for /events page. - # connpass は https://connpass.com/robots.txt を守らない場合は、アクセス制限を施すので、下記の sleep を入れるようにした https://connpass.com/about/api/ - #sleep 5 if endpoint.include?(EventService::Providers::Connpass::ENDPOINT) end end end diff --git a/lib/event_service/providers/connpass.rb b/lib/event_service/providers/connpass.rb index fe12673cf..0f468357a 100644 --- a/lib/event_service/providers/connpass.rb +++ b/lib/event_service/providers/connpass.rb @@ -14,6 +14,8 @@ def search(keyword:) # NOTE: yyyymm, yyyymmdd は文字列を要素とする配列(Array[String])で指定 def fetch_events(series_id:, yyyymm: nil, yyyymmdd: nil) + series_id = series_id.join(',') if series_id.is_a?(Array) + params = { series_id: series_id, start: 1, @@ -33,6 +35,8 @@ def fetch_events(series_id:, yyyymm: nil, yyyymmdd: nil) param_period_patern.each do |param_period| loop do + # connpass は https://connpass.com/robots.txt を守らない場合は、アクセス制限を施すので、下記の sleep を入れるようにした https://connpass.com/about/api/ + sleep 5 part = @client.get('event/', params.merge(param_period)) break if part['results_returned'].zero? diff --git a/lib/statistics/tasks/connpass.rb b/lib/statistics/tasks/connpass.rb index becb7b74b..020c06fb5 100644 --- a/lib/statistics/tasks/connpass.rb +++ b/lib/statistics/tasks/connpass.rb @@ -14,24 +14,25 @@ def initialize(dojos, period) end def run - @dojos.each do |dojo| - dojo.dojo_event_services.for(:connpass).each do |dojo_event_service| - @client.fetch_events(**@params.merge(series_id: dojo_event_service.group_id)).each do |e| - next unless e.dig('series', 'id').to_s == dojo_event_service.group_id - - EventHistory.create!(dojo_id: dojo.id, - dojo_name: dojo.name, - service_name: dojo_event_service.name, - service_group_id: dojo_event_service.group_id, - event_id: e['event_id'], - event_url: e['event_url'], - participants: e['accepted'], - evented_at: Time.zone.parse(e['started_at'])) - end - end + group_ids = @dojos.flat_map do |dojo| + dojo.dojo_event_services.for(:connpass).pluck(:group_id) + end + + @client.fetch_events(**@params.merge(series_id: group_ids)).each do |e| + dojo_event_service = DojoEventService.find_by(group_id: e.dig('series', 'id').to_s) + next unless dojo_event_service + + EventHistory.create!(dojo_id: dojo_event_service.dojo_id, + dojo_name: dojo_event_service.dojo.name, + service_name: dojo_event_service.name, + service_group_id: dojo_event_service.group_id, + event_id: e['event_id'], + event_url: e['event_url'], + participants: e['accepted'], + evented_at: Time.zone.parse(e['started_at'])) end end - + private def build_params(period) diff --git a/lib/upcoming_events/tasks/connpass.rb b/lib/upcoming_events/tasks/connpass.rb index 1d8da4a83..83704dcac 100644 --- a/lib/upcoming_events/tasks/connpass.rb +++ b/lib/upcoming_events/tasks/connpass.rb @@ -8,24 +8,25 @@ def initialize(dojos, period) end def run - @dojos.each do |dojo| - dojo.dojo_event_services.for(:connpass).each do |dojo_event_service| - @client.fetch_events(**@params.merge(series_id: dojo_event_service.group_id)).each do |e| - next unless e.dig('series', 'id').to_s == dojo_event_service.group_id - - record = dojo_event_service.upcoming_events.find_or_initialize_by(event_id: e['event_id']) - record.update!(service_name: dojo_event_service.name, - event_title: e['title'], - event_url: e['event_url'], - event_at: Time.zone.parse(e['started_at']), - event_end_at: Time.zone.parse(e['ended_at']), - participants: e['accepted'], - event_update_at: Time.zone.parse(e['updated_at']), - address: e['address'], - place: e['place'], - limit: e['limit']) - end - end + group_ids = @dojos.flat_map do |dojo| + dojo.dojo_event_services.for(:connpass).pluck(:group_id) + end + + @client.fetch_events(**@params.merge(series_id: group_ids)).each do |e| + dojo_event_service = DojoEventService.find_by(group_id: e.dig('series', 'id').to_s) + next unless dojo_event_service + + record = dojo_event_service.upcoming_events.find_or_initialize_by(event_id: e['event_id']) + record.update!(service_name: dojo_event_service.name, + event_title: e['title'], + event_url: e['event_url'], + event_at: Time.zone.parse(e['started_at']), + event_end_at: Time.zone.parse(e['ended_at']), + participants: e['accepted'], + event_update_at: Time.zone.parse(e['updated_at']), + address: e['address'], + place: e['place'], + limit: e['limit']) end end diff --git a/spec/lib/event_service/providers/connpass_spec.rb b/spec/lib/event_service/providers/connpass_spec.rb index 151547c5b..88f5072bf 100644 --- a/spec/lib/event_service/providers/connpass_spec.rb +++ b/spec/lib/event_service/providers/connpass_spec.rb @@ -18,14 +18,31 @@ end describe '#fetch_events' do - subject { described_class.new.fetch_events(series_id: 9876) } + context 'when a single series_id is given' do + subject { described_class.new.fetch_events(series_id: 9876) } - it do - expect(subject).to be_instance_of(Array) - expect(subject.size).to eq 1 - expect(subject.first['event_id']).to eq 12345 - expect(subject.first['series']['url']).to eq 'https://coderdojo-okutama.connpass.com/' - expect(subject.first['series']['id']).to eq 9876 + it do + expect(subject).to be_instance_of(Array) + expect(subject.size).to eq 1 + expect(subject.first['event_id']).to eq 12345 + expect(subject.first['series']['url']).to eq 'https://coderdojo-okutama.connpass.com/' + expect(subject.first['series']['id']).to eq 9876 + end + end + + context 'when multiple series_ids are given' do + subject { described_class.new.fetch_events(series_id: [9876, 9877]) } + + it do + expect(subject).to be_instance_of(Array) + expect(subject.size).to eq 2 + expect(subject.first['event_id']).to eq 12345 + expect(subject.first['series']['url']).to eq 'https://coderdojo-okutama.connpass.com/' + expect(subject.first['series']['id']).to eq 9876 + expect(subject.second['event_id']).to eq 12346 # assuming the second event has id 12346 + expect(subject.second['series']['url']).to eq 'https://coderdojo-okutama2.connpass.com/' + expect(subject.second['series']['id']).to eq 9877 + end end end end diff --git a/spec/support/shared_contexts/statistics.rb b/spec/support/shared_contexts/statistics.rb index b6176066a..9ea76bd7f 100644 --- a/spec/support/shared_contexts/statistics.rb +++ b/spec/support/shared_contexts/statistics.rb @@ -4,7 +4,13 @@ f.response :json, :content_type => /\bjson$/ f.adapter :test, Faraday::Adapter::Test::Stubs.new do |stub| # connpass - stub.get('/event/') { connpass_response } + stub.get('/event/') do |env| + if env.params["series_id"] == '9876,9877' + multiple_series_ids_response + else + connpass_response + end + end # doorkeeper stub.get('/events') { doorkeeper_response } @@ -21,6 +27,16 @@ RSpec.shared_context 'Use stubs for Connpass' do include_context 'Use stub connection of Faraday' + # response for multiple series_ids 9876 and 9877 + let(:multiple_series_ids_response) do + [ + 200, + { 'Content-Type' => 'application/json' }, + '{"results_returned": 2, "events": [{"event_url": "https://coderdojo-okutama.connpass.com/event/12345/", "event_type": "participation", "owner_nickname": "nalabjp", "series": {"url": "https://coderdojo-okutama.connpass.com/", "id": 9876, "title": "CoderDojo series"}, "updated_at": "2017-04-29T14:59:30+09:00", "lat": "35.801763000000", "started_at": "2017-05-07T10:00:00+09:00", "hash_tag": "CoderDojo", "title": "CoderDojo title", "event_id": 12345, "lon": "139.087656000000", "waiting": 2, "limit": 10, "owner_id": 2525, "owner_display_name": "nalabjp", "description": "CoderDojo description", "address": "Okutama-cho Tokyo", "catch": "CoderDojo catch", "accepted": 10, "ended_at": "2017-05-07T12:00:00+09:00", "place": "Tokyo"},{"event_url": "https://coderdojo-okutama.connpass.com/event/12346/", "event_type": "participation", "owner_nickname": "nalabjp", "series": {"url": "https://coderdojo-okutama2.connpass.com/", "id": 9877, "title": "CoderDojo series"}, "updated_at": "2017-04-29T14:59:30+09:00", "lat": "35.801763000000", "started_at": "2017-05-07T10:00:00+09:00", "hash_tag": "CoderDojo", "title": "CoderDojo title", "event_id": 12346, "lon": "139.087656000000", "waiting": 2, "limit": 10, "owner_id": 2525, "owner_display_name": "nalabjp", "description": "CoderDojo description", "address": "Okutama-cho Tokyo", "catch": "CoderDojo catch", "accepted": 10, "ended_at": "2017-05-07T12:00:00+09:00", "place": "Tokyo"}], "results_start": 200, "results_available": 518}' + ] + end + + # response for single series_id 9876, and for search let(:connpass_response) do [ 200,