Merge pull request #1550 from takatama/multiple-series-ids-for-connpass-api

connpass APIの呼び出し回数を削減する

Author: yasulab

lib/event_service/client.rb

@@ -23,9 +23,6 @@ def connection_for(endpoint)
 
         f.adapter  Faraday.default_adapter
       end
-      # TODO: According to the report by users, the following code fails to aggregate data for  /events page.
-      # connpass は https://connpass.com/robots.txt を守らない場合は、アクセス制限を施すので、下記の sleep を入れるようにした https://connpass.com/about/api/
-      #sleep 5 if endpoint.include?(EventService::Providers::Connpass::ENDPOINT)
     end
   end
 end

lib/event_service/providers/connpass.rb

@@ -14,6 +14,8 @@ def search(keyword:)
 
         # NOTE: yyyymm, yyyymmdd は文字列を要素とする配列(Array[String])で指定
         def fetch_events(series_id:, yyyymm: nil, yyyymmdd: nil)
+          series_id = series_id.join(',') if series_id.is_a?(Array)
+
           params = {
             series_id: series_id,
             start: 1,
@@ -33,6 +35,8 @@ def fetch_events(series_id:, yyyymm: nil, yyyymmdd: nil)
 
           param_period_patern.each do |param_period|
             loop do
+              # connpass は https://connpass.com/robots.txt を守らない場合は、アクセス制限を施すので、下記の sleep を入れるようにした https://connpass.com/about/api/
+              sleep 5
               part = @client.get('event/', params.merge(param_period))
 
               break if part['results_returned'].zero?

lib/statistics/tasks/connpass.rb

@@ -14,24 +14,25 @@ def initialize(dojos, period)
       end
 
       def run
-        @dojos.each do |dojo|
-          dojo.dojo_event_services.for(:connpass).each do |dojo_event_service|
-            @client.fetch_events(**@params.merge(series_id: dojo_event_service.group_id)).each do |e|
-              next unless e.dig('series', 'id').to_s == dojo_event_service.group_id
-        
-              EventHistory.create!(dojo_id: dojo.id,
-                                   dojo_name: dojo.name,
-                                   service_name: dojo_event_service.name,
-                                   service_group_id: dojo_event_service.group_id,
-                                   event_id: e['event_id'],
-                                   event_url: e['event_url'],
-                                   participants: e['accepted'],
-                                   evented_at: Time.zone.parse(e['started_at']))
-            end
-          end
+        group_ids = @dojos.flat_map do |dojo|
+          dojo.dojo_event_services.for(:connpass).pluck(:group_id)
+        end
+      
+        @client.fetch_events(**@params.merge(series_id: group_ids)).each do |e|
+          dojo_event_service = DojoEventService.find_by(group_id: e.dig('series', 'id').to_s)
+          next unless dojo_event_service
+      
+          EventHistory.create!(dojo_id: dojo_event_service.dojo_id,
+                               dojo_name: dojo_event_service.dojo.name,
+                               service_name: dojo_event_service.name,
+                               service_group_id: dojo_event_service.group_id,
+                               event_id: e['event_id'],
+                               event_url: e['event_url'],
+                               participants: e['accepted'],
+                               evented_at: Time.zone.parse(e['started_at']))
         end
       end
-
+      
       private
 
       def build_params(period)

lib/upcoming_events/tasks/connpass.rb

@@ -8,24 +8,25 @@ def initialize(dojos, period)
       end
 
       def run
-        @dojos.each do |dojo|
-          dojo.dojo_event_services.for(:connpass).each do |dojo_event_service|
-            @client.fetch_events(**@params.merge(series_id: dojo_event_service.group_id)).each do |e|
-              next unless e.dig('series', 'id').to_s == dojo_event_service.group_id
-
-              record = dojo_event_service.upcoming_events.find_or_initialize_by(event_id: e['event_id'])
-              record.update!(service_name: dojo_event_service.name,
-                             event_title:  e['title'],
-                             event_url:    e['event_url'],
-                             event_at:     Time.zone.parse(e['started_at']),
-                             event_end_at: Time.zone.parse(e['ended_at']),
-                             participants: e['accepted'],
-                             event_update_at: Time.zone.parse(e['updated_at']),
-                             address:      e['address'],
-                             place:        e['place'],
-                             limit:        e['limit'])
-            end
-          end
+        group_ids = @dojos.flat_map do |dojo|
+          dojo.dojo_event_services.for(:connpass).pluck(:group_id)
+        end
+      
+        @client.fetch_events(**@params.merge(series_id: group_ids)).each do |e|
+          dojo_event_service = DojoEventService.find_by(group_id: e.dig('series', 'id').to_s)
+          next unless dojo_event_service
+      
+          record = dojo_event_service.upcoming_events.find_or_initialize_by(event_id: e['event_id'])
+          record.update!(service_name: dojo_event_service.name,
+                         event_title:  e['title'],
+                         event_url:    e['event_url'],
+                         event_at:     Time.zone.parse(e['started_at']),
+                         event_end_at: Time.zone.parse(e['ended_at']),
+                         participants: e['accepted'],
+                         event_update_at: Time.zone.parse(e['updated_at']),
+                         address:      e['address'],
+                         place:        e['place'],
+                         limit:        e['limit'])
         end
       end
 

spec/lib/event_service/providers/connpass_spec.rb

@@ -18,14 +18,31 @@
   end
 
   describe '#fetch_events' do
-    subject { described_class.new.fetch_events(series_id: 9876) }
+    context 'when a single series_id is given' do
+      subject { described_class.new.fetch_events(series_id: 9876) }
 
-    it do
-      expect(subject).to be_instance_of(Array)
-      expect(subject.size).to eq 1
-      expect(subject.first['event_id']).to eq 12345
-      expect(subject.first['series']['url']).to eq 'https://coderdojo-okutama.connpass.com/'
-      expect(subject.first['series']['id']).to eq 9876
+      it do
+        expect(subject).to be_instance_of(Array)
+        expect(subject.size).to eq 1
+        expect(subject.first['event_id']).to eq 12345
+        expect(subject.first['series']['url']).to eq 'https://coderdojo-okutama.connpass.com/'
+        expect(subject.first['series']['id']).to eq 9876
+      end
+    end
+
+    context 'when multiple series_ids are given' do
+      subject { described_class.new.fetch_events(series_id: [9876, 9877]) }
+
+      it do
+        expect(subject).to be_instance_of(Array)
+        expect(subject.size).to eq 2
+        expect(subject.first['event_id']).to eq 12345
+        expect(subject.first['series']['url']).to eq 'https://coderdojo-okutama.connpass.com/'
+        expect(subject.first['series']['id']).to eq 9876
+        expect(subject.second['event_id']).to eq 12346 # assuming the second event has id 12346
+        expect(subject.second['series']['url']).to eq 'https://coderdojo-okutama2.connpass.com/'
+        expect(subject.second['series']['id']).to eq 9877
+      end
     end
   end
 end

spec/support/shared_contexts/statistics.rb

@@ -4,7 +4,13 @@
       f.response :json, :content_type => /\bjson$/
       f.adapter :test, Faraday::Adapter::Test::Stubs.new do |stub|
         # connpass
-        stub.get('/event/') { connpass_response }
+        stub.get('/event/') do |env|
+          if env.params["series_id"] == '9876,9877'
+            multiple_series_ids_response
+          else
+            connpass_response
+          end
+        end
 
         # doorkeeper
         stub.get('/events') { doorkeeper_response }
@@ -21,6 +27,16 @@
 RSpec.shared_context 'Use stubs for Connpass' do
   include_context 'Use stub connection of Faraday'
 
+  # response for multiple series_ids 9876 and 9877
+  let(:multiple_series_ids_response) do
+    [
+      200,
+      { 'Content-Type' => 'application/json' },
+      '{"results_returned": 2, "events": [{"event_url": "https://coderdojo-okutama.connpass.com/event/12345/", "event_type": "participation", "owner_nickname": "nalabjp", "series": {"url": "https://coderdojo-okutama.connpass.com/", "id": 9876, "title": "CoderDojo series"}, "updated_at": "2017-04-29T14:59:30+09:00", "lat": "35.801763000000", "started_at": "2017-05-07T10:00:00+09:00", "hash_tag": "CoderDojo", "title": "CoderDojo title", "event_id": 12345, "lon": "139.087656000000", "waiting": 2, "limit": 10, "owner_id": 2525, "owner_display_name": "nalabjp", "description": "CoderDojo description", "address": "Okutama-cho Tokyo", "catch": "CoderDojo catch", "accepted": 10, "ended_at": "2017-05-07T12:00:00+09:00", "place": "Tokyo"},{"event_url": "https://coderdojo-okutama.connpass.com/event/12346/", "event_type": "participation", "owner_nickname": "nalabjp", "series": {"url": "https://coderdojo-okutama2.connpass.com/", "id": 9877, "title": "CoderDojo series"}, "updated_at": "2017-04-29T14:59:30+09:00", "lat": "35.801763000000", "started_at": "2017-05-07T10:00:00+09:00", "hash_tag": "CoderDojo", "title": "CoderDojo title", "event_id": 12346, "lon": "139.087656000000", "waiting": 2, "limit": 10, "owner_id": 2525, "owner_display_name": "nalabjp", "description": "CoderDojo description", "address": "Okutama-cho Tokyo", "catch": "CoderDojo catch", "accepted": 10, "ended_at": "2017-05-07T12:00:00+09:00", "place": "Tokyo"}], "results_start": 200, "results_available": 518}'
+    ]
+  end
+
+  # response for single series_id 9876, and for search
   let(:connpass_response) do
     [
       200,