Merge pull request #1699 from coderdojo-japan/update-connpass-api-from-v1-to-v2

Upgrade `connpass` API: v1 -> v2

Author: yasulab

Gemfile

@@ -39,6 +39,7 @@ gem 'kramdown-parser-gfm'
 gem 'faraday'
 gem 'koala'
 gem 'lazy_high_charts', '1.5.8'
+gem 'connpass_api_v2' # https://github.com/sue445/connpass_api_v2-ruby
 
 # Protect from attacks for Security
 gem 'rack-attack'

Gemfile.lock

@@ -123,6 +123,9 @@ GEM
       logger (~> 1.5)
     concurrent-ruby (1.3.5)
     connection_pool (2.5.3)
+    connpass_api_v2 (0.1.0)
+      faraday (>= 2.0.0)
+      faraday-mashify
     crass (1.0.6)
     csv (3.3.4)
     date (3.4.1)
@@ -144,6 +147,9 @@ GEM
       faraday-net_http (>= 2.0, < 3.5)
       json
       logger
+    faraday-mashify (1.0.0)
+      faraday (~> 2.0)
+      hashie
     faraday-multipart (1.1.0)
       multipart-post (~> 2.0)
     faraday-net_http (3.4.0)
@@ -165,6 +171,7 @@ GEM
     globalid (1.2.1)
       activesupport (>= 6.1)
     hash-deep-merge (0.1.1)
+    hashie (5.0.0)
     i18n (1.14.7)
       concurrent-ruby (~> 1.0)
     io-console (0.8.0)
@@ -483,6 +490,7 @@ DEPENDENCIES
   bootsnap
   bootstrap-sass
   capybara
+  connpass_api_v2
   csv
   dotenv-rails
   factory_bot_rails

bin/c-search

@@ -1,22 +1,42 @@
-#!/bin/sh
-ENVRC_FILE=".envrc"
+#!/usr/bin/env ruby
 
-if [ ! -f "$ENVRC_FILE" ]; then
-    echo "$ENVRC_FILE が見つかりません"
-    exit 1
-fi
+require 'connpass_api_v2'
+require 'uri'
 
-source "$ENVRC_FILE"
+if ENV['CONNPASS_API_KEY'].nil?
+  puts('CONNPASS_API_KEY が設定されていません')
+  exit(1)
+end
 
-if [ -z "$FIXIE_URL" ]; then
-    echo "FIXIE_URL が設定されていません"
-    exit 1
-fi
+if ARGV.empty?
+  puts('Usage: c-search [CONNPASS_EVENT_URL | CONNPASS_EVENT_ID]')
+  exit(1)
+end
 
-if [ $# -eq 0 ]
-then
-    echo "Usage: c-search [CONNPASS_EVENT_URL | CONNPASS_EVENT_ID]"
+input = ARGV[0]
+event_id = nil
+if input =~ /^https?:\/\//
+  # URLからイベントIDを抽出
+  event_id = URI(input).path[%r{event/(\d+)}, 1]
 else
-    id=$(echo $1 | sed -e 's/[^0-9]//g')
-    curl -x "$FIXIE_URL" -sL "https://connpass.com/api/v1/event/?event_id=${id}" | jq ".events[].series.id"
-fi
+  event_id = input.gsub(/\D/, '')
+end
+
+unless event_id && !event_id.empty?
+  puts "イベントIDが特定できませんでした: #{input}"
+  exit 1
+end
+
+client = ConnpassApiV2.client(ENV['CONNPASS_API_KEY'])
+result = client.get_events(event_id: event_id)
+
+if result.results_returned > 0
+  event = result.events.first
+  puts event.fetch('id')
+  #puts "id:         #{event.fetch('id')}"
+  #puts "title:      #{event.fetch('title')}"
+  #puts "group_id:   #{event.fetch('group').fetch('id')}"
+  #puts "group_name: #{event.fetch('group').fetch('title')}"
+else
+  puts "イベントが見つかりませんでした (event_id: #{event_id})"
+end

bin/setup

@@ -34,7 +34,7 @@ FileUtils.chdir APP_ROOT do
   #today = Time.now.in_time_zone('Tokyo').to_date
   #from  = (today - 90).strftime('%Y%m')
   #to    = today.prev_month.strftime('%Y%m')
-  #if ENV['DOORKEEPER_API_TOKEN'] && ENV['FIXIE_URL']
+  #if ENV['DOORKEEPER_API_TOKEN'] && ENV['CONNPASS_API_KEY']
   #  system! "bin/rails statistics:aggregation[#{from},#{to}]"
   #  system! 'bin/rails upcoming_events:aggregation'
   #elsif ENV['DOORKEEPER_API_TOKEN']
@@ -43,7 +43,7 @@ FileUtils.chdir APP_ROOT do
   #  system! 'bin/rails upcoming_events:aggregation[connpass]'
   #else
   #  puts <<~MESSAGE
-  #    環境変数 DOORKEEPER_API_TOKEN と FIXIE_URL が設定されていないため、
+  #    環境変数 DOORKEEPER_API_TOKEN と CONNPASS_API_KEY が設定されていないため、
   #    Doorkeeper API や connpass API を使ったイベント情報の取得をスキップします。
   #
   #    なお API 経由でイベント情報を取得しなくても、rails server は実行可能です。

lib/event_service/client.rb

@@ -16,10 +16,10 @@ def get(path, params)
     def connection_for(endpoint, proxy)
       Faraday.new(endpoint, proxy: proxy) do |f|
         f.response :logger if self.class.debug
-        
+
         # faraday標準のJSONパーサーを使用
         f.response :json, parser_options: { symbolize_names: true }
-        
+
         # faraday標準のエラーハンドリングを使用
         f.response :raise_error, include_request: true
 

lib/event_service/providers/connpass.rb

@@ -1,25 +1,27 @@
+require 'connpass_api_v2'
+
 module EventService
   module Providers
       class Connpass
-        ENDPOINT = 'https://connpass.com/api/v1'.freeze
-        # NOTE: 期間は ym or ymd パラメータで指定(複数指定可能)、未指定時全期間が対象
-
         def initialize
-          @client = EventService::Client.new(ENDPOINT, proxy: ENV['FIXIE_URL'])
+          @client = ConnpassApiV2.client(ENV['CONNPASS_API_KEY'])
         end
 
         def search(keyword:)
-          @client.get('event/', { keyword: keyword, count: 100 })
+          @client.get_events(keyword: keyword, count: 100)
         end
 
         # NOTE: yyyymm, yyyymmdd は文字列を要素とする配列(Array[String])で指定
-        def fetch_events(series_id:, yyyymm: nil, yyyymmdd: nil)
-          series_id = series_id.join(',') if series_id.is_a?(Array)
+        def fetch_events(group_id:, yyyymm: nil, yyyymmdd: nil)
+          group_id = group_id.join(',') if group_id.is_a?(Array)
 
+          # API v1 -> v2 でパラメータ名が変更された
+          # https://connpass.com/about/api/v2/
+          # e.g. series_id -> group_id
           params = {
-            series_id: series_id,
-            start: 1,
-            count: 100
+            group_id: group_id,
+            start: 1,   # offset → start
+            count: 100  # limit → count
           }
 
           param_period_patern = []
@@ -35,17 +37,22 @@ def fetch_events(series_id:, yyyymm: nil, yyyymmdd: nil)
 
           param_period_patern.each do |param_period|
             loop do
-              # connpass は https://connpass.com/robots.txt を守らない場合は、アクセス制限を施すので、下記の sleep を入れるようにした https://connpass.com/about/api/
-              sleep 5
-              part = @client.get('event/', params.merge(param_period))
+              begin
+                args = params.merge(param_period).compact
+                res = @client.get_events(**args)
+              rescue ConnpassApiV2::Error => e
+                sleep 5 && retry if e.response&.status == 403
+
+                raise e
+              end
 
-              break if part['results_returned'].zero?
+              break if res.results_returned.zero?
 
-              events.push(*part.fetch('events'))
+              events.push(*res.events)
 
-              break if part.size < params[:count]
+              break if res.events.size < params[:count]
 
-              break if params[:start] + params[:count] > part['results_available']
+              break if params[:start] + params[:count] > res.results_available
 
               params[:start] += params[:count]
             end

lib/event_service/providers/doorkeeper.rb

@@ -6,7 +6,7 @@ class Doorkeeper
 
         def initialize
           @client = EventService::Client.new(ENDPOINT) do |c|
-            c.authorization(:Bearer, ENV.fetch('DOORKEEPER_API_TOKEN'))
+            c.request :authorization, 'Bearer', ENV.fetch('DOORKEEPER_API_TOKEN')
           end
           @default_since = '2010-07-01'.to_date.beginning_of_day
           @default_until = Time.zone.yesterday.end_of_day

lib/statistics/tasks/connpass.rb

@@ -20,18 +20,19 @@ def run
           dojo.dojo_event_services.for(:connpass).pluck(:group_id)
         end
 
-        @client.fetch_events(**@params.merge(series_id: group_ids)).each do |e|
+        @client.fetch_events(**@params.merge(group_id: group_ids)).each do |e|
           dojo_event_service = DojoEventService.find_by(group_id: e.dig('series', 'id').to_s)
           next unless dojo_event_service
 
           EventHistory.create!(dojo_id:          dojo_event_service.dojo_id,
                                dojo_name:        dojo_event_service.dojo.name,
                                service_name:     dojo_event_service.name,
                                service_group_id: dojo_event_service.group_id,
-                               event_id:     e['event_id'],
-                               event_url:    e['event_url'],
-                               participants: e['accepted'],
-                               evented_at:   Time.zone.parse(e['started_at']))
+                               event_id:         e.fetch('id'),
+                               event_url:        e.fetch('event_url'),
+                               participants:     e.fetch('accepted'),
+                               evented_at:       Time.zone.parse(e.fetch('started_at'))
+            )
         end
       end
 

lib/statistics/tasks/doorkeeper.rb

@@ -18,7 +18,8 @@ def initialize(dojos, period)
       def run
         @dojos.each do |dojo|
           dojo.dojo_event_services.for(:doorkeeper).each do |dojo_event_service|
-            @client.fetch_events(**@params.merge(group_id: dojo_event_service.group_id)).each do |e|
+            events = @client.fetch_events(**@params.merge(group_id: dojo_event_service.group_id))
+            (events || []).compact.each do |e|
               next unless e['group'].to_s == dojo_event_service.group_id
 
               EventHistory.create!(dojo_id:          dojo.id,

lib/upcoming_events/tasks/connpass.rb

@@ -11,22 +11,25 @@ def run
         group_ids = @dojos.flat_map do |dojo|
           dojo.dojo_event_services.for(:connpass).pluck(:group_id)
         end
-      
-        @client.fetch_events(**@params.merge(series_id: group_ids)).each do |e|
-          dojo_event_service = DojoEventService.find_by(group_id: e.dig('series', 'id').to_s)
+
+        events = @client.fetch_events(**@params.merge(group_id: group_ids))
+        puts "[connpass] Fetched events: #{events.size}"
+        events.each do |e|
+          puts "[connpass] event_id: #{e.fetch('id')}, title: #{e.fetch('title')}"
+          dojo_event_service = DojoEventService.find_by(group_id: e.dig('group', 'id').to_s)
           next unless dojo_event_service
-      
-          record = dojo_event_service.upcoming_events.find_or_initialize_by(event_id: e['event_id'])
+
+          record = dojo_event_service.upcoming_events.find_or_initialize_by(event_id: e.fetch('id'))
           record.update!(service_name: dojo_event_service.name,
-                         event_title:  e['title'],
-                         event_url:    e['event_url'],
-                         event_at:     Time.zone.parse(e['started_at']),
-                         event_end_at: Time.zone.parse(e['ended_at']),
-                         participants: e['accepted'],
-                         event_update_at: Time.zone.parse(e['updated_at']),
-                         address:      e['address'],
-                         place:        e['place'],
-                         limit:        e['limit'])
+                         event_title:  e.fetch('title'),
+                         event_url:    e.fetch('url'),
+                         event_at:     Time.zone.parse(e.fetch('started_at')),
+                         event_end_at: Time.zone.parse(e.fetch('ended_at')),
+                         participants: e.fetch('accepted'),
+                         event_update_at: Time.zone.parse(e.fetch('updated_at')),
+                         address:      e.fetch('address'),
+                         place:        e.fetch('place'),
+                         limit:        e.fetch('limit'))
         end
       end
 

lib/upcoming_events/tasks/doorkeeper.rb

@@ -10,20 +10,23 @@ def initialize(dojos, period)
       def run
         @dojos.each do |dojo|
           dojo.dojo_event_services.for(:doorkeeper).each do |dojo_event_service|
-            @client.fetch_events(**@params.merge(group_id: dojo_event_service.group_id)).each do |e|
-              next unless e['group'].to_s == dojo_event_service.group_id
+            events = @client.fetch_events(**@params.merge(group_id: dojo_event_service.group_id))
+            puts "[Doorkeeper] dojo_id: #{dojo.id}, group_id: #{dojo_event_service.group_id}, fetched events: #{events&.size || 0}"
+            (events || []).compact.each do |e|
+              puts "[Doorkeeper] event_id: #{e.fetch('id')}, title: #{e.fetch('title')}"
+              next unless e.fetch('group').to_s == dojo_event_service.group_id
 
-              record = dojo_event_service.upcoming_events.find_or_initialize_by(event_id: e['id'])
+              record = dojo_event_service.upcoming_events.find_or_initialize_by(event_id: e.fetch('id'))
               record.update!(service_name: dojo_event_service.name,
-                             event_title:  e['title'],
-                             event_url:    e['public_url'],
-                             participants: e['participants'],
-                             event_at:     Time.zone.parse(e['starts_at']),
-                             event_end_at: Time.zone.parse(e['ends_at']),
-                             event_update_at: Time.zone.parse(e['updated_at']),
-                             address:      e['address'],
-                             place:        e['venue_name'],
-                             limit:        e['ticket_limit'])
+                             event_title:  e.fetch('title'),
+                             event_url:    e.fetch('public_url'),
+                             participants: e.fetch('participants'),
+                             event_at:     Time.zone.parse(e.fetch('starts_at')),
+                             event_end_at: Time.zone.parse(e.fetch('ends_at')),
+                             event_update_at: Time.zone.parse(e.fetch('updated_at')),
+                             address:      e.fetch('address'),
+                             place:        e.fetch('venue_name'),
+                             limit:        e.fetch('ticket_limit'))
             end
           end
         end

spec/lib/event_service/providers/connpass_spec.rb

@@ -8,40 +8,28 @@
     subject { described_class.new.search(keyword: 'coderdojo') }
 
     it do
-      expect(subject).to be_instance_of(Hash)
-      expect(subject['results_returned']).to eq 1
-      expect(subject['events'].size).to eq 1
-      expect(subject['events'].first['event_id']).to eq 12345
-      expect(subject['events'].first['series']['url']).to eq 'https://coderdojo-okutama.connpass.com/'
-      expect(subject['events'].first['series']['id']).to eq 9876
+      expect(subject).to be_a(ConnpassApiV2::Response)
+      expect(subject.results_returned).to eq 100
+      expect(subject.events).to be_a(Array)
     end
   end
 
   describe '#fetch_events' do
-    context 'when a single series_id is given' do
-      subject { described_class.new.fetch_events(series_id: 9876) }
+    context 'when a single group_id is given' do
+      subject { described_class.new.fetch_events(group_id: 9876) }
 
       it do
-        expect(subject).to be_instance_of(Array)
-        expect(subject.size).to eq 1
-        expect(subject.first['event_id']).to eq 12345
-        expect(subject.first['series']['url']).to eq 'https://coderdojo-okutama.connpass.com/'
-        expect(subject.first['series']['id']).to eq 9876
+        expect(subject).to be_a(Array)
+        expect(subject.size).to eq 4
       end
     end
 
-    context 'when multiple series_ids are given' do
-      subject { described_class.new.fetch_events(series_id: [9876, 9877]) }
+    context 'when multiple group_ids are given' do
+      subject { described_class.new.fetch_events(group_id: [9876, 9877]) }
 
       it do
-        expect(subject).to be_instance_of(Array)
-        expect(subject.size).to eq 2
-        expect(subject.first['event_id']).to eq 12345
-        expect(subject.first['series']['url']).to eq 'https://coderdojo-okutama.connpass.com/'
-        expect(subject.first['series']['id']).to eq 9876
-        expect(subject.second['event_id']).to eq 12346 # assuming the second event has id 12346
-        expect(subject.second['series']['url']).to eq 'https://coderdojo-okutama2.connpass.com/'
-        expect(subject.second['series']['id']).to eq 9877
+        expect(subject).to be_a(Array)
+        expect(subject.size).to eq 5
       end
     end
   end

spec/lib/statistics/aggregation_spec.rb

@@ -32,7 +32,7 @@
     subject { Statistics::Aggregation.new(from: Time.zone.today.prev_month.strftime('%Y%m')).run }
 
     it do
-      expect{ subject }.to change{ EventHistory.count }.from(0).to(3)
+      expect{ subject }.to change{ EventHistory.count }.from(0).to(2)
     end
   end
 

spec/rails_helper.rb

@@ -1,5 +1,6 @@
 # This file is copied to spec/ when you run 'rails generate rspec:install'
 ENV['RAILS_ENV'] ||= 'test'
+ENV['CONNPASS_API_KEY'] = 'DUMMY_CONNPASS_API_KEY'
 require File.expand_path('../../config/environment', __FILE__)
 # Prevent database truncation if the environment is production
 abort("The Rails environment is running in production mode!") if Rails.env.production?