diff --git a/Gemfile b/Gemfile index 03e30eadd..7ada2bdfb 100644 --- a/Gemfile +++ b/Gemfile @@ -31,6 +31,8 @@ gem 'faraday' # https://github.com/bundler/bundler/issues/5332 gem 'faraday_middleware', '0.10' +gem 'koala' + group :development do gem 'web-console' gem 'spring' diff --git a/Gemfile.lock b/Gemfile.lock index 61f147006..6136d19e6 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -106,6 +106,10 @@ GEM jquery-ui-rails (6.0.1) railties (>= 3.2.16) json (2.1.0) + koala (3.0.0) + addressable + faraday + json (>= 1.8) kramdown (1.15.0) launchy (2.4.3) addressable (~> 2.3) @@ -305,6 +309,7 @@ DEPENDENCIES font-awesome-rails jbuilder jquery-rails + koala kramdown listen minitest-retry diff --git a/app/models/dojo_event_service.rb b/app/models/dojo_event_service.rb index 61e07b3ca..fdc3c493c 100644 --- a/app/models/dojo_event_service.rb +++ b/app/models/dojo_event_service.rb @@ -1,4 +1,4 @@ class DojoEventService < ApplicationRecord belongs_to :dojo - enum name: %i( connpass doorkeeper ) + enum name: %i( connpass doorkeeper facebook ) end diff --git a/db/migrate/20171029065909_integer_to_string_on_group_id_in_dojo_event_services.rb b/db/migrate/20171029065909_integer_to_string_on_group_id_in_dojo_event_services.rb new file mode 100644 index 000000000..985cc6a70 --- /dev/null +++ b/db/migrate/20171029065909_integer_to_string_on_group_id_in_dojo_event_services.rb @@ -0,0 +1,9 @@ +class IntegerToStringOnGroupIdInDojoEventServices < ActiveRecord::Migration[5.0] + def up + change_column :dojo_event_services, :group_id, :string, null: false + end + + def down + change_column :dojo_event_services, :group_id, :integer, null: false + end +end diff --git a/db/migrate/20171029071514_integer_to_string_on_service_group_id_and_event_id_in_event_histories.rb b/db/migrate/20171029071514_integer_to_string_on_service_group_id_and_event_id_in_event_histories.rb new file mode 100644 index 000000000..74baf8fbf --- /dev/null +++ b/db/migrate/20171029071514_integer_to_string_on_service_group_id_and_event_id_in_event_histories.rb @@ -0,0 +1,11 @@ +class IntegerToStringOnServiceGroupIdAndEventIdInEventHistories < ActiveRecord::Migration[5.0] + def up + change_column :event_histories, :service_group_id, :string, null: false + change_column :event_histories, :event_id, :string, null: false + end + + def down + change_column :event_histories, :service_group_id, :integer, null: false + change_column :event_histories, :event_id, :integer, null: false + end +end diff --git a/db/schema.rb b/db/schema.rb index b5aa62e85..329da6e32 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -16,7 +16,7 @@ t.integer "dojo_id", null: false t.integer "name", null: false t.string "url" - t.integer "group_id", null: false + t.string "group_id", null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["dojo_id"], name: "index_dojo_event_services_on_dojo_id" @@ -39,8 +39,8 @@ t.integer "dojo_id", null: false t.string "dojo_name", null: false t.string "service_name", null: false - t.integer "service_group_id", null: false - t.integer "event_id", null: false + t.string "service_group_id", null: false + t.string "event_id", null: false t.string "event_url", null: false t.integer "participants", null: false t.datetime "evented_at", null: false diff --git a/lib/statistics/aggregation.rb b/lib/statistics/aggregation.rb index f1c1b4109..f88a2827a 100644 --- a/lib/statistics/aggregation.rb +++ b/lib/statistics/aggregation.rb @@ -4,9 +4,11 @@ class << self def run(date:) cnps_dojos = Dojo.joins(:dojo_event_service).where(dojo_event_services: { name: :connpass }).to_a drkp_dojos = Dojo.joins(:dojo_event_service).where(dojo_event_services: { name: :doorkeeper }).to_a + fsbk_dojos = Dojo.joins(:dojo_event_service).where(dojo_event_services: { name: :facebook }).to_a Connpass.run(cnps_dojos, date) Doorkeeper.run(drkp_dojos, date) + Facebook.run(fsbk_dojos, date) end end @@ -20,7 +22,7 @@ def run(dojos, date) dojos.each do |dojo| cnps.fetch_events(params.merge(series_id: dojo.dojo_event_service.group_id)).each do |e| - next unless e.dig('series', 'id') == dojo.dojo_event_service.group_id + next unless e.dig('series', 'id').to_s == dojo.dojo_event_service.group_id EventHistory.create!(dojo_id: dojo.id, dojo_name: dojo.name, @@ -47,7 +49,7 @@ def run(dojos, date) dojos.each do |dojo| drkp.fetch_events(params.merge(group_id: dojo.dojo_event_service.group_id)).each do |e| - next unless e['group'] == dojo.dojo_event_service.group_id + next unless e['group'].to_s == dojo.dojo_event_service.group_id EventHistory.create!(dojo_id: dojo.id, dojo_name: dojo.name, @@ -62,5 +64,32 @@ def run(dojos, date) end end end + + class Facebook + class << self + def run(dojos, date) + fsbk = Client::Facebook.new + params = { + since_at: date.beginning_of_month, + until_at: date.end_of_month + } + + dojos.each do |dojo| + fsbk.fetch_events(params.merge(group_id: dojo.dojo_event_service.group_id)).each do |e| + next unless e.dig('owner', 'id') == dojo.dojo_event_service.group_id + + EventHistory.create!(dojo_id: dojo.id, + dojo_name: dojo.name, + service_name: dojo.dojo_event_service.name, + service_group_id: dojo.dojo_event_service.group_id, + event_id: e['id'], + event_url: "https://www.facebook.com/events/#{e['id']}", + participants: e['attending_count'], + evented_at: Time.zone.parse(e['start_time'])) + end + end + end + end + end end end diff --git a/lib/statistics/client.rb b/lib/statistics/client.rb index d5200a90a..817e03611 100644 --- a/lib/statistics/client.rb +++ b/lib/statistics/client.rb @@ -1,7 +1,5 @@ module Statistics class Client - class APIRateLimitError < ::StandardError; end - class_attribute :debug self.debug = false @@ -81,32 +79,62 @@ def search(keyword:) end def fetch_events(group_id:, since_at: @default_since, until_at: @default_until) - params = { - page: 1, - since: since_at, - until: until_at - } - events = [] + begin + params = { + page: 1, + since: since_at, + until: until_at + } + events = [] - loop do - part = @client.get("groups/#{group_id}/events", params) + loop do + part = @client.get("groups/#{group_id}/events", params) + + break if part.size.zero? + + events.push(*part.map { |e| e['event'] }) - break if part.size.zero? + break if part.size < 25 # 25 items / 1 request - events.push(*part.map { |e| e['event'] }) + params[:page] += 1 + end - break if part.size < 25 # 25 items / 1 request + events + rescue Faraday::ClientError => e + raise e unless e.response[:status] == 429 - params[:page] += 1 + puts 'API rate limit exceeded.' + puts "This task will retry in 60 seconds from now(#{Time.zone.now})." + sleep 60 + retry end + end + end - events - rescue Faraday::ClientError => e - if e.response[:status] == 429 - raise Client::APIRateLimitError - else - raise e + class Facebook + class_attribute :access_token + + def initialize + @client = Koala::Facebook::API.new(self.access_token) + end + + def fetch_events(group_id:, since_at: nil, until_at: nil) + params = { + fields: %i(attending_count start_time owner), + limit: 100, + since: since_at, + until: until_at + }.compact + + events = [] + + collection = @client.get_object("#{group_id}/events", params) + events.push(*collection.to_a) + while !collection.empty? && collection.paging['next'] do + events.push(*collection.next_page.to_a) end + + events end end end diff --git a/lib/tasks/oauth.rake b/lib/tasks/oauth.rake new file mode 100644 index 000000000..bd731c2d9 --- /dev/null +++ b/lib/tasks/oauth.rake @@ -0,0 +1,6 @@ +namespace :oauth do + desc 'Facebookのaccess tokenを取得します' + task :facebook_access_token, [:app_id, :app_secret] => :environment do |_tasks, args| + puts 'Access Token: ' + Koala::Facebook::OAuth.new(args[:app_id], args[:app_secret]).get_app_access_token + end +end diff --git a/lib/tasks/statistics.rake b/lib/tasks/statistics.rake index 6b9b115fa..25684448e 100644 --- a/lib/tasks/statistics.rake +++ b/lib/tasks/statistics.rake @@ -16,6 +16,10 @@ namespace :statistics do d } + notify_idobata = -> (msg) { + puts `curl --data-urlencode "source=#{msg}" -s #{ENV['IDOBATA_HOOK_URL']} -o /dev/null -w "idobata: %{http_code}"` if ENV.key?('IDOBATA_HOOK_URL') + } + from = if args[:from] if args[:from].length == 4 date_from_str.call(args[:from]).beginning_of_year @@ -35,23 +39,24 @@ namespace :statistics do Time.current.prev_month.end_of_month end + Statistics::Client::Facebook.access_token = Koala::Facebook::OAuth.new(ENV['FACEBOOK_APP_ID'], ENV['FACEBOOK_APP_SECRET']).get_app_access_token + EventHistory.where(evented_at: from..to).delete_all - loop.with_object([from]) { |_, list| - nm = list.last.next_month - raise StopIteration if nm > to - list << nm - }.each { |date| - begin + begin + loop.with_object([from]) { |_, list| + nm = list.last.next_month + raise StopIteration if nm > to + list << nm + }.each { |date| puts "Aggregate for #{date.strftime('%Y/%m')}" Statistics::Aggregation.run(date: date) - rescue Statistics::Client::APIRateLimitError - puts 'API rate limit exceeded.' - puts "This task will retry in 60 seconds from now(#{Time.zone.now})." - sleep 60 - retry - end - } + } + + notify_idobata.call("#{from.strftime('%Y/%m')}~#{to.strftime('%Y/%m')}のイベント履歴の集計を行いました") + rescue + notify_idobata.call("#{from.strftime('%Y/%m')}~#{to.strftime('%Y/%m')}のイベント履歴の集計でエラーが発生しました") + end end desc 'キーワードからイベント情報を検索します' diff --git a/spec/lib/statistics/aggregation_spec.rb b/spec/lib/statistics/aggregation_spec.rb index 92fa272e1..92150c43b 100644 --- a/spec/lib/statistics/aggregation_spec.rb +++ b/spec/lib/statistics/aggregation_spec.rb @@ -2,7 +2,9 @@ require 'statistics' RSpec.describe Statistics::Aggregation do - include_context 'Use stubs for Faraday' + include_context 'Use stubs for Connpass' + include_context 'Use stubs for Doorkeeper' + include_context 'Use stubs for Facebook' before(:all) do Dojo.destroy_all @@ -16,14 +18,16 @@ before do d1 = Dojo.create(name: 'Dojo1', email: 'info@dojo1.com', description: 'CoderDojo1', tags: %w(CoderDojo1), url: 'https://dojo1.com') d2 = Dojo.create(name: 'Dojo2', email: 'info@dojo2.com', description: 'CoderDojo2', tags: %w(CoderDojo2), url: 'https://dojo2.com') + d3 = Dojo.create(name: 'Dojo3', email: 'info@dojo3.com', description: 'CoderDojo3', tags: %w(CoderDojo3), url: 'https://dojo3.com') DojoEventService.create(dojo_id: d1.id, name: :connpass, group_id: 9876) DojoEventService.create(dojo_id: d2.id, name: :doorkeeper, group_id: 5555) + DojoEventService.create(dojo_id: d3.id, name: :facebook, group_id: 123451234512345) end subject { Statistics::Aggregation.run(date: Time.current) } it do - expect{ subject }.to change{EventHistory.count}.from(0).to(2) + expect{ subject }.to change{EventHistory.count}.from(0).to(3) end end end diff --git a/spec/lib/statistics/client_spec.rb b/spec/lib/statistics/client_spec.rb index f5313b72b..a3f3424be 100644 --- a/spec/lib/statistics/client_spec.rb +++ b/spec/lib/statistics/client_spec.rb @@ -2,9 +2,9 @@ require 'statistics' RSpec.describe Statistics::Client do - include_context 'Use stubs for Faraday' - context 'Connpass' do + include_context 'Use stubs for Connpass' + describe '#search' do subject { Statistics::Client::Connpass.new.search(keyword: 'coderdojo') } @@ -32,6 +32,8 @@ end context 'Doorkeeper' do + include_context 'Use stubs for Doorkeeper' + describe '#search' do subject { Statistics::Client::Doorkeeper.new.search(keyword: 'coderdojo') } @@ -54,4 +56,19 @@ end end end + + context 'Facebook' do + include_context 'Use stubs for Facebook' + + describe '#fetch_events' do + subject { Statistics::Client::Facebook.new.fetch_events(group_id: 123451234512345) } + + it do + expect(subject).to be_instance_of(Array) + expect(subject.size).to eq 1 + expect(subject.first['id']).to eq '125500978166443' + expect(subject.first.dig('owner', 'id')).to eq '123451234512345' + end + end + end end diff --git a/spec/support/shared_contexts/statistics.rb b/spec/support/shared_contexts/statistics.rb index fb303cb39..4cf3d1ed8 100644 --- a/spec/support/shared_contexts/statistics.rb +++ b/spec/support/shared_contexts/statistics.rb @@ -1,4 +1,26 @@ -RSpec.shared_context 'Use stubs for Faraday' do +RSpec.shared_context 'Use stub connection of Faraday' do + let(:stub_connection) do + Faraday.new do |f| + f.response :json, :content_type => /\bjson$/ + f.adapter :test, Faraday::Adapter::Test::Stubs.new do |stub| + # connpass + stub.get('/event/') { connpass_response } + + # doorkeeper + stub.get('/events') { doorkeeper_response } + stub.get('/groups/5555/events') { doorkeeper_response } + end + end + end + + before do + allow_any_instance_of(Statistics::Client).to receive(:connection_for).and_return(stub_connection) + end +end + +RSpec.shared_context 'Use stubs for Connpass' do + include_context 'Use stub connection of Faraday' + let(:connpass_response) do [ 200, @@ -6,6 +28,10 @@ '{"results_returned": 1, "events": [{"event_url": "https://coderdojo-okutama.connpass.com/event/12345/", "event_type": "participation", "owner_nickname": "nalabjp", "series": {"url": "https://coderdojo-okutama.connpass.com/", "id": 9876, "title": "CoderDojo series"}, "updated_at": "2017-04-29T14:59:30+09:00", "lat": "35.801763000000", "started_at": "2017-05-07T10:00:00+09:00", "hash_tag": "CoderDojo", "title": "CoderDojo title", "event_id": 12345, "lon": "139.087656000000", "waiting": 2, "limit": 10, "owner_id": 2525, "owner_display_name": "nalabjp", "description": "CoderDojo description", "address": "Okutama-cho Tokyo", "catch": "CoderDojo catch", "accepted": 10, "ended_at": "2017-05-07T12:00:00+09:00", "place": "Tokyo"}], "results_start": 200, "results_available": 518}' ] end +end + +RSpec.shared_context 'Use stubs for Doorkeeper' do + include_context 'Use stub connection of Faraday' let(:doorkeeper_response) do [ @@ -14,22 +40,25 @@ '[{"event":{"title":"CoderDojo title","id":1234,"starts_at":"2017-05-28T01:00:00.000Z","ends_at":"2017-05-28T04:00:00.000Z","venue_name":"奥多摩町","address":"奥多摩町","lat":"35.801763000000","long":"139.087656000000","ticket_limit":30,"published_at":"2017-04-22T03:43:04.000Z","updated_at":"2017-05-10T11:31:21.810Z","group":5555,"banner":null,"description":"CoderDojo description","public_url":"https://coderdojo-okutama.doorkeeper.jp/events/8888","participants":12,"waitlisted":0}}]' ] end +end - let(:stub_connection) do - Faraday.new do |f| - f.response :json, :content_type => /\bjson$/ - f.adapter :test, Faraday::Adapter::Test::Stubs.new do |stub| - # connpass - stub.get('/event/') { connpass_response } - - # doorkeeper - stub.get('/events') { doorkeeper_response } - stub.get('/groups/5555/events') { doorkeeper_response } - end - end +RSpec.shared_context 'Use stubs for Facebook' do + let(:facebook_response) do + resp = OpenStruct.new(data: { + "data" => [ + {"attending_count"=>1, "start_time"=>"2017-10-29T13:00:00+0900", "owner"=>{"name"=>"CoderDojo ひばりヶ丘", "id"=>"123451234512345"}, "id"=>"125500978166443"} + ], + "paging" => { + "cursors" => { + "before" => "QVFIUjVOd2tKSmZA6S01fR0NFNWN2aFJlc01JUnpqRW5aMFFkeHdBS3NTcUt1b3JfazUzM3FtVGhCYlN6bE1OS1lxZAzQ0YjVSNWRRVWRfd182SXh3LUN6VXZAB", + "after" => "QVFIUmZA1cEk5QlV0VFc2Ri1BT3JEOWl3M1gzemRZAZAkpkaFdjNTEwUDdtaERLdFpwYV9CejVuX3hLV2kyVm5Gem9KSTAzTGg0dUd4SjNLXzBlSTZAJMVAtdmln" + } + } + }) + Koala::Facebook::API::GraphCollection.new(resp, nil) end before do - allow_any_instance_of(Statistics::Client).to receive(:connection_for).and_return(stub_connection) + allow_any_instance_of(Koala::Facebook::API).to receive(:get_object).and_return(facebook_response) end end