From 9a845a21a91d2a6f6da8fb18a30d9753bd9e2254 Mon Sep 17 00:00:00 2001 From: nalabjp Date: Mon, 9 Oct 2017 23:45:21 +0900 Subject: [PATCH 01/14] Gem `koala` --- Gemfile | 2 ++ Gemfile.lock | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/Gemfile b/Gemfile index 7527d27dc..0f6dd4ffc 100644 --- a/Gemfile +++ b/Gemfile @@ -31,6 +31,8 @@ gem 'faraday' # https://github.com/bundler/bundler/issues/5332 gem 'faraday_middleware', '0.10' +gem 'koala' + group :development do gem 'web-console' gem 'spring' diff --git a/Gemfile.lock b/Gemfile.lock index a353d148a..2e7a877a6 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -103,6 +103,10 @@ GEM jquery-ui-rails (6.0.1) railties (>= 3.2.16) json (2.0.3) + koala (3.0.0) + addressable + faraday + json (>= 1.8) kramdown (1.13.2) launchy (2.4.3) addressable (~> 2.3) @@ -289,6 +293,7 @@ DEPENDENCIES font-awesome-rails jbuilder jquery-rails + koala kramdown minitest-retry pg From 16c8169a237e6e6081e00725e14d46359eaf71b4 Mon Sep 17 00:00:00 2001 From: nalabjp Date: Wed, 18 Oct 2017 03:33:32 +0900 Subject: [PATCH 02/14] Get access token for Facebook API --- lib/tasks/oauth.rake | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 lib/tasks/oauth.rake diff --git a/lib/tasks/oauth.rake b/lib/tasks/oauth.rake new file mode 100644 index 000000000..bd731c2d9 --- /dev/null +++ b/lib/tasks/oauth.rake @@ -0,0 +1,6 @@ +namespace :oauth do + desc 'Facebookのaccess tokenを取得します' + task :facebook_access_token, [:app_id, :app_secret] => :environment do |_tasks, args| + puts 'Access Token: ' + Koala::Facebook::OAuth.new(args[:app_id], args[:app_secret]).get_app_access_token + end +end From f246a2e8397b1d0393daa11df7d0c7cee46d4dfe Mon Sep 17 00:00:00 2001 From: nalabjp Date: Sun, 29 Oct 2017 12:47:54 +0900 Subject: [PATCH 03/14] Split shared context --- spec/lib/statistics/aggregation_spec.rb | 3 +- spec/lib/statistics/client_spec.rb | 6 ++-- spec/support/shared_contexts/statistics.rb | 42 +++++++++++++--------- 3 files changed, 31 insertions(+), 20 deletions(-) diff --git a/spec/lib/statistics/aggregation_spec.rb b/spec/lib/statistics/aggregation_spec.rb index 37a20aa23..b680d93cb 100644 --- a/spec/lib/statistics/aggregation_spec.rb +++ b/spec/lib/statistics/aggregation_spec.rb @@ -2,7 +2,8 @@ require 'statistics' RSpec.describe Statistics::Aggregation do - include_context 'Use stubs for Faraday' + include_context 'Use stubs for Connpass' + include_context 'Use stubs for Doorkeeper' before(:all) do Dojo.delete_all diff --git a/spec/lib/statistics/client_spec.rb b/spec/lib/statistics/client_spec.rb index f5313b72b..36fc2fcab 100644 --- a/spec/lib/statistics/client_spec.rb +++ b/spec/lib/statistics/client_spec.rb @@ -2,9 +2,9 @@ require 'statistics' RSpec.describe Statistics::Client do - include_context 'Use stubs for Faraday' - context 'Connpass' do + include_context 'Use stubs for Connpass' + describe '#search' do subject { Statistics::Client::Connpass.new.search(keyword: 'coderdojo') } @@ -32,6 +32,8 @@ end context 'Doorkeeper' do + include_context 'Use stubs for Doorkeeper' + describe '#search' do subject { Statistics::Client::Doorkeeper.new.search(keyword: 'coderdojo') } diff --git a/spec/support/shared_contexts/statistics.rb b/spec/support/shared_contexts/statistics.rb index fb303cb39..76d505349 100644 --- a/spec/support/shared_contexts/statistics.rb +++ b/spec/support/shared_contexts/statistics.rb @@ -1,20 +1,4 @@ -RSpec.shared_context 'Use stubs for Faraday' do - let(:connpass_response) do - [ - 200, - { 'Content-Type' => 'application/json' }, - '{"results_returned": 1, "events": [{"event_url": "https://coderdojo-okutama.connpass.com/event/12345/", "event_type": "participation", "owner_nickname": "nalabjp", "series": {"url": "https://coderdojo-okutama.connpass.com/", "id": 9876, "title": "CoderDojo series"}, "updated_at": "2017-04-29T14:59:30+09:00", "lat": "35.801763000000", "started_at": "2017-05-07T10:00:00+09:00", "hash_tag": "CoderDojo", "title": "CoderDojo title", "event_id": 12345, "lon": "139.087656000000", "waiting": 2, "limit": 10, "owner_id": 2525, "owner_display_name": "nalabjp", "description": "CoderDojo description", "address": "Okutama-cho Tokyo", "catch": "CoderDojo catch", "accepted": 10, "ended_at": "2017-05-07T12:00:00+09:00", "place": "Tokyo"}], "results_start": 200, "results_available": 518}' - ] - end - - let(:doorkeeper_response) do - [ - 200, - { 'Content-Type' => 'application/json' }, - '[{"event":{"title":"CoderDojo title","id":1234,"starts_at":"2017-05-28T01:00:00.000Z","ends_at":"2017-05-28T04:00:00.000Z","venue_name":"奥多摩町","address":"奥多摩町","lat":"35.801763000000","long":"139.087656000000","ticket_limit":30,"published_at":"2017-04-22T03:43:04.000Z","updated_at":"2017-05-10T11:31:21.810Z","group":5555,"banner":null,"description":"CoderDojo description","public_url":"https://coderdojo-okutama.doorkeeper.jp/events/8888","participants":12,"waitlisted":0}}]' - ] - end - +RSpec.shared_context 'Use stub connection of Faraday' do let(:stub_connection) do Faraday.new do |f| f.response :json, :content_type => /\bjson$/ @@ -33,3 +17,27 @@ allow_any_instance_of(Statistics::Client).to receive(:connection_for).and_return(stub_connection) end end + +RSpec.shared_context 'Use stubs for Connpass' do + include_context 'Use stub connection of Faraday' + + let(:connpass_response) do + [ + 200, + { 'Content-Type' => 'application/json' }, + '{"results_returned": 1, "events": [{"event_url": "https://coderdojo-okutama.connpass.com/event/12345/", "event_type": "participation", "owner_nickname": "nalabjp", "series": {"url": "https://coderdojo-okutama.connpass.com/", "id": 9876, "title": "CoderDojo series"}, "updated_at": "2017-04-29T14:59:30+09:00", "lat": "35.801763000000", "started_at": "2017-05-07T10:00:00+09:00", "hash_tag": "CoderDojo", "title": "CoderDojo title", "event_id": 12345, "lon": "139.087656000000", "waiting": 2, "limit": 10, "owner_id": 2525, "owner_display_name": "nalabjp", "description": "CoderDojo description", "address": "Okutama-cho Tokyo", "catch": "CoderDojo catch", "accepted": 10, "ended_at": "2017-05-07T12:00:00+09:00", "place": "Tokyo"}], "results_start": 200, "results_available": 518}' + ] + end +end + +RSpec.shared_context 'Use stubs for Doorkeeper' do + include_context 'Use stub connection of Faraday' + + let(:doorkeeper_response) do + [ + 200, + { 'Content-Type' => 'application/json' }, + '[{"event":{"title":"CoderDojo title","id":1234,"starts_at":"2017-05-28T01:00:00.000Z","ends_at":"2017-05-28T04:00:00.000Z","venue_name":"奥多摩町","address":"奥多摩町","lat":"35.801763000000","long":"139.087656000000","ticket_limit":30,"published_at":"2017-04-22T03:43:04.000Z","updated_at":"2017-05-10T11:31:21.810Z","group":5555,"banner":null,"description":"CoderDojo description","public_url":"https://coderdojo-okutama.doorkeeper.jp/events/8888","participants":12,"waitlisted":0}}]' + ] + end +end From 78347bfa4d3e0826907eaf28a3f4683e5960c488 Mon Sep 17 00:00:00 2001 From: nalabjp Date: Sun, 29 Oct 2017 12:57:42 +0900 Subject: [PATCH 04/14] Add implementation of facebook api --- lib/statistics/client.rb | 25 ++++++++++++++++++++++ spec/lib/statistics/client_spec.rb | 15 +++++++++++++ spec/support/shared_contexts/statistics.rb | 21 ++++++++++++++++++ 3 files changed, 61 insertions(+) diff --git a/lib/statistics/client.rb b/lib/statistics/client.rb index 9ec627d52..92f939b95 100644 --- a/lib/statistics/client.rb +++ b/lib/statistics/client.rb @@ -101,5 +101,30 @@ def fetch_events(group_id:, since_at: @default_since, until_at: @default_until) events end end + + class Facebook + def initialize + @client = Koala::Facebook::API.new(ENV.fetch('FACEBOOK_ACCESS_TOKEN')) + end + + def fetch_events(group_id:, since_at: nil, until_at: nil) + params = { + fields: %i(attending_count start_time owner), + limit: 100, + since: since_at, + until: until_at + }.compact! + + events = [] + + collection = @client.get_object("#{group_id}/events", params) + events.push(*collection.to_a) + while collection.paging['next'] do + events.push(*collection.next_page.to_a) + end + + events + end + end end end diff --git a/spec/lib/statistics/client_spec.rb b/spec/lib/statistics/client_spec.rb index 36fc2fcab..a3f3424be 100644 --- a/spec/lib/statistics/client_spec.rb +++ b/spec/lib/statistics/client_spec.rb @@ -56,4 +56,19 @@ end end end + + context 'Facebook' do + include_context 'Use stubs for Facebook' + + describe '#fetch_events' do + subject { Statistics::Client::Facebook.new.fetch_events(group_id: 123451234512345) } + + it do + expect(subject).to be_instance_of(Array) + expect(subject.size).to eq 1 + expect(subject.first['id']).to eq '125500978166443' + expect(subject.first.dig('owner', 'id')).to eq '123451234512345' + end + end + end end diff --git a/spec/support/shared_contexts/statistics.rb b/spec/support/shared_contexts/statistics.rb index 76d505349..4cf3d1ed8 100644 --- a/spec/support/shared_contexts/statistics.rb +++ b/spec/support/shared_contexts/statistics.rb @@ -41,3 +41,24 @@ ] end end + +RSpec.shared_context 'Use stubs for Facebook' do + let(:facebook_response) do + resp = OpenStruct.new(data: { + "data" => [ + {"attending_count"=>1, "start_time"=>"2017-10-29T13:00:00+0900", "owner"=>{"name"=>"CoderDojo ひばりヶ丘", "id"=>"123451234512345"}, "id"=>"125500978166443"} + ], + "paging" => { + "cursors" => { + "before" => "QVFIUjVOd2tKSmZA6S01fR0NFNWN2aFJlc01JUnpqRW5aMFFkeHdBS3NTcUt1b3JfazUzM3FtVGhCYlN6bE1OS1lxZAzQ0YjVSNWRRVWRfd182SXh3LUN6VXZAB", + "after" => "QVFIUmZA1cEk5QlV0VFc2Ri1BT3JEOWl3M1gzemRZAZAkpkaFdjNTEwUDdtaERLdFpwYV9CejVuX3hLV2kyVm5Gem9KSTAzTGg0dUd4SjNLXzBlSTZAJMVAtdmln" + } + } + }) + Koala::Facebook::API::GraphCollection.new(resp, nil) + end + + before do + allow_any_instance_of(Koala::Facebook::API).to receive(:get_object).and_return(facebook_response) + end +end From 3344ffec55676ce42d6d03ca3e61f30d2d2ba04b Mon Sep 17 00:00:00 2001 From: nalabjp Date: Sun, 29 Oct 2017 16:12:46 +0900 Subject: [PATCH 05/14] Change column type from integer to string for facebook --- ...teger_to_string_on_group_id_in_dojo_event_services.rb | 9 +++++++++ db/schema.rb | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 db/migrate/20171029065909_integer_to_string_on_group_id_in_dojo_event_services.rb diff --git a/db/migrate/20171029065909_integer_to_string_on_group_id_in_dojo_event_services.rb b/db/migrate/20171029065909_integer_to_string_on_group_id_in_dojo_event_services.rb new file mode 100644 index 000000000..985cc6a70 --- /dev/null +++ b/db/migrate/20171029065909_integer_to_string_on_group_id_in_dojo_event_services.rb @@ -0,0 +1,9 @@ +class IntegerToStringOnGroupIdInDojoEventServices < ActiveRecord::Migration[5.0] + def up + change_column :dojo_event_services, :group_id, :string, null: false + end + + def down + change_column :dojo_event_services, :group_id, :integer, null: false + end +end diff --git a/db/schema.rb b/db/schema.rb index c4ba02606..5c5b06f78 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,13 +10,13 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170820090605) do +ActiveRecord::Schema.define(version: 20171029065909) do create_table "dojo_event_services", force: :cascade do |t| t.integer "dojo_id", null: false t.integer "name", null: false t.string "url" - t.integer "group_id", null: false + t.string "group_id", null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["dojo_id"], name: "index_dojo_event_services_on_dojo_id" From e0b2147af48b275155fa99340af3b81504e39df8 Mon Sep 17 00:00:00 2001 From: nalabjp Date: Sun, 29 Oct 2017 23:15:21 +0900 Subject: [PATCH 06/14] Change column type from integer to string for facebook --- ...ervice_group_id_and_event_id_in_event_histories.rb | 11 +++++++++++ db/schema.rb | 6 +++--- 2 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 db/migrate/20171029071514_integer_to_string_on_service_group_id_and_event_id_in_event_histories.rb diff --git a/db/migrate/20171029071514_integer_to_string_on_service_group_id_and_event_id_in_event_histories.rb b/db/migrate/20171029071514_integer_to_string_on_service_group_id_and_event_id_in_event_histories.rb new file mode 100644 index 000000000..74baf8fbf --- /dev/null +++ b/db/migrate/20171029071514_integer_to_string_on_service_group_id_and_event_id_in_event_histories.rb @@ -0,0 +1,11 @@ +class IntegerToStringOnServiceGroupIdAndEventIdInEventHistories < ActiveRecord::Migration[5.0] + def up + change_column :event_histories, :service_group_id, :string, null: false + change_column :event_histories, :event_id, :string, null: false + end + + def down + change_column :event_histories, :service_group_id, :integer, null: false + change_column :event_histories, :event_id, :integer, null: false + end +end diff --git a/db/schema.rb b/db/schema.rb index 5c5b06f78..ad2f0b4be 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20171029065909) do +ActiveRecord::Schema.define(version: 20171029071514) do create_table "dojo_event_services", force: :cascade do |t| t.integer "dojo_id", null: false @@ -38,8 +38,8 @@ t.integer "dojo_id", null: false t.string "dojo_name", null: false t.string "service_name", null: false - t.integer "service_group_id", null: false - t.integer "event_id", null: false + t.string "service_group_id", null: false + t.string "event_id", null: false t.string "event_url", null: false t.integer "participants", null: false t.datetime "evented_at", null: false From 9589fab6623b1f617d02bd49d60768ff4265e7db Mon Sep 17 00:00:00 2001 From: nalabjp Date: Sun, 29 Oct 2017 23:27:35 +0900 Subject: [PATCH 07/14] Fix change column type --- lib/statistics/aggregation.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/statistics/aggregation.rb b/lib/statistics/aggregation.rb index f1c1b4109..19b0642a7 100644 --- a/lib/statistics/aggregation.rb +++ b/lib/statistics/aggregation.rb @@ -20,7 +20,7 @@ def run(dojos, date) dojos.each do |dojo| cnps.fetch_events(params.merge(series_id: dojo.dojo_event_service.group_id)).each do |e| - next unless e.dig('series', 'id') == dojo.dojo_event_service.group_id + next unless e.dig('series', 'id').to_s == dojo.dojo_event_service.group_id EventHistory.create!(dojo_id: dojo.id, dojo_name: dojo.name, @@ -47,7 +47,7 @@ def run(dojos, date) dojos.each do |dojo| drkp.fetch_events(params.merge(group_id: dojo.dojo_event_service.group_id)).each do |e| - next unless e['group'] == dojo.dojo_event_service.group_id + next unless e['group'].to_s == dojo.dojo_event_service.group_id EventHistory.create!(dojo_id: dojo.id, dojo_name: dojo.name, From 213320cdfaa95c923e066b53c73e88a16983c3ff Mon Sep 17 00:00:00 2001 From: nalabjp Date: Sun, 29 Oct 2017 23:27:58 +0900 Subject: [PATCH 08/14] Support facebook on Statistics::Aggregation --- app/models/dojo_event_service.rb | 2 +- lib/statistics/aggregation.rb | 29 +++++++++++++++++++++++++ spec/lib/statistics/aggregation_spec.rb | 5 ++++- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/app/models/dojo_event_service.rb b/app/models/dojo_event_service.rb index 61e07b3ca..fdc3c493c 100644 --- a/app/models/dojo_event_service.rb +++ b/app/models/dojo_event_service.rb @@ -1,4 +1,4 @@ class DojoEventService < ApplicationRecord belongs_to :dojo - enum name: %i( connpass doorkeeper ) + enum name: %i( connpass doorkeeper facebook ) end diff --git a/lib/statistics/aggregation.rb b/lib/statistics/aggregation.rb index 19b0642a7..f88a2827a 100644 --- a/lib/statistics/aggregation.rb +++ b/lib/statistics/aggregation.rb @@ -4,9 +4,11 @@ class << self def run(date:) cnps_dojos = Dojo.joins(:dojo_event_service).where(dojo_event_services: { name: :connpass }).to_a drkp_dojos = Dojo.joins(:dojo_event_service).where(dojo_event_services: { name: :doorkeeper }).to_a + fsbk_dojos = Dojo.joins(:dojo_event_service).where(dojo_event_services: { name: :facebook }).to_a Connpass.run(cnps_dojos, date) Doorkeeper.run(drkp_dojos, date) + Facebook.run(fsbk_dojos, date) end end @@ -62,5 +64,32 @@ def run(dojos, date) end end end + + class Facebook + class << self + def run(dojos, date) + fsbk = Client::Facebook.new + params = { + since_at: date.beginning_of_month, + until_at: date.end_of_month + } + + dojos.each do |dojo| + fsbk.fetch_events(params.merge(group_id: dojo.dojo_event_service.group_id)).each do |e| + next unless e.dig('owner', 'id') == dojo.dojo_event_service.group_id + + EventHistory.create!(dojo_id: dojo.id, + dojo_name: dojo.name, + service_name: dojo.dojo_event_service.name, + service_group_id: dojo.dojo_event_service.group_id, + event_id: e['id'], + event_url: "https://www.facebook.com/events/#{e['id']}", + participants: e['attending_count'], + evented_at: Time.zone.parse(e['start_time'])) + end + end + end + end + end end end diff --git a/spec/lib/statistics/aggregation_spec.rb b/spec/lib/statistics/aggregation_spec.rb index b680d93cb..91cbb7f39 100644 --- a/spec/lib/statistics/aggregation_spec.rb +++ b/spec/lib/statistics/aggregation_spec.rb @@ -4,6 +4,7 @@ RSpec.describe Statistics::Aggregation do include_context 'Use stubs for Connpass' include_context 'Use stubs for Doorkeeper' + include_context 'Use stubs for Facebook' before(:all) do Dojo.delete_all @@ -21,14 +22,16 @@ before do d1 = Dojo.create(name: 'Dojo1', email: 'info@dojo1.com', description: 'CoderDojo1', tags: %w(CoderDojo1), url: 'https://dojo1.com') d2 = Dojo.create(name: 'Dojo2', email: 'info@dojo2.com', description: 'CoderDojo2', tags: %w(CoderDojo2), url: 'https://dojo2.com') + d3 = Dojo.create(name: 'Dojo3', email: 'info@dojo3.com', description: 'CoderDojo3', tags: %w(CoderDojo3), url: 'https://dojo3.com') DojoEventService.create(dojo_id: d1.id, name: :connpass, group_id: 9876) DojoEventService.create(dojo_id: d2.id, name: :doorkeeper, group_id: 5555) + DojoEventService.create(dojo_id: d3.id, name: :facebook, group_id: 123451234512345) end subject { Statistics::Aggregation.run(date: Time.current) } it do - expect{ subject }.to change{EventHistory.count}.from(0).to(2) + expect{ subject }.to change{EventHistory.count}.from(0).to(3) end end end From 59022eab06f4841f07303659ba36019f1f1aa988 Mon Sep 17 00:00:00 2001 From: nalabjp Date: Mon, 30 Oct 2017 02:18:47 +0900 Subject: [PATCH 09/14] Add dummy environment variable for CI --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 2651fe9fc..651505176 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,3 +20,4 @@ script: env: global: - TZ='Asia/Tokyo' + - FACEBOOK_ACCESS_TOKEN='dummy_token' From fdf0956f7ec36874e720c380f463fbda15af7d02 Mon Sep 17 00:00:00 2001 From: nalabjp Date: Tue, 31 Oct 2017 21:41:40 +0900 Subject: [PATCH 10/14] Remove unnecessary bang --- lib/statistics/client.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/statistics/client.rb b/lib/statistics/client.rb index be8e70f40..774ca8e01 100644 --- a/lib/statistics/client.rb +++ b/lib/statistics/client.rb @@ -113,7 +113,7 @@ def fetch_events(group_id:, since_at: nil, until_at: nil) limit: 100, since: since_at, until: until_at - }.compact! + }.compact events = [] From 9d5736ffcf749a77a29a9b5367aa830daefd9d76 Mon Sep 17 00:00:00 2001 From: nalabjp Date: Tue, 31 Oct 2017 21:42:03 +0900 Subject: [PATCH 11/14] Check response is empty --- lib/statistics/client.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/statistics/client.rb b/lib/statistics/client.rb index 774ca8e01..c4b16f161 100644 --- a/lib/statistics/client.rb +++ b/lib/statistics/client.rb @@ -119,7 +119,7 @@ def fetch_events(group_id:, since_at: nil, until_at: nil) collection = @client.get_object("#{group_id}/events", params) events.push(*collection.to_a) - while collection.paging['next'] do + while !collection.empty? && collection.paging['next'] do events.push(*collection.next_page.to_a) end From de9841cfb18f20386be7eefa3e29fbbbf7baf311 Mon Sep 17 00:00:00 2001 From: nalabjp Date: Wed, 1 Nov 2017 03:22:28 +0900 Subject: [PATCH 12/14] Fetch access token each Rake task execution --- .travis.yml | 1 - lib/statistics/client.rb | 4 +++- lib/tasks/statistics.rake | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 651505176..2651fe9fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,4 +20,3 @@ script: env: global: - TZ='Asia/Tokyo' - - FACEBOOK_ACCESS_TOKEN='dummy_token' diff --git a/lib/statistics/client.rb b/lib/statistics/client.rb index 724402086..bf1e13f50 100644 --- a/lib/statistics/client.rb +++ b/lib/statistics/client.rb @@ -111,8 +111,10 @@ def fetch_events(group_id:, since_at: @default_since, until_at: @default_until) end class Facebook + class_attribute :access_token + def initialize - @client = Koala::Facebook::API.new(ENV.fetch('FACEBOOK_ACCESS_TOKEN')) + @client = Koala::Facebook::API.new(self.access_token) end def fetch_events(group_id:, since_at: nil, until_at: nil) diff --git a/lib/tasks/statistics.rake b/lib/tasks/statistics.rake index 6b9b115fa..1da1e15c2 100644 --- a/lib/tasks/statistics.rake +++ b/lib/tasks/statistics.rake @@ -35,6 +35,8 @@ namespace :statistics do Time.current.prev_month.end_of_month end + Statistics::Client::Facebook.access_token = Koala::Facebook::OAuth.new(ENV['FACEBOOK_APP_ID'], ENV['FACEBOOK_APP_SECRET']).get_app_access_token + EventHistory.where(evented_at: from..to).delete_all loop.with_object([from]) { |_, list| From a7da867f7dd8d17f167a2b151d4c81b11ec1add2 Mon Sep 17 00:00:00 2001 From: nalabjp Date: Fri, 3 Nov 2017 01:46:21 +0900 Subject: [PATCH 13/14] Reduce the range of retry --- lib/statistics/client.rb | 43 ++++++++++++++++++++------------------- lib/tasks/statistics.rake | 11 ++-------- 2 files changed, 24 insertions(+), 30 deletions(-) diff --git a/lib/statistics/client.rb b/lib/statistics/client.rb index bf1e13f50..817e03611 100644 --- a/lib/statistics/client.rb +++ b/lib/statistics/client.rb @@ -1,7 +1,5 @@ module Statistics class Client - class APIRateLimitError < ::StandardError; end - class_attribute :debug self.debug = false @@ -81,31 +79,34 @@ def search(keyword:) end def fetch_events(group_id:, since_at: @default_since, until_at: @default_until) - params = { - page: 1, - since: since_at, - until: until_at - } - events = [] + begin + params = { + page: 1, + since: since_at, + until: until_at + } + events = [] - loop do - part = @client.get("groups/#{group_id}/events", params) + loop do + part = @client.get("groups/#{group_id}/events", params) - break if part.size.zero? + break if part.size.zero? - events.push(*part.map { |e| e['event'] }) + events.push(*part.map { |e| e['event'] }) - break if part.size < 25 # 25 items / 1 request + break if part.size < 25 # 25 items / 1 request - params[:page] += 1 - end + params[:page] += 1 + end - events - rescue Faraday::ClientError => e - if e.response[:status] == 429 - raise Client::APIRateLimitError - else - raise e + events + rescue Faraday::ClientError => e + raise e unless e.response[:status] == 429 + + puts 'API rate limit exceeded.' + puts "This task will retry in 60 seconds from now(#{Time.zone.now})." + sleep 60 + retry end end end diff --git a/lib/tasks/statistics.rake b/lib/tasks/statistics.rake index 1da1e15c2..8a18b0428 100644 --- a/lib/tasks/statistics.rake +++ b/lib/tasks/statistics.rake @@ -44,15 +44,8 @@ namespace :statistics do raise StopIteration if nm > to list << nm }.each { |date| - begin - puts "Aggregate for #{date.strftime('%Y/%m')}" - Statistics::Aggregation.run(date: date) - rescue Statistics::Client::APIRateLimitError - puts 'API rate limit exceeded.' - puts "This task will retry in 60 seconds from now(#{Time.zone.now})." - sleep 60 - retry - end + puts "Aggregate for #{date.strftime('%Y/%m')}" + Statistics::Aggregation.run(date: date) } end From cca2f3acd2ec8edef6f8434b1f7daf88d5251533 Mon Sep 17 00:00:00 2001 From: nalabjp Date: Fri, 3 Nov 2017 02:34:58 +0900 Subject: [PATCH 14/14] Notify to idobata when task is finished --- lib/tasks/statistics.rake | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/lib/tasks/statistics.rake b/lib/tasks/statistics.rake index 8a18b0428..25684448e 100644 --- a/lib/tasks/statistics.rake +++ b/lib/tasks/statistics.rake @@ -16,6 +16,10 @@ namespace :statistics do d } + notify_idobata = -> (msg) { + puts `curl --data-urlencode "source=#{msg}" -s #{ENV['IDOBATA_HOOK_URL']} -o /dev/null -w "idobata: %{http_code}"` if ENV.key?('IDOBATA_HOOK_URL') + } + from = if args[:from] if args[:from].length == 4 date_from_str.call(args[:from]).beginning_of_year @@ -39,14 +43,20 @@ namespace :statistics do EventHistory.where(evented_at: from..to).delete_all - loop.with_object([from]) { |_, list| - nm = list.last.next_month - raise StopIteration if nm > to - list << nm - }.each { |date| - puts "Aggregate for #{date.strftime('%Y/%m')}" - Statistics::Aggregation.run(date: date) - } + begin + loop.with_object([from]) { |_, list| + nm = list.last.next_month + raise StopIteration if nm > to + list << nm + }.each { |date| + puts "Aggregate for #{date.strftime('%Y/%m')}" + Statistics::Aggregation.run(date: date) + } + + notify_idobata.call("#{from.strftime('%Y/%m')}~#{to.strftime('%Y/%m')}のイベント履歴の集計を行いました") + rescue + notify_idobata.call("#{from.strftime('%Y/%m')}~#{to.strftime('%Y/%m')}のイベント履歴の集計でエラーが発生しました") + end end desc 'キーワードからイベント情報を検索します'