Merge pull request #164 from coderdojo-japan/support-facebook-api

Support Facebook API in Statistics::Aggregation

Author: nalabjp

Gemfile

@@ -31,6 +31,8 @@ gem 'faraday'
 # https://github.com/bundler/bundler/issues/5332
 gem 'faraday_middleware', '0.10'
 
+gem 'koala'
+
 group :development do
   gem 'web-console'
   gem 'spring'

Gemfile.lock

@@ -106,6 +106,10 @@ GEM
     jquery-ui-rails (6.0.1)
       railties (>= 3.2.16)
     json (2.1.0)
+    koala (3.0.0)
+      addressable
+      faraday
+      json (>= 1.8)
     kramdown (1.15.0)
     launchy (2.4.3)
       addressable (~> 2.3)
@@ -305,6 +309,7 @@ DEPENDENCIES
   font-awesome-rails
   jbuilder
   jquery-rails
+  koala
   kramdown
   listen
   minitest-retry

app/models/dojo_event_service.rb

@@ -1,4 +1,4 @@
 class DojoEventService < ApplicationRecord
   belongs_to :dojo
-  enum name: %i( connpass doorkeeper )
+  enum name: %i( connpass doorkeeper facebook )
 end

db/migrate/20171029065909_integer_to_string_on_group_id_in_dojo_event_services.rb

@@ -0,0 +1,9 @@
+class IntegerToStringOnGroupIdInDojoEventServices < ActiveRecord::Migration[5.0]
+  def up
+    change_column :dojo_event_services, :group_id, :string, null: false
+  end
+
+  def down
+    change_column :dojo_event_services, :group_id, :integer, null: false
+  end
+end

db/migrate/20171029071514_integer_to_string_on_service_group_id_and_event_id_in_event_histories.rb

@@ -0,0 +1,11 @@
+class IntegerToStringOnServiceGroupIdAndEventIdInEventHistories < ActiveRecord::Migration[5.0]
+  def up
+    change_column :event_histories, :service_group_id, :string, null: false
+    change_column :event_histories, :event_id, :string, null: false
+  end
+
+  def down
+    change_column :event_histories, :service_group_id, :integer, null: false
+    change_column :event_histories, :event_id, :integer, null: false
+  end
+end

db/schema.rb

@@ -16,7 +16,7 @@
     t.integer "dojo_id", null: false
     t.integer "name", null: false
     t.string "url"
-    t.integer "group_id", null: false
+    t.string "group_id", null: false
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.index ["dojo_id"], name: "index_dojo_event_services_on_dojo_id"
@@ -39,8 +39,8 @@
     t.integer "dojo_id", null: false
     t.string "dojo_name", null: false
     t.string "service_name", null: false
-    t.integer "service_group_id", null: false
-    t.integer "event_id", null: false
+    t.string "service_group_id", null: false
+    t.string "event_id", null: false
     t.string "event_url", null: false
     t.integer "participants", null: false
     t.datetime "evented_at", null: false

lib/statistics/aggregation.rb

@@ -4,9 +4,11 @@ class << self
       def run(date:)
         cnps_dojos = Dojo.joins(:dojo_event_service).where(dojo_event_services: { name: :connpass }).to_a
         drkp_dojos = Dojo.joins(:dojo_event_service).where(dojo_event_services: { name: :doorkeeper }).to_a
+        fsbk_dojos = Dojo.joins(:dojo_event_service).where(dojo_event_services: { name: :facebook }).to_a
 
         Connpass.run(cnps_dojos, date)
         Doorkeeper.run(drkp_dojos, date)
+        Facebook.run(fsbk_dojos, date)
       end
     end
 
@@ -20,7 +22,7 @@ def run(dojos, date)
 
           dojos.each do |dojo|
             cnps.fetch_events(params.merge(series_id: dojo.dojo_event_service.group_id)).each do |e|
-              next unless e.dig('series', 'id') == dojo.dojo_event_service.group_id
+              next unless e.dig('series', 'id').to_s == dojo.dojo_event_service.group_id
 
               EventHistory.create!(dojo_id: dojo.id,
                                    dojo_name: dojo.name,
@@ -47,7 +49,7 @@ def run(dojos, date)
 
           dojos.each do |dojo|
             drkp.fetch_events(params.merge(group_id: dojo.dojo_event_service.group_id)).each do |e|
-              next unless e['group'] == dojo.dojo_event_service.group_id
+              next unless e['group'].to_s == dojo.dojo_event_service.group_id
 
               EventHistory.create!(dojo_id: dojo.id,
                                    dojo_name: dojo.name,
@@ -62,5 +64,32 @@ def run(dojos, date)
         end
       end
     end
+
+    class Facebook
+      class << self
+        def run(dojos, date)
+          fsbk = Client::Facebook.new
+          params = {
+            since_at: date.beginning_of_month,
+            until_at: date.end_of_month
+          }
+
+          dojos.each do |dojo|
+            fsbk.fetch_events(params.merge(group_id: dojo.dojo_event_service.group_id)).each do |e|
+              next unless e.dig('owner', 'id') == dojo.dojo_event_service.group_id
+
+              EventHistory.create!(dojo_id: dojo.id,
+                                   dojo_name: dojo.name,
+                                   service_name: dojo.dojo_event_service.name,
+                                   service_group_id: dojo.dojo_event_service.group_id,
+                                   event_id: e['id'],
+                                   event_url: "https://www.facebook.com/events/#{e['id']}",
+                                   participants: e['attending_count'],
+                                   evented_at: Time.zone.parse(e['start_time']))
+            end
+          end
+        end
+      end
+    end
   end
 end

lib/statistics/client.rb

@@ -1,7 +1,5 @@
 module Statistics
   class Client
-    class APIRateLimitError < ::StandardError; end
-
     class_attribute :debug
     self.debug = false
 
@@ -81,32 +79,62 @@ def search(keyword:)
       end
 
       def fetch_events(group_id:, since_at: @default_since, until_at: @default_until)
-        params = {
-          page: 1,
-          since: since_at,
-          until: until_at
-        }
-        events = []
+        begin
+          params = {
+            page: 1,
+            since: since_at,
+            until: until_at
+          }
+          events = []
 
-        loop do
-          part = @client.get("groups/#{group_id}/events", params)
+          loop do
+            part = @client.get("groups/#{group_id}/events", params)
+
+            break if part.size.zero?
+
+            events.push(*part.map { |e| e['event'] })
 
-          break if part.size.zero?
+            break if part.size < 25   # 25 items / 1 request
 
-          events.push(*part.map { |e| e['event'] })
+            params[:page] += 1
+          end
 
-          break if part.size < 25   # 25 items / 1 request
+          events
+        rescue Faraday::ClientError => e
+          raise e unless e.response[:status] == 429
 
-          params[:page] += 1
+          puts 'API rate limit exceeded.'
+          puts "This task will retry in 60 seconds from now(#{Time.zone.now})."
+          sleep 60
+          retry
         end
+      end
+    end
 
-        events
-      rescue Faraday::ClientError => e
-        if e.response[:status] == 429
-          raise Client::APIRateLimitError
-        else
-          raise e
+    class Facebook
+      class_attribute :access_token
+
+      def initialize
+        @client = Koala::Facebook::API.new(self.access_token)
+      end
+
+      def fetch_events(group_id:, since_at: nil, until_at: nil)
+        params = {
+          fields: %i(attending_count start_time owner),
+          limit: 100,
+          since: since_at,
+          until: until_at
+        }.compact
+
+        events = []
+
+        collection = @client.get_object("#{group_id}/events", params)
+        events.push(*collection.to_a)
+        while !collection.empty? && collection.paging['next'] do
+          events.push(*collection.next_page.to_a)
         end
+
+        events
       end
     end
   end

lib/tasks/oauth.rake

@@ -0,0 +1,6 @@
+namespace :oauth do
+  desc 'Facebookのaccess tokenを取得します'
+  task :facebook_access_token, [:app_id, :app_secret] => :environment do |_tasks, args|
+    puts 'Access Token: ' + Koala::Facebook::OAuth.new(args[:app_id], args[:app_secret]).get_app_access_token
+  end
+end

lib/tasks/statistics.rake

@@ -16,6 +16,10 @@ namespace :statistics do
       d
     }
 
+    notify_idobata = -> (msg) {
+      puts `curl --data-urlencode "source=#{msg}" -s #{ENV['IDOBATA_HOOK_URL']} -o /dev/null -w "idobata: %{http_code}"` if ENV.key?('IDOBATA_HOOK_URL')
+    }
+
     from = if args[:from]
              if args[:from].length == 4
                date_from_str.call(args[:from]).beginning_of_year
@@ -35,23 +39,24 @@ namespace :statistics do
            Time.current.prev_month.end_of_month
          end
 
+    Statistics::Client::Facebook.access_token = Koala::Facebook::OAuth.new(ENV['FACEBOOK_APP_ID'], ENV['FACEBOOK_APP_SECRET']).get_app_access_token
+
     EventHistory.where(evented_at: from..to).delete_all
 
-    loop.with_object([from]) { |_, list|
-      nm = list.last.next_month
-      raise StopIteration if nm > to
-      list << nm
-    }.each { |date|
-      begin
+    begin
+      loop.with_object([from]) { |_, list|
+        nm = list.last.next_month
+        raise StopIteration if nm > to
+        list << nm
+      }.each { |date|
         puts "Aggregate for #{date.strftime('%Y/%m')}"
         Statistics::Aggregation.run(date: date)
-      rescue Statistics::Client::APIRateLimitError
-        puts 'API rate limit exceeded.'
-        puts "This task will retry in 60 seconds from now(#{Time.zone.now})."
-        sleep 60
-        retry
-      end
-    }
+      }
+
+      notify_idobata.call("#{from.strftime('%Y/%m')}~#{to.strftime('%Y/%m')}のイベント履歴の集計を行いました")
+    rescue
+      notify_idobata.call("#{from.strftime('%Y/%m')}~#{to.strftime('%Y/%m')}のイベント履歴の集計でエラーが発生しました")
+    end
   end
 
   desc 'キーワードからイベント情報を検索します'

spec/lib/statistics/aggregation_spec.rb

@@ -2,7 +2,9 @@
 require 'statistics'
 
 RSpec.describe Statistics::Aggregation do
-  include_context 'Use stubs for Faraday'
+  include_context 'Use stubs for Connpass'
+  include_context 'Use stubs for Doorkeeper'
+  include_context 'Use stubs for Facebook'
 
   before(:all) do
     Dojo.destroy_all
@@ -16,14 +18,16 @@
     before do
       d1 = Dojo.create(name: 'Dojo1', email: 'info@dojo1.com', description: 'CoderDojo1', tags: %w(CoderDojo1), url: 'https://dojo1.com')
       d2 = Dojo.create(name: 'Dojo2', email: 'info@dojo2.com', description: 'CoderDojo2', tags: %w(CoderDojo2), url: 'https://dojo2.com')
+      d3 = Dojo.create(name: 'Dojo3', email: 'info@dojo3.com', description: 'CoderDojo3', tags: %w(CoderDojo3), url: 'https://dojo3.com')
       DojoEventService.create(dojo_id: d1.id, name: :connpass, group_id: 9876)
       DojoEventService.create(dojo_id: d2.id, name: :doorkeeper, group_id: 5555)
+      DojoEventService.create(dojo_id: d3.id, name: :facebook, group_id: 123451234512345)
     end
 
     subject { Statistics::Aggregation.run(date: Time.current) }
 
     it do
-      expect{ subject }.to change{EventHistory.count}.from(0).to(2)
+      expect{ subject }.to change{EventHistory.count}.from(0).to(3)
     end
   end
 end

spec/lib/statistics/client_spec.rb

@@ -2,9 +2,9 @@
 require 'statistics'
 
 RSpec.describe Statistics::Client do
-  include_context 'Use stubs for Faraday'
-
   context 'Connpass' do
+    include_context 'Use stubs for Connpass'
+
     describe '#search' do
       subject { Statistics::Client::Connpass.new.search(keyword: 'coderdojo') }
 
@@ -32,6 +32,8 @@
   end
 
   context 'Doorkeeper' do
+    include_context 'Use stubs for Doorkeeper'
+
     describe '#search' do
       subject { Statistics::Client::Doorkeeper.new.search(keyword: 'coderdojo') }
 
@@ -54,4 +56,19 @@
       end
     end
   end
+
+  context 'Facebook' do
+    include_context 'Use stubs for Facebook'
+
+    describe '#fetch_events' do
+      subject { Statistics::Client::Facebook.new.fetch_events(group_id: 123451234512345) }
+
+      it do
+        expect(subject).to be_instance_of(Array)
+        expect(subject.size).to eq 1
+        expect(subject.first['id']).to eq '125500978166443'
+        expect(subject.first.dig('owner', 'id')).to eq '123451234512345'
+      end
+    end
+  end
 end