diff --git a/db/migrate/20180306024000_create_recently_events.rb b/db/migrate/20180306024000_create_recently_events.rb new file mode 100644 index 000000000..0f06a8f11 --- /dev/null +++ b/db/migrate/20180306024000_create_recently_events.rb @@ -0,0 +1,13 @@ +class CreateRecentlyEvents < ActiveRecord::Migration[5.1] + def change + create_table :recently_events do |t| + t.integer :dojo_event_service_id, null: false + t.string :event_id, null: false + t.string :event_url, null: false + t.datetime :event_at, null: false + + t.index ["dojo_event_service_id"], name: "index_recently_events_on_dojo_event_service_id" + t.index ["event_at"], name: "index_recently_events_on_event_at" + end + end +end diff --git a/db/schema.rb b/db/schema.rb index faab8f699..f2d2dbc1d 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20180117164209) do +ActiveRecord::Schema.define(version: 20180306024000) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -61,6 +61,15 @@ t.index ["region"], name: "index_prefectures_on_region" end + create_table "recently_events", force: :cascade do |t| + t.integer "dojo_event_service_id", null: false + t.string "event_id", null: false + t.string "event_url", null: false + t.datetime "event_at", null: false + t.index ["dojo_event_service_id"], name: "index_recently_events_on_dojo_event_service_id" + t.index ["event_at"], name: "index_recently_events_on_event_at" + end + add_foreign_key "dojo_event_services", "dojos" add_foreign_key "event_histories", "dojos" end diff --git a/lib/eventservice.rb b/lib/eventservice.rb new file mode 100644 index 000000000..098735119 --- /dev/null +++ b/lib/eventservice.rb @@ -0,0 +1,3 @@ +module EventService; end + +require_relative 'eventservice/providers' diff --git a/lib/statistics/providers.rb b/lib/eventservice/providers.rb similarity index 90% rename from lib/statistics/providers.rb rename to lib/eventservice/providers.rb index 7f5069eef..1f923b608 100644 --- a/lib/statistics/providers.rb +++ b/lib/eventservice/providers.rb @@ -1,4 +1,4 @@ -module Statistics +module EventService module Providers end end diff --git a/lib/eventservice/providers/connpass.rb b/lib/eventservice/providers/connpass.rb new file mode 100644 index 000000000..14dd10808 --- /dev/null +++ b/lib/eventservice/providers/connpass.rb @@ -0,0 +1,42 @@ +module EventService + module Providers + class Connpass + ENDPOINT = 'https://connpass.com/api/v1'.freeze + + def initialize + @client = Statistics::Client.new(ENDPOINT) + end + + def search(keyword:) + @client.get('event/', { keyword: keyword, count: 100 }) + end + + def fetch_events(series_id:, yyyymm: nil, yyyymmdd: nil) + params = { + series_id: series_id, + start: 1, + count: 100 + } + params[:ym] = yyyymm if yyyymm + params[:ymd] = yyyymmdd if yyyymmdd + events = [] + + loop do + part = @client.get('event/', params) + + break if part['results_returned'].zero? + + events.push(*part.fetch('events')) + + break if part.size < params[:count] + + break if params[:start] + params[:count] > part['results_available'] + + params[:start] += params[:count] + end + + events + end + end + end +end diff --git a/lib/eventservice/providers/doorkeeper.rb b/lib/eventservice/providers/doorkeeper.rb new file mode 100644 index 000000000..285b184c1 --- /dev/null +++ b/lib/eventservice/providers/doorkeeper.rb @@ -0,0 +1,51 @@ +module EventService + module Providers + class Doorkeeper + ENDPOINT = 'https://api.doorkeeper.jp'.freeze + + def initialize + @client = Statistics::Client.new(ENDPOINT) do |c| + c.authorization(:Bearer, ENV.fetch('DOORKEEPER_API_TOKEN')) + end + @default_since = Time.zone.parse('2010-07-01') + @default_until = Time.zone.now.end_of_day + end + + def search(keyword:) + @client.get('events', q: keyword, since: @default_since, expand: 'group') + end + + def fetch_events(group_id:, since_at: @default_since, until_at: @default_until) + begin + params = { + page: 1, + since: since_at, + until: until_at + } + events = [] + + loop do + part = @client.get("groups/#{group_id}/events", params) + + break if part.size.zero? + + events.push(*part.map { |e| e['event'] }) + + break if part.size < 25 # 25 items / 1 request + + params[:page] += 1 + end + + events + rescue Faraday::ClientError => e + raise e unless e.response[:status] == 429 + + puts 'API rate limit exceeded.' + puts "This task will retry in 60 seconds from now(#{Time.zone.now})." + sleep 60 + retry + end + end + end + end +end diff --git a/lib/eventservice/providers/facebook.rb b/lib/eventservice/providers/facebook.rb new file mode 100644 index 000000000..ff7d72ff3 --- /dev/null +++ b/lib/eventservice/providers/facebook.rb @@ -0,0 +1,34 @@ +module EventService + module Providers + class Facebook + class_attribute :access_token + + def initialize + @client = Koala::Facebook::API.new(self.access_token) + end + + def fetch_events(group_id:, since_at: nil, until_at: nil) + params = { + fields: %i(attending_count start_time owner), + limit: 100 + }.tap do |h| + # @note FacebookのGraph APIはPDTがタイムゾーンとなっており、 + # JST<->PDTのオフセット8時間を追加した時刻をパラメータとする必要がある + # @see https://github.com/coderdojo-japan/coderdojo.jp/pull/182#discussion_r148935458 + h[:since] = since_at.since(8.hours).to_i if since_at + h[:until] = until_at.since(8.hours).to_i if until_at + end + + events = [] + + collection = @client.get_object("#{group_id}/events", params) + events.push(*collection.to_a) + while !collection.empty? && collection.paging['next'] do + events.push(*collection.next_page.to_a) + end + + events + end + end + end +end diff --git a/lib/eventservice/providers/static_yaml.rb b/lib/eventservice/providers/static_yaml.rb new file mode 100644 index 000000000..7a458d97f --- /dev/null +++ b/lib/eventservice/providers/static_yaml.rb @@ -0,0 +1,11 @@ +module EventService + module Providers + class StaticYaml + YAML_FILE = Rails.root.join('db', 'static_event_histories.yaml') + + def fetch_events + YAML.load_file(YAML_FILE) || [] + end + end + end +end diff --git a/lib/statistics.rb b/lib/statistics.rb index e543557ac..f4c3d90aa 100644 --- a/lib/statistics.rb +++ b/lib/statistics.rb @@ -1,6 +1,6 @@ module Statistics; end require_relative 'statistics/client' -require_relative 'statistics/providers' require_relative 'statistics/tasks' require_relative 'statistics/aggregation' +require_relative 'eventservice' diff --git a/lib/statistics/providers/connpass.rb b/lib/statistics/providers/connpass.rb deleted file mode 100644 index 9689ad425..000000000 --- a/lib/statistics/providers/connpass.rb +++ /dev/null @@ -1,42 +0,0 @@ -module Statistics - module Providers - class Connpass - ENDPOINT = 'https://connpass.com/api/v1'.freeze - - def initialize - @client = Statistics::Client.new(ENDPOINT) - end - - def search(keyword:) - @client.get('event/', { keyword: keyword, count: 100 }) - end - - def fetch_events(series_id:, yyyymm: nil, yyyymmdd: nil) - params = { - series_id: series_id, - start: 1, - count: 100 - } - params[:ym] = yyyymm if yyyymm - params[:ymd] = yyyymmdd if yyyymmdd - events = [] - - loop do - part = @client.get('event/', params) - - break if part['results_returned'].zero? - - events.push(*part.fetch('events')) - - break if part.size < params[:count] - - break if params[:start] + params[:count] > part['results_available'] - - params[:start] += params[:count] - end - - events - end - end - end -end diff --git a/lib/statistics/providers/doorkeeper.rb b/lib/statistics/providers/doorkeeper.rb deleted file mode 100644 index 557dea9dc..000000000 --- a/lib/statistics/providers/doorkeeper.rb +++ /dev/null @@ -1,51 +0,0 @@ -module Statistics - module Providers - class Doorkeeper - ENDPOINT = 'https://api.doorkeeper.jp'.freeze - - def initialize - @client = Statistics::Client.new(ENDPOINT) do |c| - c.authorization(:Bearer, ENV.fetch('DOORKEEPER_API_TOKEN')) - end - @default_since = Time.zone.parse('2010-07-01') - @default_until = Time.zone.now.end_of_day - end - - def search(keyword:) - @client.get('events', q: keyword, since: @default_since, expand: 'group') - end - - def fetch_events(group_id:, since_at: @default_since, until_at: @default_until) - begin - params = { - page: 1, - since: since_at, - until: until_at - } - events = [] - - loop do - part = @client.get("groups/#{group_id}/events", params) - - break if part.size.zero? - - events.push(*part.map { |e| e['event'] }) - - break if part.size < 25 # 25 items / 1 request - - params[:page] += 1 - end - - events - rescue Faraday::ClientError => e - raise e unless e.response[:status] == 429 - - puts 'API rate limit exceeded.' - puts "This task will retry in 60 seconds from now(#{Time.zone.now})." - sleep 60 - retry - end - end - end - end -end diff --git a/lib/statistics/providers/facebook.rb b/lib/statistics/providers/facebook.rb deleted file mode 100644 index e7b2eb7b2..000000000 --- a/lib/statistics/providers/facebook.rb +++ /dev/null @@ -1,34 +0,0 @@ -module Statistics - module Providers - class Facebook - class_attribute :access_token - - def initialize - @client = Koala::Facebook::API.new(self.access_token) - end - - def fetch_events(group_id:, since_at: nil, until_at: nil) - params = { - fields: %i(attending_count start_time owner), - limit: 100 - }.tap do |h| - # @note FacebookのGraph APIはPDTがタイムゾーンとなっており、 - # JST<->PDTのオフセット8時間を追加した時刻をパラメータとする必要がある - # @see https://github.com/coderdojo-japan/coderdojo.jp/pull/182#discussion_r148935458 - h[:since] = since_at.since(8.hours).to_i if since_at - h[:until] = until_at.since(8.hours).to_i if until_at - end - - events = [] - - collection = @client.get_object("#{group_id}/events", params) - events.push(*collection.to_a) - while !collection.empty? && collection.paging['next'] do - events.push(*collection.next_page.to_a) - end - - events - end - end - end -end diff --git a/lib/statistics/providers/static_yaml.rb b/lib/statistics/providers/static_yaml.rb deleted file mode 100644 index 6c496a3d2..000000000 --- a/lib/statistics/providers/static_yaml.rb +++ /dev/null @@ -1,11 +0,0 @@ -module Statistics - module Providers - class StaticYaml - YAML_FILE = Rails.root.join('db', 'static_event_histories.yaml') - - def fetch_events - YAML.load_file(YAML_FILE) || [] - end - end - end -end diff --git a/lib/statistics/tasks/connpass.rb b/lib/statistics/tasks/connpass.rb index 13cc2f1dc..acbf01319 100644 --- a/lib/statistics/tasks/connpass.rb +++ b/lib/statistics/tasks/connpass.rb @@ -6,7 +6,7 @@ def self.delete_event_histories(period) end def initialize(dojos, date, weekly) - @client = Providers::Connpass.new + @client = EventService::Providers::Connpass.new @dojos = dojos @params = build_params(date, weekly) end diff --git a/lib/statistics/tasks/doorkeeper.rb b/lib/statistics/tasks/doorkeeper.rb index 7997c4312..a1c246145 100644 --- a/lib/statistics/tasks/doorkeeper.rb +++ b/lib/statistics/tasks/doorkeeper.rb @@ -6,7 +6,7 @@ def self.delete_event_histories(period) end def initialize(dojos, date, weekly) - @client = Providers::Doorkeeper.new + @client = EventService::Providers::Doorkeeper.new @dojos = dojos @params = build_params(date, weekly) end diff --git a/lib/statistics/tasks/facebook.rb b/lib/statistics/tasks/facebook.rb index 6c6ee2b06..1980fa046 100644 --- a/lib/statistics/tasks/facebook.rb +++ b/lib/statistics/tasks/facebook.rb @@ -6,7 +6,7 @@ def self.delete_event_histories(period) end def initialize(dojos, date, weekly) - @client = Providers::Facebook.new + @client = EventService::Providers::Facebook.new @dojos = dojos @params = build_params(date, weekly) end diff --git a/lib/statistics/tasks/static_yaml.rb b/lib/statistics/tasks/static_yaml.rb index a783d4b52..304fab304 100644 --- a/lib/statistics/tasks/static_yaml.rb +++ b/lib/statistics/tasks/static_yaml.rb @@ -6,7 +6,7 @@ def self.delete_event_histories(_period) end def initialize(dojos, _date, _weekly) - @client = Providers::StaticYaml.new + @client = EventService::Providers::StaticYaml.new @dojos = dojos end diff --git a/lib/tasks/statistics.rake b/lib/tasks/statistics.rake index b1611aded..4d9a952b7 100644 --- a/lib/tasks/statistics.rake +++ b/lib/tasks/statistics.rake @@ -3,7 +3,7 @@ require_relative '../statistics.rb' namespace :statistics do desc '月次/週次のイベント履歴を集計します' task :aggregation, [:from, :to] => :environment do |tasks, args| - Statistics::Providers::Facebook.access_token = Koala::Facebook::OAuth.new(ENV['FACEBOOK_APP_ID'], ENV['FACEBOOK_APP_SECRET']).get_app_access_token + EventService::Providers::Facebook.access_token = Koala::Facebook::OAuth.new(ENV['FACEBOOK_APP_ID'], ENV['FACEBOOK_APP_SECRET']).get_app_access_token Statistics::Aggregation.new(args).run end @@ -14,9 +14,9 @@ namespace :statistics do require 'pp' puts 'Searching Connpass' - pp Statistics::Providers::Connpass.new.search(keyword: args[:keyword]) + pp EventService::Providers::Connpass.new.search(keyword: args[:keyword]) puts 'Searching Doorkeeper' - pp Statistics::Providers::Doorkeeper.new.search(keyword: args[:keyword]) + pp EventService::Providers::Doorkeeper.new.search(keyword: args[:keyword]) end end diff --git a/spec/lib/statistics/providers/connpass_spec.rb b/spec/lib/eventservice/providers/connpass_spec.rb similarity index 92% rename from spec/lib/statistics/providers/connpass_spec.rb rename to spec/lib/eventservice/providers/connpass_spec.rb index 6dace0f18..e2649e2dd 100644 --- a/spec/lib/statistics/providers/connpass_spec.rb +++ b/spec/lib/eventservice/providers/connpass_spec.rb @@ -1,7 +1,7 @@ require 'rails_helper' -require 'statistics' +require 'eventservice' -RSpec.describe Statistics::Providers::Connpass do +RSpec.describe EventService::Providers::Connpass do include_context 'Use stubs for Connpass' describe '#search' do diff --git a/spec/lib/statistics/providers/doorkeeper_spec.rb b/spec/lib/eventservice/providers/doorkeeper_spec.rb similarity index 89% rename from spec/lib/statistics/providers/doorkeeper_spec.rb rename to spec/lib/eventservice/providers/doorkeeper_spec.rb index f53bce364..35a1dcc79 100644 --- a/spec/lib/statistics/providers/doorkeeper_spec.rb +++ b/spec/lib/eventservice/providers/doorkeeper_spec.rb @@ -1,7 +1,7 @@ require 'rails_helper' -require 'statistics' +require 'eventservice' -RSpec.describe Statistics::Providers::Doorkeeper do +RSpec.describe EventService::Providers::Doorkeeper do include_context 'Use stubs for Doorkeeper' describe '#search' do diff --git a/spec/lib/statistics/providers/facebook_spec.rb b/spec/lib/eventservice/providers/facebook_spec.rb similarity index 84% rename from spec/lib/statistics/providers/facebook_spec.rb rename to spec/lib/eventservice/providers/facebook_spec.rb index 9ab64af3f..3e1352900 100644 --- a/spec/lib/statistics/providers/facebook_spec.rb +++ b/spec/lib/eventservice/providers/facebook_spec.rb @@ -1,7 +1,7 @@ require 'rails_helper' -require 'statistics' +require 'eventservice' -RSpec.describe Statistics::Providers::Facebook do +RSpec.describe EventService::Providers::Facebook do include_context 'Use stubs for Facebook' describe '#fetch_events' do