From 34d54dd191415494dca7220d59ba4512adb0dbc0 Mon Sep 17 00:00:00 2001 From: AnaTofuZ Date: Tue, 20 Mar 2018 10:58:24 +0900 Subject: [PATCH 01/10] first commit From 2cf308d1f47341934b42f1ded657a499dbca5e40 Mon Sep 17 00:00:00 2001 From: AnaTofuZ Date: Tue, 20 Mar 2018 14:56:53 +0900 Subject: [PATCH 02/10] =?UTF-8?q?=E3=82=B3=E3=83=94=E3=83=BC=E3=81=A8?= =?UTF-8?q?=E3=83=A1=E3=82=BD=E3=83=83=E3=83=89=E7=AD=89=E3=81=AE=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Statisticsの資産を使いたかった為にコピーして大まかなメソッドをリネームし た --- lib/upcoming/aggregation.rb | 161 ++++++++++++++++++++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 lib/upcoming/aggregation.rb diff --git a/lib/upcoming/aggregation.rb b/lib/upcoming/aggregation.rb new file mode 100644 index 000000000..e40648d77 --- /dev/null +++ b/lib/upcoming/aggregation.rb @@ -0,0 +1,161 @@ +module Upcoming + class Aggregation + def initialize(args) + @from = TimeWithZone.beginning_of_week + @to = TimeWithZone.end_of_month + @dojos = fetch_dojos + end + + def run + "Upcoming::Aggregation::#{@mode.camelize}".constantize.new(@dojos, @from, @to).run + end + + private + + + def fetch_dojos + { + externals: find_dojos_by(DojoEventService::EXTERNAL_SERVICES), + internals: find_dojos_by(DojoEventService::INTERNAL_SERVICES) + } + end + + def find_dojos_by(services) + services.each.with_object({}) do |name, hash| + hash[name] = Dojo.eager_load(:dojo_event_services).where(dojo_event_services: { name: name }).to_a + end + end + + class Base + def initialize(dojos, from, to) + @externals = dojos[:externals] + @internals = dojos[:internals] + @list = build_list(from, to) + @from = from + @to = to + end + + def run + with_notifying do + delete_event_histories + execute + execute_once + end + end + + private + + def with_notifying + yield + Notifier.notify_success(date_format(@from), date_format(@to)) + rescue => e + Notifier.notify_failure(date_format(@from), date_format(@to), e) + end + + ## TODO: modofy methods + def delete_event_histories + (@externals.keys + @internals.keys).each do |kind| + "Upcoming::Tasks::#{kind.to_s.camelize}".constantize.delete_event_histories(@from..@to) + end + end + + def execute + raise NotImplementedError.new("You must implement #{self.class}##{__method__}") + end + + def execute_once + @internals.each do |kind, list| + "Upcoming::Tasks::#{kind.to_s.camelize}".constantize.new(list, nil, nil).run + end + end + + def build_list(_from, _to) + raise NotImplementedError.new("You must implement #{self.class}##{__method__}") + end + + def date_format(_date) + raise NotImplementedError.new("You must implement #{self.class}##{__method__}") + end + end + + class Weekly < Base + private + + def execute + @list.each do |date| + puts "Aggregate for #{date_format(date)}~#{date_format(date.end_of_week)}" + + @externals.each do |kind, list| + "Statistics::Tasks::#{kind.to_s.camelize}".constantize.new(list, date, true).run + end + end + end + + def build_list(from, to) + loop.with_object([from]) do |_, list| + nw = list.last.next_week + raise StopIteration if nw > to + list << nw + end + end + + def date_format(date) + date.strftime('%Y/%m/%d') + end + end + + class Monthly < Base + private + + def execute + @list.each do |date| + puts "Aggregate for #{date_format(date)}" + + @externals.each do |kind, list| + "Statistics::Tasks::#{kind.to_s.camelize}".constantize.new(list, date, false).run + end + end + end + + def build_list(from, to) + loop.with_object([from]) do |_, list| + nm = list.last.next_month + raise StopIteration if nm > to + list << nm + end + end + + def date_format(date) + date.strftime('%Y/%m') + end + end + + class Notifier + class << self + def notify_success(from, to) + notify("#{from}~#{to}のイベント履歴の集計を行いました") + end + + def notify_failure(from, to, exception) + notify("#{from}~#{to}のイベント履歴の集計でエラーが発生しました\n#{exception.message}\n#{exception.backtrace.join("\n")}") + end + + private + + def idobata_hook_url + return @idobata_hook_url if defined?(@idobata_hook_url) + @idobata_hook_url = ENV['IDOBATA_HOOK_URL'] + end + + def notifierable? + idobata_hook_url.present? + end + + def notify(msg) + $stdout.puts msg + puts `curl --data-urlencode "source=#{msg}" -s #{idobata_hook_url} -o /dev/null -w "idobata: %{http_code}"` if notifierable? + end + end + end + end +end From d984fe57a7bfdaa732e39245448c53dc5d14a482 Mon Sep 17 00:00:00 2001 From: AnaTofuZ Date: Tue, 20 Mar 2018 14:57:59 +0900 Subject: [PATCH 03/10] =?UTF-8?q?Statistics=E3=81=8B=E3=82=89=E3=82=B3?= =?UTF-8?q?=E3=83=94=E3=83=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit そのままrun系のメソッドが使えそうな気持ちだった為コピーしてきた --- lib/upcoming/tasks/connpass.rb | 51 +++++++++++++++++++++++++++++++ lib/upcoming/tasks/doorkeeper.rb | 50 ++++++++++++++++++++++++++++++ lib/upcoming/tasks/facebook.rb | 50 ++++++++++++++++++++++++++++++ lib/upcoming/tasks/static_yaml.rb | 33 ++++++++++++++++++++ 4 files changed, 184 insertions(+) create mode 100644 lib/upcoming/tasks/connpass.rb create mode 100644 lib/upcoming/tasks/doorkeeper.rb create mode 100644 lib/upcoming/tasks/facebook.rb create mode 100644 lib/upcoming/tasks/static_yaml.rb diff --git a/lib/upcoming/tasks/connpass.rb b/lib/upcoming/tasks/connpass.rb new file mode 100644 index 000000000..497b07627 --- /dev/null +++ b/lib/upcoming/tasks/connpass.rb @@ -0,0 +1,51 @@ +module Upcoming + module Tasks + class Connpass + def self.delete_event_histories(period) + EventHistory.for(:connpass).within(period).delete_all + end + + def initialize(dojos, date, weekly) + @client = EventService::Providers::Connpass.new + @dojos = dojos + @params = build_params(date, weekly) + end + + def run + @dojos.each do |dojo| + dojo.dojo_event_services.for(:connpass).each do |dojo_event_service| + @client.fetch_events(@params.merge(series_id: dojo_event_service.group_id)).each do |e| + next unless e.dig('series', 'id').to_s == dojo_event_service.group_id + + binding.pry + UpcomingEvent.create!(dojo_event_service_id: e.['id'], + event_id: e['event_id'], + event_url: e['event_url'], + event_at: Time.zone.parse(e['started_at'])) + end + end + end + end + + private + + def build_params(date, weekly) + if weekly + week_days = loop.with_object([date]) { |_, list| + nd = list.last.next_day + raise StopIteration if nd > date.end_of_week + list << nd + }.map { |date| date.strftime('%Y%m%d') } + + { + yyyymmdd: week_days.join(',') + } + else + { + yyyymm: "#{date.year}#{date.month}" + } + end + end + end + end +end diff --git a/lib/upcoming/tasks/doorkeeper.rb b/lib/upcoming/tasks/doorkeeper.rb new file mode 100644 index 000000000..a1c246145 --- /dev/null +++ b/lib/upcoming/tasks/doorkeeper.rb @@ -0,0 +1,50 @@ +module Statistics + module Tasks + class Doorkeeper + def self.delete_event_histories(period) + EventHistory.for(:doorkeeper).within(period).delete_all + end + + def initialize(dojos, date, weekly) + @client = EventService::Providers::Doorkeeper.new + @dojos = dojos + @params = build_params(date, weekly) + end + + def run + @dojos.each do |dojo| + dojo.dojo_event_services.for(:doorkeeper).each do |dojo_event_service| + @client.fetch_events(@params.merge(group_id: dojo_event_service.group_id)).each do |e| + next unless e['group'].to_s == dojo_event_service.group_id + + EventHistory.create!(dojo_id: dojo.id, + dojo_name: dojo.name, + service_name: dojo_event_service.name, + service_group_id: dojo_event_service.group_id, + event_id: e['id'], + event_url: e['public_url'], + participants: e['participants'], + evented_at: Time.zone.parse(e['starts_at'])) + end + end + end + end + + private + + def build_params(date, weekly) + if weekly + { + since_at: date.beginning_of_week, + until_at: date.end_of_week + } + else + { + since_at: date.beginning_of_month, + until_at: date.end_of_month + } + end + end + end + end +end diff --git a/lib/upcoming/tasks/facebook.rb b/lib/upcoming/tasks/facebook.rb new file mode 100644 index 000000000..1980fa046 --- /dev/null +++ b/lib/upcoming/tasks/facebook.rb @@ -0,0 +1,50 @@ +module Statistics + module Tasks + class Facebook + def self.delete_event_histories(period) + EventHistory.for(:facebook).within(period).delete_all + end + + def initialize(dojos, date, weekly) + @client = EventService::Providers::Facebook.new + @dojos = dojos + @params = build_params(date, weekly) + end + + def run + @dojos.each do |dojo| + dojo.dojo_event_services.for(:facebook).each do |dojo_event_service| + @client.fetch_events(@params.merge(group_id: dojo_event_service.group_id)).each do |e| + next unless e.dig('owner', 'id') == dojo_event_service.group_id + + EventHistory.create!(dojo_id: dojo.id, + dojo_name: dojo.name, + service_name: dojo_event_service.name, + service_group_id: dojo_event_service.group_id, + event_id: e['id'], + event_url: "https://www.facebook.com/events/#{e['id']}", + participants: e['attending_count'], + evented_at: Time.zone.parse(e['start_time'])) + end + end + end + end + + private + + def build_params(date, weekly) + if weekly + { + since_at: date.beginning_of_week, + until_at: date.end_of_week + } + else + { + since_at: date.beginning_of_month, + until_at: date.end_of_month + } + end + end + end + end +end diff --git a/lib/upcoming/tasks/static_yaml.rb b/lib/upcoming/tasks/static_yaml.rb new file mode 100644 index 000000000..304fab304 --- /dev/null +++ b/lib/upcoming/tasks/static_yaml.rb @@ -0,0 +1,33 @@ +module Statistics + module Tasks + class StaticYaml + def self.delete_event_histories(_period) + EventHistory.for(:static_yaml).delete_all + end + + def initialize(dojos, _date, _weekly) + @client = EventService::Providers::StaticYaml.new + @dojos = dojos + end + + def run + dojos_hash = @dojos.index_by(&:id) + @client.fetch_events.each do |e| + dojo = dojos_hash[e['dojo_id'].to_i] + next unless dojo + + evented_at = Time.zone.parse(e['evented_at']) + event_id = "#{SecureRandom.uuid}" + + EventHistory.create!(dojo_id: dojo.id, + dojo_name: dojo.name, + service_name: self.class.name.demodulize.underscore, + event_id: event_id, + event_url: "https://dummy.url/#{event_id}", + participants: e['participants'], + evented_at: evented_at) + end + end + end + end +end From 206746633e421d2c36ca35a77a7bd26eeb5a7486 Mon Sep 17 00:00:00 2001 From: AnaTofuZ Date: Tue, 20 Mar 2018 15:37:35 +0900 Subject: [PATCH 04/10] remove Weekly class --- lib/upcoming/aggregation.rb | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/lib/upcoming/aggregation.rb b/lib/upcoming/aggregation.rb index e40648d77..c62320cee 100644 --- a/lib/upcoming/aggregation.rb +++ b/lib/upcoming/aggregation.rb @@ -78,31 +78,6 @@ def date_format(_date) end end - class Weekly < Base - private - - def execute - @list.each do |date| - puts "Aggregate for #{date_format(date)}~#{date_format(date.end_of_week)}" - - @externals.each do |kind, list| - "Statistics::Tasks::#{kind.to_s.camelize}".constantize.new(list, date, true).run - end - end - end - - def build_list(from, to) - loop.with_object([from]) do |_, list| - nw = list.last.next_week - raise StopIteration if nw > to - list << nw - end - end - - def date_format(date) - date.strftime('%Y/%m/%d') - end - end class Monthly < Base private From f641b1368b8b1a670a028ac0a65af6621ae4b61d Mon Sep 17 00:00:00 2001 From: AnaTofuZ Date: Thu, 22 Mar 2018 10:42:30 +0900 Subject: [PATCH 05/10] =?UTF-8?q?=E6=96=87=E8=A8=80=E3=81=AE=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/upcoming/aggregation.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/upcoming/aggregation.rb b/lib/upcoming/aggregation.rb index c62320cee..0e5e109d4 100644 --- a/lib/upcoming/aggregation.rb +++ b/lib/upcoming/aggregation.rb @@ -108,11 +108,11 @@ def date_format(date) class Notifier class << self def notify_success(from, to) - notify("#{from}~#{to}のイベント履歴の集計を行いました") + notify("#{from}~#{to}のイベント登録を行いました") end def notify_failure(from, to, exception) - notify("#{from}~#{to}のイベント履歴の集計でエラーが発生しました\n#{exception.message}\n#{exception.backtrace.join("\n")}") + notify("#{from}~#{to}のイベント登録でエラーが発生しました\n#{exception.message}\n#{exception.backtrace.join("\n")}") end private From fefce950f7c1ba63fc509cec4ce8a3fc1a32238f Mon Sep 17 00:00:00 2001 From: AnaTofuZ Date: Thu, 22 Mar 2018 14:22:04 +0900 Subject: [PATCH 06/10] =?UTF-8?q?UpcomingEvent=E3=82=92=E4=BF=9D=E5=AD=98?= =?UTF-8?q?=E3=81=99=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB=E4=BF=AE=E6=AD=A3?= =?UTF-8?q?=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit connpassで作成したフォーマットに合わせてfacebookを修正した --- lib/upcoming/tasks/connpass.rb | 3 +-- lib/upcoming/tasks/facebook.rb | 10 +++------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/lib/upcoming/tasks/connpass.rb b/lib/upcoming/tasks/connpass.rb index 497b07627..6c34b3946 100644 --- a/lib/upcoming/tasks/connpass.rb +++ b/lib/upcoming/tasks/connpass.rb @@ -17,8 +17,7 @@ def run @client.fetch_events(@params.merge(series_id: dojo_event_service.group_id)).each do |e| next unless e.dig('series', 'id').to_s == dojo_event_service.group_id - binding.pry - UpcomingEvent.create!(dojo_event_service_id: e.['id'], + UpcomingEvent.create!(dojo_event_service_id: e['id'], event_id: e['event_id'], event_url: e['event_url'], event_at: Time.zone.parse(e['started_at'])) diff --git a/lib/upcoming/tasks/facebook.rb b/lib/upcoming/tasks/facebook.rb index 1980fa046..94fd6d5df 100644 --- a/lib/upcoming/tasks/facebook.rb +++ b/lib/upcoming/tasks/facebook.rb @@ -2,7 +2,7 @@ module Statistics module Tasks class Facebook def self.delete_event_histories(period) - EventHistory.for(:facebook).within(period).delete_all + UpcomingEvent.for(:facebook).within(period).delete_all end def initialize(dojos, date, weekly) @@ -17,14 +17,10 @@ def run @client.fetch_events(@params.merge(group_id: dojo_event_service.group_id)).each do |e| next unless e.dig('owner', 'id') == dojo_event_service.group_id - EventHistory.create!(dojo_id: dojo.id, - dojo_name: dojo.name, - service_name: dojo_event_service.name, - service_group_id: dojo_event_service.group_id, + UpcomingEvent.create!(dojo_event_service_id: e['id'], event_id: e['id'], event_url: "https://www.facebook.com/events/#{e['id']}", - participants: e['attending_count'], - evented_at: Time.zone.parse(e['start_time'])) + event_at: Time.zone.parse(e['start_time'])) end end end From 3e83ceb7e64a01fb8a1401e5692c3039fc4c6b31 Mon Sep 17 00:00:00 2001 From: AnaTofuZ Date: Thu, 22 Mar 2018 14:31:03 +0900 Subject: [PATCH 07/10] =?UTF-8?q?UpcomingEvent=20model=E3=82=92=E5=AE=9A?= =?UTF-8?q?=E7=BE=A9=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit schemeを定義していたが利用するmodelを定義していなかった為 event_historyを参考にmodelを定義した。 --- app/models/upcoming_event.rb | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 app/models/upcoming_event.rb diff --git a/app/models/upcoming_event.rb b/app/models/upcoming_event.rb new file mode 100644 index 000000000..b795dcdf4 --- /dev/null +++ b/app/models/upcoming_event.rb @@ -0,0 +1,9 @@ +class UpcomingEvent < ApplicationRecord + belongs_to :dojo + + validates :dojo_event_service_id, presence: true + validates :event_id, presence: true + validates :event_url,presence: true + + validates :event_at,presence: true +end \ No newline at end of file From 792ea89e924e091932532f20c219f81d9fa3d9ba Mon Sep 17 00:00:00 2001 From: AnaTofuZ Date: Thu, 22 Mar 2018 14:57:29 +0900 Subject: [PATCH 08/10] =?UTF-8?q?=E5=90=8D=E5=89=8D=E7=A9=BA=E9=96=93?= =?UTF-8?q?=E3=81=A8=E5=91=BC=E3=81=B3=E5=87=BA=E3=81=97=E3=83=A1=E3=82=BD?= =?UTF-8?q?=E3=83=83=E3=83=89=E3=81=AE=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/upcoming/tasks/connpass.rb | 2 +- lib/upcoming/tasks/doorkeeper.rb | 12 ++++-------- lib/upcoming/tasks/facebook.rb | 2 +- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/lib/upcoming/tasks/connpass.rb b/lib/upcoming/tasks/connpass.rb index 6c34b3946..7dd2f1610 100644 --- a/lib/upcoming/tasks/connpass.rb +++ b/lib/upcoming/tasks/connpass.rb @@ -2,7 +2,7 @@ module Upcoming module Tasks class Connpass def self.delete_event_histories(period) - EventHistory.for(:connpass).within(period).delete_all + UpcomingEvent.for(:connpass).within(period).delete_all end def initialize(dojos, date, weekly) diff --git a/lib/upcoming/tasks/doorkeeper.rb b/lib/upcoming/tasks/doorkeeper.rb index a1c246145..cee45975b 100644 --- a/lib/upcoming/tasks/doorkeeper.rb +++ b/lib/upcoming/tasks/doorkeeper.rb @@ -1,8 +1,8 @@ -module Statistics +module Upcoming module Tasks class Doorkeeper def self.delete_event_histories(period) - EventHistory.for(:doorkeeper).within(period).delete_all + UpcomingEvent.for(:doorkeeper).within(period).delete_all end def initialize(dojos, date, weekly) @@ -17,14 +17,10 @@ def run @client.fetch_events(@params.merge(group_id: dojo_event_service.group_id)).each do |e| next unless e['group'].to_s == dojo_event_service.group_id - EventHistory.create!(dojo_id: dojo.id, - dojo_name: dojo.name, - service_name: dojo_event_service.name, - service_group_id: dojo_event_service.group_id, + UpcomingEvent.create!(dojo_event_service_id: e['id'], event_id: e['id'], event_url: e['public_url'], - participants: e['participants'], - evented_at: Time.zone.parse(e['starts_at'])) + event_at: Time.zone.parse(e['starts_at'])) end end end diff --git a/lib/upcoming/tasks/facebook.rb b/lib/upcoming/tasks/facebook.rb index 94fd6d5df..6986a4846 100644 --- a/lib/upcoming/tasks/facebook.rb +++ b/lib/upcoming/tasks/facebook.rb @@ -1,4 +1,4 @@ -module Statistics +module Upcoming module Tasks class Facebook def self.delete_event_histories(period) From 261ab2349a18252160011e605dbab609497a77d7 Mon Sep 17 00:00:00 2001 From: aokabin Date: Tue, 26 Jun 2018 17:16:39 +0900 Subject: [PATCH 09/10] =?UTF-8?q?upcoming/aggregation=E3=81=AE=E3=82=A2?= =?UTF-8?q?=E3=83=83=E3=83=97=E3=83=87=E3=83=BC=E3=83=88=E3=81=8A=E3=82=88?= =?UTF-8?q?=E3=81=B3upcoming=E3=82=B9=E3=82=AF=E3=83=AA=E3=83=97=E3=83=88?= =?UTF-8?q?=E3=81=AE=E4=BD=9C=E6=88=90=E3=80=81rake=E3=82=BF=E3=82=B9?= =?UTF-8?q?=E3=82=AF=E3=81=AE=E4=BD=9C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/upcoming_event.rb | 2 ++ lib/tasks/upcoming.rake | 9 +++++++++ lib/upcoming.rb | 5 +++++ lib/upcoming/aggregation.rb | 17 +++++++---------- lib/upcoming/tasks/connpass.rb | 26 ++++++-------------------- lib/upcoming/tasks/doorkeeper.rb | 25 +++++++++---------------- lib/upcoming/tasks/facebook.rb | 25 +++++++++---------------- lib/upcoming/tasks/static_yaml.rb | 8 ++++---- 8 files changed, 51 insertions(+), 66 deletions(-) create mode 100644 lib/tasks/upcoming.rake create mode 100644 lib/upcoming.rb diff --git a/app/models/upcoming_event.rb b/app/models/upcoming_event.rb index b795dcdf4..0218fea4b 100644 --- a/app/models/upcoming_event.rb +++ b/app/models/upcoming_event.rb @@ -6,4 +6,6 @@ class UpcomingEvent < ApplicationRecord validates :event_url,presence: true validates :event_at,presence: true + + scope :for, ->(service) { where(service_name: service) } end \ No newline at end of file diff --git a/lib/tasks/upcoming.rake b/lib/tasks/upcoming.rake new file mode 100644 index 000000000..fad877abb --- /dev/null +++ b/lib/tasks/upcoming.rake @@ -0,0 +1,9 @@ +require_relative '../upcoming.rb' + +namespace :upcoming do + desc '月次のイベント開催予定を集計します' + task :aggregation, [:from, :to] => :environment do |tasks, args| + EventService::Providers::Facebook.access_token = Koala::Facebook::OAuth.new(ENV['FACEBOOK_APP_ID'], ENV['FACEBOOK_APP_SECRET']).get_app_access_token + Upcoming::Aggregation.new(args).run + end +end diff --git a/lib/upcoming.rb b/lib/upcoming.rb new file mode 100644 index 000000000..a411e7384 --- /dev/null +++ b/lib/upcoming.rb @@ -0,0 +1,5 @@ +module Upcoming; end + +require_relative 'upcoming/tasks' +require_relative 'upcoming/aggregation' +require_relative 'event_service' diff --git a/lib/upcoming/aggregation.rb b/lib/upcoming/aggregation.rb index 0e5e109d4..ed34ca9ce 100644 --- a/lib/upcoming/aggregation.rb +++ b/lib/upcoming/aggregation.rb @@ -2,17 +2,16 @@ module Upcoming class Aggregation def initialize(args) @from = TimeWithZone.beginning_of_week - @to = TimeWithZone.end_of_month + @to = TimeWithZone.next_month @dojos = fetch_dojos end def run - "Upcoming::Aggregation::#{@mode.camelize}".constantize.new(@dojos, @from, @to).run + Upcoming::Aggregation::Monthly.new(@dojos, @from, @to).run end private - def fetch_dojos { externals: find_dojos_by(DojoEventService::EXTERNAL_SERVICES), @@ -37,7 +36,7 @@ def initialize(dojos, from, to) def run with_notifying do - delete_event_histories + delete_upcoming_event execute execute_once end @@ -52,10 +51,9 @@ def with_notifying Notifier.notify_failure(date_format(@from), date_format(@to), e) end - ## TODO: modofy methods - def delete_event_histories + def delete_upcoming_event (@externals.keys + @internals.keys).each do |kind| - "Upcoming::Tasks::#{kind.to_s.camelize}".constantize.delete_event_histories(@from..@to) + "Upcoming::Tasks::#{kind.to_s.camelize}".constantize.delete_upcoming_event end end @@ -65,7 +63,7 @@ def execute def execute_once @internals.each do |kind, list| - "Upcoming::Tasks::#{kind.to_s.camelize}".constantize.new(list, nil, nil).run + "Upcoming::Tasks::#{kind.to_s.camelize}".constantize.new(list, nil).run end end @@ -78,7 +76,6 @@ def date_format(_date) end end - class Monthly < Base private @@ -87,7 +84,7 @@ def execute puts "Aggregate for #{date_format(date)}" @externals.each do |kind, list| - "Statistics::Tasks::#{kind.to_s.camelize}".constantize.new(list, date, false).run + "Upcoming::Tasks::#{kind.to_s.camelize}".constantize.new(list, date).run end end end diff --git a/lib/upcoming/tasks/connpass.rb b/lib/upcoming/tasks/connpass.rb index 7dd2f1610..d4f669640 100644 --- a/lib/upcoming/tasks/connpass.rb +++ b/lib/upcoming/tasks/connpass.rb @@ -1,14 +1,14 @@ module Upcoming module Tasks class Connpass - def self.delete_event_histories(period) - UpcomingEvent.for(:connpass).within(period).delete_all + def self.delete_upcoming_event + UpcomingEvent.for(:connpass).delete_all end - def initialize(dojos, date, weekly) + def initialize(dojos, date) @client = EventService::Providers::Connpass.new @dojos = dojos - @params = build_params(date, weekly) + @params = build_params(date) end def run @@ -28,22 +28,8 @@ def run private - def build_params(date, weekly) - if weekly - week_days = loop.with_object([date]) { |_, list| - nd = list.last.next_day - raise StopIteration if nd > date.end_of_week - list << nd - }.map { |date| date.strftime('%Y%m%d') } - - { - yyyymmdd: week_days.join(',') - } - else - { - yyyymm: "#{date.year}#{date.month}" - } - end + def build_params(date) + { yyyymm: "#{date.year}#{date.month}" } end end end diff --git a/lib/upcoming/tasks/doorkeeper.rb b/lib/upcoming/tasks/doorkeeper.rb index cee45975b..f509c2f31 100644 --- a/lib/upcoming/tasks/doorkeeper.rb +++ b/lib/upcoming/tasks/doorkeeper.rb @@ -1,14 +1,14 @@ module Upcoming module Tasks class Doorkeeper - def self.delete_event_histories(period) - UpcomingEvent.for(:doorkeeper).within(period).delete_all + def self.delete_upcoming_event + UpcomingEvent.for(:doorkeeper).delete_all end - def initialize(dojos, date, weekly) + def initialize(dojos, date) @client = EventService::Providers::Doorkeeper.new @dojos = dojos - @params = build_params(date, weekly) + @params = build_params(date) end def run @@ -28,18 +28,11 @@ def run private - def build_params(date, weekly) - if weekly - { - since_at: date.beginning_of_week, - until_at: date.end_of_week - } - else - { - since_at: date.beginning_of_month, - until_at: date.end_of_month - } - end + def build_params(date) + { + since_at: date.beginning_of_month, + until_at: date.end_of_month + } end end end diff --git a/lib/upcoming/tasks/facebook.rb b/lib/upcoming/tasks/facebook.rb index 6986a4846..5251f77fe 100644 --- a/lib/upcoming/tasks/facebook.rb +++ b/lib/upcoming/tasks/facebook.rb @@ -1,14 +1,14 @@ module Upcoming module Tasks class Facebook - def self.delete_event_histories(period) - UpcomingEvent.for(:facebook).within(period).delete_all + def self.delete_upcoming_event + UpcomingEvent.for(:facebook).delete_all end - def initialize(dojos, date, weekly) + def initialize(dojos, date) @client = EventService::Providers::Facebook.new @dojos = dojos - @params = build_params(date, weekly) + @params = build_params(date) end def run @@ -28,18 +28,11 @@ def run private - def build_params(date, weekly) - if weekly - { - since_at: date.beginning_of_week, - until_at: date.end_of_week - } - else - { - since_at: date.beginning_of_month, - until_at: date.end_of_month - } - end + def build_params(date) + { + since_at: date.beginning_of_month, + until_at: date.end_of_month + } end end end diff --git a/lib/upcoming/tasks/static_yaml.rb b/lib/upcoming/tasks/static_yaml.rb index 304fab304..2840616f4 100644 --- a/lib/upcoming/tasks/static_yaml.rb +++ b/lib/upcoming/tasks/static_yaml.rb @@ -1,11 +1,11 @@ -module Statistics +module Upcoming module Tasks class StaticYaml - def self.delete_event_histories(_period) + def self.delete_upcoming_event EventHistory.for(:static_yaml).delete_all end - def initialize(dojos, _date, _weekly) + def initialize(dojos, _date) @client = EventService::Providers::StaticYaml.new @dojos = dojos end @@ -19,7 +19,7 @@ def run evented_at = Time.zone.parse(e['evented_at']) event_id = "#{SecureRandom.uuid}" - EventHistory.create!(dojo_id: dojo.id, + UpcomingEvent.create!(dojo_id: dojo.id, dojo_name: dojo.name, service_name: self.class.name.demodulize.underscore, event_id: event_id, From 6f5ee7d1b15d207e788c9872da5c2a80a4df2644 Mon Sep 17 00:00:00 2001 From: aokabin Date: Mon, 23 Jul 2018 23:39:14 +0900 Subject: [PATCH 10/10] =?UTF-8?q?Connpass=E3=81=A8Doorkeeper=E3=81=A8stati?= =?UTF-8?q?c=5Fyaml=E3=81=B8=E3=81=AE=E5=AF=BE=E5=BF=9C=E3=81=AFOK?= =?UTF-8?q?=E3=80=81Facebook=E3=81=AF=E3=81=BE=E3=81=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Gemfile | 1 + Gemfile.lock | 7 ++++++- app/models/dojo_event_service.rb | 2 ++ app/models/upcoming_event.rb | 3 ++- lib/upcoming/aggregation.rb | 4 ++-- lib/upcoming/tasks.rb | 9 +++++++++ lib/upcoming/tasks/connpass.rb | 2 +- lib/upcoming/tasks/doorkeeper.rb | 2 +- lib/upcoming/tasks/static_yaml.rb | 20 ++++++++++---------- logfile | 24 ++++++++++++++++++++++++ 10 files changed, 58 insertions(+), 16 deletions(-) create mode 100644 lib/upcoming/tasks.rb create mode 100644 logfile diff --git a/Gemfile b/Gemfile index fa6b8f76f..28f52f5cd 100644 --- a/Gemfile +++ b/Gemfile @@ -48,6 +48,7 @@ group :development do gem 'web-console' gem 'spring' gem 'listen' + gem 'dotenv-rails' end group :development, :test do diff --git a/Gemfile.lock b/Gemfile.lock index 6e37112db..adf5064d9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -76,6 +76,10 @@ GEM declarative (0.0.10) declarative-option (0.1.0) diff-lcs (1.3) + dotenv (2.5.0) + dotenv-rails (2.5.0) + dotenv (= 2.5.0) + railties (>= 3.2, < 6.0) erubi (1.7.1) erubis (2.7.0) ethon (0.11.0) @@ -385,6 +389,7 @@ DEPENDENCIES bootstrap-sass capybara coffee-rails + dotenv-rails faraday faraday_middleware (= 0.10) font-awesome-rails @@ -426,4 +431,4 @@ RUBY VERSION ruby 2.4.2p198 BUNDLED WITH - 1.16.1 + 1.16.3 diff --git a/app/models/dojo_event_service.rb b/app/models/dojo_event_service.rb index bcde8ceab..7e3372e33 100644 --- a/app/models/dojo_event_service.rb +++ b/app/models/dojo_event_service.rb @@ -2,6 +2,8 @@ class DojoEventService < ApplicationRecord EXTERNAL_SERVICES = %i( connpass doorkeeper facebook ) INTERNAL_SERVICES = %i( static_yaml ) + has_many :upcoming_events + belongs_to :dojo enum name: EXTERNAL_SERVICES + INTERNAL_SERVICES diff --git a/app/models/upcoming_event.rb b/app/models/upcoming_event.rb index 0218fea4b..d0112b39a 100644 --- a/app/models/upcoming_event.rb +++ b/app/models/upcoming_event.rb @@ -1,5 +1,6 @@ class UpcomingEvent < ApplicationRecord belongs_to :dojo + belongs_to :dojo_event_service validates :dojo_event_service_id, presence: true validates :event_id, presence: true @@ -7,5 +8,5 @@ class UpcomingEvent < ApplicationRecord validates :event_at,presence: true - scope :for, ->(service) { where(service_name: service) } + scope :for, ->(service) { UpcomingEvent.where(dojo_event_service: DojoEventService.for(service)) } end \ No newline at end of file diff --git a/lib/upcoming/aggregation.rb b/lib/upcoming/aggregation.rb index ed34ca9ce..f47b5dc18 100644 --- a/lib/upcoming/aggregation.rb +++ b/lib/upcoming/aggregation.rb @@ -1,8 +1,8 @@ module Upcoming class Aggregation def initialize(args) - @from = TimeWithZone.beginning_of_week - @to = TimeWithZone.next_month + @from = Time.current.beginning_of_week + @to = Time.current.next_month @dojos = fetch_dojos end diff --git a/lib/upcoming/tasks.rb b/lib/upcoming/tasks.rb new file mode 100644 index 000000000..3ae84daaa --- /dev/null +++ b/lib/upcoming/tasks.rb @@ -0,0 +1,9 @@ +module Upcoming + module Tasks + end +end + +require_relative 'tasks/connpass' +require_relative 'tasks/doorkeeper' +require_relative 'tasks/facebook' +require_relative 'tasks/static_yaml' diff --git a/lib/upcoming/tasks/connpass.rb b/lib/upcoming/tasks/connpass.rb index d4f669640..d6c90c384 100644 --- a/lib/upcoming/tasks/connpass.rb +++ b/lib/upcoming/tasks/connpass.rb @@ -17,7 +17,7 @@ def run @client.fetch_events(@params.merge(series_id: dojo_event_service.group_id)).each do |e| next unless e.dig('series', 'id').to_s == dojo_event_service.group_id - UpcomingEvent.create!(dojo_event_service_id: e['id'], + UpcomingEvent.create!(dojo_event_service: dojo_event_service, event_id: e['event_id'], event_url: e['event_url'], event_at: Time.zone.parse(e['started_at'])) diff --git a/lib/upcoming/tasks/doorkeeper.rb b/lib/upcoming/tasks/doorkeeper.rb index f509c2f31..047dd7ac5 100644 --- a/lib/upcoming/tasks/doorkeeper.rb +++ b/lib/upcoming/tasks/doorkeeper.rb @@ -17,7 +17,7 @@ def run @client.fetch_events(@params.merge(group_id: dojo_event_service.group_id)).each do |e| next unless e['group'].to_s == dojo_event_service.group_id - UpcomingEvent.create!(dojo_event_service_id: e['id'], + UpcomingEvent.create!(dojo_event_service: dojo_event_service, event_id: e['id'], event_url: e['public_url'], event_at: Time.zone.parse(e['starts_at'])) diff --git a/lib/upcoming/tasks/static_yaml.rb b/lib/upcoming/tasks/static_yaml.rb index 2840616f4..f8350e03c 100644 --- a/lib/upcoming/tasks/static_yaml.rb +++ b/lib/upcoming/tasks/static_yaml.rb @@ -2,7 +2,7 @@ module Upcoming module Tasks class StaticYaml def self.delete_upcoming_event - EventHistory.for(:static_yaml).delete_all + UpcomingEvent.for(:static_yaml).delete_all end def initialize(dojos, _date) @@ -16,16 +16,16 @@ def run dojo = dojos_hash[e['dojo_id'].to_i] next unless dojo - evented_at = Time.zone.parse(e['evented_at']) - event_id = "#{SecureRandom.uuid}" + dojo.dojo_event_services.for(:static_yaml).each do |dojo_event_service| + evented_at = Time.zone.parse(e['evented_at']) + event_id = "#{SecureRandom.uuid}" - UpcomingEvent.create!(dojo_id: dojo.id, - dojo_name: dojo.name, - service_name: self.class.name.demodulize.underscore, - event_id: event_id, - event_url: "https://dummy.url/#{event_id}", - participants: e['participants'], - evented_at: evented_at) + UpcomingEvent.create!(dojo_event_service: dojo_event_service, + event_id: event_id, + event_url: "https://dummy.url/#{event_id}", + event_at: evented_at) + + end end end end diff --git a/logfile b/logfile new file mode 100644 index 000000000..d24ba0432 --- /dev/null +++ b/logfile @@ -0,0 +1,24 @@ +2018-06-26 17:30:30.851 JST [91625] LOG: listening on IPv6 address "::1", port 5432 +2018-06-26 17:30:30.851 JST [91625] LOG: listening on IPv4 address "127.0.0.1", port 5432 +2018-06-26 17:30:30.852 JST [91625] LOG: listening on Unix socket "/tmp/.s.PGSQL.5432" +2018-06-26 17:30:30.978 JST [91626] LOG: database system was shut down at 2018-06-13 16:41:22 JST +2018-06-26 17:30:30.998 JST [91625] LOG: database system is ready to accept connections +2018-06-26 17:30:53.572 JST [91837] ERROR: column upcoming_events.service_name does not exist at character 37 +2018-06-26 17:30:53.572 JST [91837] STATEMENT: DELETE FROM "upcoming_events" WHERE "upcoming_events"."service_name" = $1 +2018-06-26 17:31:34.879 JST [92027] ERROR: database "coderdojo_jp_development" already exists +2018-06-26 17:31:34.879 JST [92027] STATEMENT: CREATE DATABASE "coderdojo_jp_development" ENCODING = 'unicode' +2018-06-26 17:31:34.890 JST [92028] ERROR: database "coderdojo_jp_test" already exists +2018-06-26 17:31:34.890 JST [92028] STATEMENT: CREATE DATABASE "coderdojo_jp_test" ENCODING = 'unicode' +2018-06-26 17:32:08.025 JST [92199] ERROR: column upcoming_events.service_name does not exist at character 37 +2018-06-26 17:32:08.025 JST [92199] STATEMENT: DELETE FROM "upcoming_events" WHERE "upcoming_events"."service_name" = $1 +2018-06-28 14:54:47.903 JST [91625] LOG: received smart shutdown request +2018-06-28 14:54:47.906 JST [91625] LOG: worker process: logical replication launcher (PID 91632) exited with exit code 1 +2018-06-28 14:54:47.906 JST [91627] LOG: shutting down +2018-06-28 14:54:47.925 JST [91625] LOG: database system is shut down +2018-07-23 21:50:52.263 JST [90475] LOG: listening on IPv6 address "::1", port 5432 +2018-07-23 21:50:52.265 JST [90475] LOG: listening on IPv4 address "127.0.0.1", port 5432 +2018-07-23 21:50:52.269 JST [90475] LOG: listening on Unix socket "/tmp/.s.PGSQL.5432" +2018-07-23 21:50:52.400 JST [90476] LOG: database system was shut down at 2018-07-23 16:48:21 JST +2018-07-23 21:50:52.419 JST [90475] LOG: database system is ready to accept connections +2018-07-23 21:51:02.976 JST [90575] ERROR: column upcoming_events.service_name does not exist at character 37 +2018-07-23 21:51:02.976 JST [90575] STATEMENT: DELETE FROM "upcoming_events" WHERE "upcoming_events"."service_name" = $1