Merge pull request #128 from nalabjp/statistics

Implementation to aggregate statistical information #12

Author: yasulab

.travis.yml

@@ -12,4 +12,11 @@ deploy:
   run:
       - "bundle exec rails db:migrate"
       - "bundle exec rails dojos:update_db_by_yaml"
-      
+cache:
+  - bundler
+script:
+  - bundle exec rake db:migrate --trace
+  - bundle exec rake
+env:
+  global:
+    - TZ='Asia/Tokyo'

Gemfile

@@ -26,6 +26,7 @@ gem 'secure_headers'
 # Rendering legal documents
 gem 'kramdown'
 
+gem 'faraday'
 # TODO: Delete this if the following issue is fixed
 # https://github.com/bundler/bundler/issues/5332
 gem 'faraday_middleware', '0.10'

Gemfile.lock

@@ -284,6 +284,7 @@ DEPENDENCIES
   bootstrap-sass
   capybara
   coffee-rails
+  faraday
   faraday_middleware (= 0.10)
   font-awesome-rails
   jbuilder
@@ -315,4 +316,4 @@ RUBY VERSION
    ruby 2.4.0p0
 
 BUNDLED WITH
-   1.13.7
+   1.15.3

app/models/dojo.rb

@@ -3,6 +3,9 @@ class Dojo < ApplicationRecord
   NUM_OF_WHOLE_DOJOS = "1,250"
   NUM_OF_JAPAN_DOJOS = Dojo.count.to_s
 
+  has_one :dojo_event_service
+  has_many :event_histories
+
   serialize :tags
   default_scope -> { order(order: :asc) }
   before_save { self.email = self.email.downcase }

app/models/dojo_event_service.rb

@@ -0,0 +1,4 @@
+class DojoEventService < ApplicationRecord
+  belongs_to :dojo
+  enum name: %i( connpass doorkeeper )
+end

app/models/event_history.rb

@@ -0,0 +1,11 @@
+class EventHistory < ApplicationRecord
+  belongs_to :dojo
+
+  validates :dojo_name, presence: true
+  validates :service_name, presence: true, uniqueness: { scope: :event_id  }
+  validates :service_group_id, presence: true
+  validates :event_id, presence: true
+  validates :event_url, presence: true, uniqueness: true, allow_nil: true
+  validates :participants, presence: true
+  validates :evented_at, presence: true
+end

config/application.rb

@@ -11,5 +11,8 @@ class Application < Rails::Application
     # Settings in config/environments/* take precedence over those specified here.
     # Application configuration should go into files in config/initializers
     # -- all .rb files in that directory are automatically loaded.
+
+    # Timezone
+    config.time_zone = 'Asia/Tokyo'
   end
 end

db/migrate/20170730050122_create_event_histories.rb

@@ -0,0 +1,18 @@
+class CreateEventHistories < ActiveRecord::Migration[5.0]
+  def change
+    create_table :event_histories do |t|
+      t.references :dojo, foreign_key: true, index: true, null: false
+      t.string :dojo_name, null: false
+      t.string :service_name, null: false
+      t.integer :service_group_id, null: false
+      t.integer :event_id, null: false
+      t.string :event_url, unique: true, null: false
+      t.integer :participants, null: false
+      t.datetime :evented_at, null: false
+      t.timestamps
+
+      t.index [:service_name, :event_id], unique: true
+      t.index [:evented_at, :dojo_id]
+    end
+  end
+end

db/migrate/20170730114150_create_dojo_event_services.rb

@@ -0,0 +1,11 @@
+class CreateDojoEventServices < ActiveRecord::Migration[5.0]
+  def change
+    create_table :dojo_event_services do |t|
+      t.references :dojo, foreign_key: true, index: true, null: false
+      t.string :name
+      t.string :url
+      t.integer :group_id
+      t.timestamps
+    end
+  end
+end

db/migrate/20170820085052_add_not_null_to_dojo_event_services.rb

@@ -0,0 +1,6 @@
+class AddNotNullToDojoEventServices < ActiveRecord::Migration[5.0]
+  def change
+    change_column_null :dojo_event_services, :name, false
+    change_column_null :dojo_event_services, :group_id, false
+  end
+end

db/migrate/20170820090605_change_column_type_dojo_event_services.rb

@@ -0,0 +1,9 @@
+class ChangeColumnTypeDojoEventServices < ActiveRecord::Migration[5.0]
+  def up
+    change_column :dojo_event_services, :name, :integer, null: false
+  end
+
+  def down
+    change_column :dojo_event_services, :name, :string, null: false
+  end
+end

db/schema.rb

@@ -10,7 +10,17 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20161217063010) do
+ActiveRecord::Schema.define(version: 20170820090605) do
+
+  create_table "dojo_event_services", force: :cascade do |t|
+    t.integer  "dojo_id",    null: false
+    t.integer  "name",       null: false
+    t.string   "url"
+    t.integer  "group_id",   null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["dojo_id"], name: "index_dojo_event_services_on_dojo_id"
+  end
 
   create_table "dojos", force: :cascade do |t|
     t.string   "name"
@@ -24,4 +34,20 @@
     t.datetime "updated_at",                        null: false
   end
 
+  create_table "event_histories", force: :cascade do |t|
+    t.integer  "dojo_id",          null: false
+    t.string   "dojo_name",        null: false
+    t.string   "service_name",     null: false
+    t.integer  "service_group_id", null: false
+    t.integer  "event_id",         null: false
+    t.string   "event_url",        null: false
+    t.integer  "participants",     null: false
+    t.datetime "evented_at",       null: false
+    t.datetime "created_at",       null: false
+    t.datetime "updated_at",       null: false
+    t.index ["dojo_id"], name: "index_event_histories_on_dojo_id"
+    t.index ["evented_at", "dojo_id"], name: "index_event_histories_on_evented_at_and_dojo_id"
+    t.index ["service_name", "event_id"], name: "index_event_histories_on_service_name_and_event_id", unique: true
+  end
+
 end

lib/statistics.rb

@@ -0,0 +1,4 @@
+module Statistics; end
+
+require_relative 'statistics/client'
+require_relative 'statistics/aggregation'

lib/statistics/aggregation.rb

@@ -0,0 +1,66 @@
+module Statistics
+  class Aggregation
+    class << self
+      def run(date:)
+        cnps_dojos = Dojo.joins(:dojo_event_service).where(dojo_event_services: { name: :connpass }).to_a
+        drkp_dojos = Dojo.joins(:dojo_event_service).where(dojo_event_services: { name: :doorkeeper }).to_a
+
+        Connpass.run(cnps_dojos, date)
+        Doorkeeper.run(drkp_dojos, date)
+      end
+    end
+
+    class Connpass
+      class << self
+        def run(dojos, date)
+          cnps = Client::Connpass.new
+          params = {
+            yyyymm: "#{date.year}#{date.month}"
+          }
+
+          dojos.each do |dojo|
+            cnps.fetch_events(params.merge(series_id: dojo.dojo_event_service.group_id)).each do |e|
+              next unless e.dig('series', 'id') == dojo.dojo_event_service.group_id
+
+              EventHistory.create!(dojo_id: dojo.id,
+                                   dojo_name: dojo.name,
+                                   service_name: dojo.dojo_event_service.name,
+                                   service_group_id: dojo.dojo_event_service.group_id,
+                                   event_id: e['event_id'],
+                                   event_url: e['event_url'],
+                                   participants: e['accepted'],
+                                   evented_at: Time.zone.parse(e['started_at']))
+            end
+          end
+        end
+      end
+    end
+
+    class Doorkeeper
+      class << self
+        def run(dojos, date)
+          drkp = Client::Doorkeeper.new
+          params = {
+            since_at: date.beginning_of_month,
+            until_at: date.end_of_month
+          }
+
+          dojos.each do |dojo|
+            drkp.fetch_events(params.merge(group_id: dojo.dojo_event_service.group_id)).each do |e|
+              next unless e['group'] == dojo.dojo_event_service.group_id
+
+              EventHistory.create!(dojo_id: dojo.id,
+                                   dojo_name: dojo.name,
+                                   service_name: dojo.dojo_event_service.name,
+                                   service_group_id: dojo.dojo_event_service.group_id,
+                                   event_id: e['id'],
+                                   event_url: e['public_url'],
+                                   participants: e['participants'],
+                                   evented_at: Time.zone.parse(e['starts_at']))
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/statistics/client.rb

@@ -0,0 +1,105 @@
+module Statistics
+  class Client
+    class_attribute :debug
+    self.debug = false
+
+    def initialize(endpoint, &block)
+      @conn = connection_for(endpoint, &block)
+    end
+
+    def get(path, params)
+      @conn.get(path, params).body
+    end
+
+    private
+
+    def connection_for(endpoint)
+      Faraday.new(endpoint) do |f|
+        f.response :logger if self.class.debug
+        f.response :json, :content_type => /\bjson$/
+        f.response :raise_error
+
+        yield f if block_given?
+
+        f.adapter  Faraday.default_adapter
+      end
+    end
+
+    class Connpass
+      ENDPOINT = 'https://connpass.com/api/v1'.freeze
+
+      def initialize
+        @client = Client.new(ENDPOINT)
+      end
+
+      def search(keyword:)
+        @client.get('event/', { keyword: keyword, count: 100 })
+      end
+
+      def fetch_events(series_id:, yyyymm: nil)
+        params = {
+          series_id: series_id,
+          start: 1,
+          count: 100
+        }
+        params[:ym] = yyyymm if yyyymm
+        events = []
+
+        loop do
+          part = @client.get('event/', params)
+
+          break if part['results_returned'].zero?
+
+          events.push(*part.fetch('events'))
+
+          break if part.size < params[:count]
+
+          break if params[:start] + params[:count] > part['results_available']
+
+          params[:start] += params[:count]
+        end
+
+        events
+      end
+    end
+
+    class Doorkeeper
+      ENDPOINT = 'https://api.doorkeeper.jp'.freeze
+
+      def initialize
+        @client = Client.new(ENDPOINT) do |c|
+          c.authorization(:Bearer, ENV.fetch('DOORKEEPER_API_TOKEN'))
+        end
+        @default_since = Time.zone.parse('2010-07-01')
+        @default_until = Time.zone.now.end_of_day
+      end
+
+      def search(keyword:)
+        @client.get('events', q: keyword, since: @default_since)
+      end
+
+      def fetch_events(group_id:, since_at: @default_since, until_at: @default_until)
+        params = {
+          page: 1,
+          since: since_at,
+          until: until_at
+        }
+        events = []
+
+        loop do
+          part = @client.get("groups/#{group_id}/events", params)
+
+          break if part.size.zero?
+
+          events.push(*part.map { |e| e['event'] })
+
+          break if part.size < 25   # 25 items / 1 request
+
+          params[:page] += 1
+        end
+
+        events
+      end
+    end
+  end
+end

lib/tasks/statistics.rake

@@ -0,0 +1,36 @@
+require_relative '../statistics.rb'
+
+namespace :statistics do
+  desc '月次のイベント履歴を集計します'
+  task :aggregation, [:yyyymm] => :environment do |tasks, args|
+    date = Time.current.prev_month.beginning_of_month
+    if args[:yyyymm].present?
+      date = %w(%Y%m %Y/%m %Y-%m).map do |fmt|
+               begin
+                 Time.zone.strptime(args[:yyyymm], fmt)
+               rescue ArgumentError
+               end
+             end.compact.first
+    end
+
+    raise ArgumentError, "Invalid format: #{args[:yyyymm]}" if date.nil?
+
+
+    EventHistory.where(evented_at: date.beginning_of_month..date.end_of_month).delete_all
+
+    Statistics::Aggregation.run(date: date)
+  end
+
+  desc 'キーワードからイベント情報を検索します'
+  task :search, [:keyword] => :environment do |tasks, args|
+    raise ArgumentError, 'Require the keyword' if args[:keyword].nil?
+
+    require 'pp'
+
+    puts 'Searching Connpass'
+    pp Statistics::Client::Connpass.new.search(keyword: args[:keyword])
+
+    puts 'Searching Doorkeeper'
+    pp Statistics::Client::Doorkeeper.new.search(keyword: args[:keyword])
+  end
+end