Merge pull request #398 from coderdojo-japan/mod_dojo_event_services_upsert

facebook のイベント履歴収集＋登録処理のエラーを解消する

Author: chicaco

.travis.yml

@@ -22,6 +22,7 @@ before_script:
 - psql -c 'create database coderdojo_jp_test;' -U postgres
 script:
 - bundle exec rake db:migrate --trace
+- bundle exec rake db:seed --trace
 - if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then bundle exec rspec spec; fi
 - if [ "$TRAVIS_PULL_REQUEST" != "false" ] && [ "$TRAVIS_PULL_REQUEST_SLUG" = "coderdojo-japan/coderdojo.jp"
   ]; then bundle exec rspec spec; fi

Gemfile

@@ -65,6 +65,7 @@ group :development, :test do
   gem 'selenium-webdriver'
   gem 'capybara'
   gem 'rspec-rails', '~> 3.5'
+  gem 'factory_bot_rails'
 
   gem 'dotenv-rails'
 end

Gemfile.lock

@@ -90,6 +90,11 @@ GEM
     ethon (0.11.0)
       ffi (>= 1.3.0)
     execjs (2.7.0)
+    factory_bot (5.0.2)
+      activesupport (>= 4.2.0)
+    factory_bot_rails (5.0.1)
+      factory_bot (~> 5.0.0)
+      railties (>= 4.2.0)
     faraday (0.15.4)
       multipart-post (>= 1.2, < 3)
     faraday_middleware (0.12.2)
@@ -401,6 +406,7 @@ DEPENDENCIES
   coffee-rails
   dotenv-rails
   dumper
+  factory_bot_rails
   faraday
   faraday_middleware
   flamegraph

lib/tasks/dojo_event_services.rake

@@ -1,12 +1,13 @@
 namespace :dojo_event_services do
   desc '現在のyamlファイルを元にデータベースを更新します'
   task upsert: :environment do
-    result = { inserted: [], updated: [], kept: [], skipped: [] }
+    result = { inserted: [], updated: [], deleted: [], kept: [], skipped: [] }
+    reserved_ids = []
 
     list = YAML.load_file(Rails.root.join('db','dojo_event_services.yaml'))
     list.each do |des|
       unless DojoEventService.names.keys.include?(des['name'])
-        event_names = DojoEventService.names.keys.map {|s| "`#{s}`" }
+        event_names = DojoEventService.names.keys.map { |s| "`#{s}`" }
         result[:skipped] << [des['dojo_id'], "Not used #{event_names.join(' or ')}"]
         next
       end
@@ -16,8 +17,10 @@ namespace :dojo_event_services do
         result[:skipped] << [des['dojo_id'], 'Not found record in `dojos` table']
         next
       end
-
+      # 比較対象は URL を除く dojo_id, name, group_id
+      url = des.delete('url')
       dojo_event_service = dojo.dojo_event_services.find_or_initialize_by(des)
+      dojo_event_service.url = url
       if dojo_event_service.changed?
         changes = dojo_event_service.changes
         new_record = dojo_event_service.new_record?
@@ -26,17 +29,22 @@ namespace :dojo_event_services do
       else
         result[:kept] << ["#{des['dojo_id']}:#{dojo_event_service.id}"]
       end
+      reserved_ids << dojo_event_service.id
+    end
+
+    DojoEventService.where.not(id: reserved_ids).find_each do |d|
+      result[:deleted] << ["#{d.dojo_id}:#{d.id}"]
+      d.destroy
     end
 
     # Dump result
-    if !result[:inserted].empty? || !result[:updated].empty?
-      result[:skipped] = result[:skipped].uniq {|s| s.first }
+    if result[:inserted] || result[:updated]
+      result[:skipped] = result[:skipped].uniq { |s| s.first }
       result.except!(:kept, :skipped) unless ENV.key?('DEBUG')
-      sorted = result.sort_by {|_, v| v.length }.reverse.to_h
-      puts
+      sorted = result.sort_by { |_, v| v.length }.reverse.to_h
       sorted.each do |k, v|
         puts "#{k.to_s.camelcase}: #{v.length}"
-        v.each {|x| puts "  #{x.join(': ')}"}
+        v.each { |x| puts "  #{x.join(': ')}" }
       end
     end
   end

spec/factories/dojo_event_services.rb

@@ -0,0 +1,10 @@
+require 'factory_bot'
+
+FactoryBot.define do
+  factory :dojo_event_service do
+    # dojo_id
+    name     { :connpass }
+    url      { '' }
+    group_id { '9999' }
+  end
+end

spec/factories/dojos.rb

@@ -0,0 +1,8 @@
+FactoryBot.define do
+  factory :dojo do
+    name          { 'dojo name' }
+    email         { '' }
+    description   { '' }
+    prefecture_id { 13 }
+  end
+end

spec/lib/tasks/dojo_event_services_spec.rb

@@ -0,0 +1,93 @@
+require 'rails_helper'
+require 'rake'
+
+RSpec.describe 'dojo_event_services' do
+  before(:all) do
+    @rake = Rake::Application.new
+    Rake.application = @rake
+    Rake.application.rake_require 'tasks/dojo_event_services'
+    Rake::Task.define_task(:environment)
+  end
+
+  before(:each) do
+    @rake[task].reenable
+  end
+
+  describe 'dojo_event_services:upsert' do
+    before :each do
+      @dojo_1 = create(:dojo, name: 'dojo_1', email: 'mail_1@test', tags: %w(Scratch ラズベリーパイ), description: '月1回開催', prefecture_id: 13)
+      @dojo_2 = create(:dojo, name: 'dojo_2', email: 'mail_2@test', tags: %w(Scratch), description: '隔週開催', prefecture_id: 13)
+      @dojo_3 = create(:dojo, name: 'dojo_3', email: 'mail_3@test', tags: %w(Scratch ラズベリーパイ Webサイト), description: '月1回開催', prefecture_id: 13)
+
+      create(:dojo_event_service, dojo_id: @dojo_1.id, name: :doorkeeper, group_id: '10001', url: 'https://coder-dojo-11.doorkeeper.jp/')
+      create(:dojo_event_service, dojo_id: @dojo_1.id, name: :doorkeeper, group_id: '10002', url: 'https://coder-dojo-12.doorkeeper.jp/')
+      create(:dojo_event_service, dojo_id: @dojo_2.id, name: :connpass,   group_id: '20001', url: 'https://coder-dojo-21.connpass.jp/')
+    end
+
+    let(:task) { 'dojo_event_services:upsert' }
+
+    it '単純追加' do
+      allow(YAML).to receive(:load_file).and_return([
+        { 'dojo_id' => @dojo_1.id, "name" => 'doorkeeper', 'group_id' => '10001', 'url' => 'https://coder-dojo-11.doorkeeper.jp/' },
+        { 'dojo_id' => @dojo_1.id, "name" => 'doorkeeper', 'group_id' => '10002', 'url' => 'https://coder-dojo-12.doorkeeper.jp/' },
+        { 'dojo_id' => @dojo_2.id, "name" => 'connpass',   'group_id' => '20001', 'url' => 'https://coder-dojo-21.connpass.jp/' },
+        { 'dojo_id' => @dojo_3.id, "name" => 'facebook',   'group_id' => '30001', 'url' => 'https://coder-dojo-31.facebook.com/' }
+      ])
+
+      # before
+      expect(DojoEventService.count).to eq(3)
+
+      # exec
+      expect(@rake[task].invoke).to be_truthy
+
+      # after
+      expect(DojoEventService.count).to eq(4)
+      new_records = DojoEventService.where(dojo_id: @dojo_3.id, name: :facebook, group_id: '30001')
+      expect(new_records.count).to eq(1)
+      expect(new_records.first.url).to eq('https://coder-dojo-31.facebook.com/')
+    end
+
+    it '単純更新' do
+      allow(YAML).to receive(:load_file).and_return([
+        { 'dojo_id' => @dojo_1.id, "name" => 'doorkeeper', 'group_id' => '10001', 'url' => 'https://coder-dojo-11.doorkeeper.jp/' },
+        { 'dojo_id' => @dojo_1.id, "name" => 'doorkeeper', 'group_id' => '10002', 'url' => 'https://coder-dojo-12.doorkeeper.jp/' },
+        { 'dojo_id' => @dojo_2.id, "name" => 'connpass',   'group_id' => '20001', 'url' => 'https://coder-dojo-21.connpass.jp/zzz' }
+      ])
+
+      # before
+      expect(DojoEventService.count).to eq(3)
+
+      # exec
+      expect(@rake[task].invoke).to be_truthy
+
+      # after
+      expect(DojoEventService.count).to eq(3)
+      mod_records = DojoEventService.where(dojo_id: @dojo_2.id, name: :connpass, group_id: '20001')
+      expect(mod_records.count).to eq(1)
+      expect(mod_records.first.url).to eq('https://coder-dojo-21.connpass.jp/zzz')
+    end
+
+    it '余剰データ削除 & 更新' do
+      create(:dojo_event_service, dojo_id: @dojo_1.id, name: :doorkeeper, group_id: '10002', url: 'https://coder-dojo-12.doorkeeper.jp/abc')
+
+      allow(YAML).to receive(:load_file).and_return([
+        { 'dojo_id' => @dojo_1.id, "name" => 'doorkeeper', 'group_id' => '10001', 'url' => 'https://coder-dojo-11.doorkeeper.jp/' },
+        { 'dojo_id' => @dojo_1.id, "name" => 'doorkeeper', 'group_id' => '10002', 'url' => 'https://coder-dojo-12.doorkeeper.jp/12345' },
+        { 'dojo_id' => @dojo_2.id, "name" => 'connpass',   'group_id' => '20001', 'url' => 'https://coder-dojo-21.connpass.jp/' }
+      ])
+
+      # before
+      expect(DojoEventService.count).to eq(4)
+      expect(DojoEventService.where(dojo_id: @dojo_1.id, name: :doorkeeper, group_id: '10002').size).to eq(2)
+
+      # exec
+      expect(@rake[task].invoke).to be_truthy
+
+      # after
+      expect(DojoEventService.count).to eq(3)
+      mod_records = DojoEventService.where(dojo_id: @dojo_1.id, name: :doorkeeper, group_id: '10002')
+      expect(mod_records.count).to eq(1)
+      expect(mod_records.first.url).to eq('https://coder-dojo-12.doorkeeper.jp/12345')
+    end
+  end
+end

spec/rails_helper.rb

@@ -62,4 +62,6 @@
       example.skip
     end
   end
+
+  config.include FactoryBot::Syntax::Methods
 end