feat: add background job processing with Sidekiq

- Implement BatchGenerateWeatherArtsWorker to handle batch
  processing of weather art generation.
- Create GenerateWeatherArtWorker for individual weather art
  generation tasks.
- Update Dockerfile to include redis-tools for Sidekiq support.
- Modify Gemfile to add sidekiq and sidekiq-scheduler gems.
- Configure Sidekiq in initializers and set up routes for
  Sidekiq dashboard.
- Include a sidekiq.yml configuration for scheduling jobs.
- Create compose.yaml for Docker services including web,
  database, Redis, and Sidekiq workers.

These changes introduce background processing capabilities
using Sidekiq, allowing for efficient generation of weather
art through scheduled and managed job queues, optimizing
performance and scalability.

Dockerfile

@@ -16,7 +16,7 @@ WORKDIR /rails
 
 # Install base packages
 RUN apt-get update -qq && \
-    apt-get install --no-install-recommends -y curl libjemalloc2 libvips sqlite3 libpq5 && \
+    apt-get install --no-install-recommends -y curl libjemalloc2 libvips sqlite3 libpq5 redis-tools && \
     rm -rf /var/lib/apt/lists /var/cache/apt/archives
 
 # Set production environment

Gemfile

@@ -50,6 +50,8 @@ gem "ruby-openai", "~> 7.3"
 gem "httparty", "~> 0.22.0"
 gem "down", "~> 5.4"
 gem "aws-sdk-s3", "~> 1.177"
+gem 'sidekiq', '~> 7.3'
+gem 'sidekiq-scheduler', '~> 5.0'
 
 group :development, :test do
   # See https://guides.rubyonrails.org/debugging_rails_applications.html#debugging-with-the-debug-gem

Gemfile.lock

@@ -338,6 +338,8 @@ GEM
       i18n
     rdoc (6.11.0)
       psych (>= 4.0.0)
+    redis-client (0.23.2)
+      connection_pool
     regexp_parser (2.10.0)
     reline (0.6.0)
       io-console (~> 0.5)
@@ -380,6 +382,8 @@ GEM
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
     rubyzip (2.4.1)
+    rufus-scheduler (3.9.2)
+      fugit (~> 1.1, >= 1.11.1)
     securerandom (0.4.1)
     selenium-webdriver (4.28.0)
       base64 (~> 0.2)
@@ -387,6 +391,16 @@ GEM
       rexml (~> 3.2, >= 3.2.5)
       rubyzip (>= 1.2.2, < 3.0)
       websocket (~> 1.0)
+    sidekiq (7.3.8)
+      base64
+      connection_pool (>= 2.3.0)
+      logger
+      rack (>= 2.2.4)
+      redis-client (>= 0.22.2)
+    sidekiq-scheduler (5.0.6)
+      rufus-scheduler (~> 3.2)
+      sidekiq (>= 6, < 8)
+      tilt (>= 1.4.0, < 3)
     solid_cable (3.0.5)
       actioncable (>= 7.2)
       activejob (>= 7.2)
@@ -426,6 +440,7 @@ GEM
     thruster (0.1.10-arm64-darwin)
     thruster (0.1.10-x86_64-darwin)
     thruster (0.1.10-x86_64-linux)
+    tilt (2.6.0)
     timeout (0.4.3)
     turbo-rails (2.0.11)
       actionpack (>= 6.0.0)
@@ -489,6 +504,8 @@ DEPENDENCIES
   rubocop-rails-omakase
   ruby-openai (~> 7.3)
   selenium-webdriver
+  sidekiq (~> 7.3)
+  sidekiq-scheduler (~> 5.0)
   solid_cable
   solid_cache
   solid_queue

app/workers/batch_generate_weather_arts_job.rb

@@ -0,0 +1,33 @@
+class BatchGenerateWeatherArtsWorker
+  include Sidekiq::Worker
+
+  def perform(*args)
+    start_time = Time.current
+    max_duration = 50.minutes
+
+    cities_to_process = get_eligible_cities
+
+    cities_to_process.each do |city|
+      break if Time.current - start_time > max_duration
+
+      GenerateWeatherArtJob.perform_now(city)
+      sleep 1.minute # 确保不超过API限制
+    end
+  end
+
+  private
+
+  def get_eligible_cities
+    City.active
+        .where(active: true)
+        .where("last_weather_fetch IS NULL OR last_weather_fetch < ?", Date.today)
+    # .select { |city| early_morning_in_timezone?(city.timezone) }
+  end
+
+  # def early_morning_in_timezone?(timezone)
+  #   return false if timezone.blank?
+
+  #   time = Time.current.in_time_zone(timezone)
+  #   time.hour == 2
+  # end
+end

app/workers/generate_weather_art_job.rb

@@ -0,0 +1,44 @@
+class GenerateWeatherArtWorker
+
+  def perform(*args)
+    city = args[0]
+    return if city.last_weather_fetch&.today?
+
+    weather_service = WeatherService.new
+    ai_service = AiService.new
+
+    # 获取天气数据
+    weather_data = weather_service.get_weather(city.latitude, city.longitude)
+    return unless weather_data
+
+    # 生成提示词
+    prompt = ai_service.generate_prompt(city, weather_data)
+    return unless prompt
+
+    # 生成图像
+    image_url = ai_service.generate_image(prompt)
+    return unless image_url
+
+    # 创建天气艺术记录
+    weather_art = city.weather_arts.create!(
+      weather_date: Date.today,
+      **weather_data,
+      prompt: prompt
+    )
+
+    # 下载并附加图像
+    tempfile = Down.download(image_url)
+    weather_art.image.attach(
+      io: tempfile,
+      filename: "#{city.country.name}-#{city.name.parameterize}-#{Time.current.strftime('%Y%m%d-%H%M%S')}.png"
+    )
+
+    # 更新城市状态
+    city.update!(
+      last_weather_fetch: Time.current,
+      last_image_generation: Time.current
+    )
+  rescue => e
+    Rails.logger.error "Error generating weather art for #{city.name}: #{e.message}"
+  end
+end

compose.yaml

@@ -0,0 +1,41 @@
+version: '3.8'
+
+services:
+  web:
+    image: songtianlun/today_ai_weather:latest
+    ports:
+      - "2222:3000"
+    environment:
+      - RAILS_ENV=production
+      - DATABASE_URL=postgresql://postgres:xxx@db:5432/db
+      - RAILS_MASTER_KEY=xxx
+      - REDIS_URL=redis://redis:6379/0
+    depends_on:
+      - db
+      - redis
+
+  sidekiq:
+    image: songtianlun/today_ai_weather:latest
+    command: bundle exec sidekiq
+    environment:
+      - RAILS_ENV=production
+      - DATABASE_URL=postgresql://postgres:xxx@db:5432/db
+      - RAILS_MASTER_KEY=xxx
+      - REDIS_URL=redis://redis:6379/0
+    depends_on:
+      - db
+      - redis
+
+  db:
+    image: postgres:16
+    volumes:
+      - ./data/pg:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_PASSWORD=xxx
+      - POSTGRES_DB=db
+
+  redis:
+    image: redis:7-alpine
+    volumes:
+      - ./data/redis:/data
+    command: redis-server --appendonly yes

config/initializers/sidekiq.rb

@@ -0,0 +1,9 @@
+require 'sidekiq'
+require 'sidekiq-scheduler'
+
+Sidekiq.configure_server do |config|
+  config.on(:startup) do
+    Sidekiq.schedule = YAML.load_file(File.expand_path('../../sidekiq.yml', __FILE__))
+    Sidekiq::Scheduler.reload_schedule!
+  end
+end

config/routes.rb

@@ -1,3 +1,5 @@
+require 'sidekiq/web'
+
 Rails.application.routes.draw do
   root "home#index"
 
@@ -17,6 +19,11 @@ Rails.application.routes.draw do
   get "home/index"
   devise_for :admin_users, ActiveAdmin::Devise.config
   ActiveAdmin.routes(self)
+
+  # mount Sidekiq::Web => '/sidekiq'
+  authenticate :admin_user do
+    mount Sidekiq::Web => '/sidekiq'
+  end
   # Define your application routes per the DSL in https://guides.rubyonrails.org/routing.html
 
   # Reveal health status on /up that returns 200 if the app boots with no exceptions, otherwise 500.

config/schedule.rb

@@ -16,8 +16,8 @@
 # every 4.days do
 #   runner "AnotherModel.prune_old_records"
 # end
-every 2.hour do
+# every 2.hour do
-  runner "BatchGenerateWeatherArtsJob.perform_later"
+#   runner "BatchGenerateWeatherArtsJob.perform_later"
-end
+# end
 
 # Learn more: http://github.com/javan/whenever

config/sidekiq.yml

@@ -0,0 +1,4 @@
+:schedule:
+  sample_job:
+    cron: '0 * * * *'  # 每小时执行
+    class: BatchGenerateWeatherArtsJob.perform_later