From 2bcfea30ee4d5fc6d1a263905b69618a477f9ee1 Mon Sep 17 00:00:00 2001 From: songtianlun Date: Wed, 22 Jan 2025 17:58:25 +0800 Subject: [PATCH] feat: add background job processing with Sidekiq - Implement BatchGenerateWeatherArtsWorker to handle batch processing of weather art generation. - Create GenerateWeatherArtWorker for individual weather art generation tasks. - Update Dockerfile to include redis-tools for Sidekiq support. - Modify Gemfile to add sidekiq and sidekiq-scheduler gems. - Configure Sidekiq in initializers and set up routes for Sidekiq dashboard. - Include a sidekiq.yml configuration for scheduling jobs. - Create compose.yaml for Docker services including web, database, Redis, and Sidekiq workers. These changes introduce background processing capabilities using Sidekiq, allowing for efficient generation of weather art through scheduled and managed job queues, optimizing performance and scalability. --- Dockerfile | 2 +- Gemfile | 2 + Gemfile.lock | 17 +++++++ .../batch_generate_weather_arts_job.rb | 33 ++++++++++++++ app/workers/generate_weather_art_job.rb | 44 +++++++++++++++++++ compose.yaml | 41 +++++++++++++++++ config/initializers/sidekiq.rb | 9 ++++ config/routes.rb | 7 +++ config/schedule.rb | 6 +-- config/sidekiq.yml | 4 ++ 10 files changed, 161 insertions(+), 4 deletions(-) create mode 100644 app/workers/batch_generate_weather_arts_job.rb create mode 100644 app/workers/generate_weather_art_job.rb create mode 100644 compose.yaml create mode 100644 config/initializers/sidekiq.rb create mode 100644 config/sidekiq.yml diff --git a/Dockerfile b/Dockerfile index 726c028..a95cddc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ WORKDIR /rails # Install base packages RUN apt-get update -qq && \ - apt-get install --no-install-recommends -y curl libjemalloc2 libvips sqlite3 libpq5 && \ + apt-get install --no-install-recommends -y curl libjemalloc2 libvips sqlite3 libpq5 redis-tools && \ rm -rf /var/lib/apt/lists /var/cache/apt/archives # Set production environment diff --git a/Gemfile b/Gemfile index e2acac9..a903a58 100644 --- a/Gemfile +++ b/Gemfile @@ -50,6 +50,8 @@ gem "ruby-openai", "~> 7.3" gem "httparty", "~> 0.22.0" gem "down", "~> 5.4" gem "aws-sdk-s3", "~> 1.177" +gem 'sidekiq', '~> 7.3' +gem 'sidekiq-scheduler', '~> 5.0' group :development, :test do # See https://guides.rubyonrails.org/debugging_rails_applications.html#debugging-with-the-debug-gem diff --git a/Gemfile.lock b/Gemfile.lock index 49fe51d..311aba6 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -338,6 +338,8 @@ GEM i18n rdoc (6.11.0) psych (>= 4.0.0) + redis-client (0.23.2) + connection_pool regexp_parser (2.10.0) reline (0.6.0) io-console (~> 0.5) @@ -380,6 +382,8 @@ GEM ruby-progressbar (1.13.0) ruby2_keywords (0.0.5) rubyzip (2.4.1) + rufus-scheduler (3.9.2) + fugit (~> 1.1, >= 1.11.1) securerandom (0.4.1) selenium-webdriver (4.28.0) base64 (~> 0.2) @@ -387,6 +391,16 @@ GEM rexml (~> 3.2, >= 3.2.5) rubyzip (>= 1.2.2, < 3.0) websocket (~> 1.0) + sidekiq (7.3.8) + base64 + connection_pool (>= 2.3.0) + logger + rack (>= 2.2.4) + redis-client (>= 0.22.2) + sidekiq-scheduler (5.0.6) + rufus-scheduler (~> 3.2) + sidekiq (>= 6, < 8) + tilt (>= 1.4.0, < 3) solid_cable (3.0.5) actioncable (>= 7.2) activejob (>= 7.2) @@ -426,6 +440,7 @@ GEM thruster (0.1.10-arm64-darwin) thruster (0.1.10-x86_64-darwin) thruster (0.1.10-x86_64-linux) + tilt (2.6.0) timeout (0.4.3) turbo-rails (2.0.11) actionpack (>= 6.0.0) @@ -489,6 +504,8 @@ DEPENDENCIES rubocop-rails-omakase ruby-openai (~> 7.3) selenium-webdriver + sidekiq (~> 7.3) + sidekiq-scheduler (~> 5.0) solid_cable solid_cache solid_queue diff --git a/app/workers/batch_generate_weather_arts_job.rb b/app/workers/batch_generate_weather_arts_job.rb new file mode 100644 index 0000000..0b588e0 --- /dev/null +++ b/app/workers/batch_generate_weather_arts_job.rb @@ -0,0 +1,33 @@ +class BatchGenerateWeatherArtsWorker + include Sidekiq::Worker + + def perform(*args) + start_time = Time.current + max_duration = 50.minutes + + cities_to_process = get_eligible_cities + + cities_to_process.each do |city| + break if Time.current - start_time > max_duration + + GenerateWeatherArtJob.perform_now(city) + sleep 1.minute # 确保不超过API限制 + end + end + + private + + def get_eligible_cities + City.active + .where(active: true) + .where("last_weather_fetch IS NULL OR last_weather_fetch < ?", Date.today) + # .select { |city| early_morning_in_timezone?(city.timezone) } + end + + # def early_morning_in_timezone?(timezone) + # return false if timezone.blank? + + # time = Time.current.in_time_zone(timezone) + # time.hour == 2 + # end +end diff --git a/app/workers/generate_weather_art_job.rb b/app/workers/generate_weather_art_job.rb new file mode 100644 index 0000000..a4abe91 --- /dev/null +++ b/app/workers/generate_weather_art_job.rb @@ -0,0 +1,44 @@ +class GenerateWeatherArtWorker + + def perform(*args) + city = args[0] + return if city.last_weather_fetch&.today? + + weather_service = WeatherService.new + ai_service = AiService.new + + # 获取天气数据 + weather_data = weather_service.get_weather(city.latitude, city.longitude) + return unless weather_data + + # 生成提示词 + prompt = ai_service.generate_prompt(city, weather_data) + return unless prompt + + # 生成图像 + image_url = ai_service.generate_image(prompt) + return unless image_url + + # 创建天气艺术记录 + weather_art = city.weather_arts.create!( + weather_date: Date.today, + **weather_data, + prompt: prompt + ) + + # 下载并附加图像 + tempfile = Down.download(image_url) + weather_art.image.attach( + io: tempfile, + filename: "#{city.country.name}-#{city.name.parameterize}-#{Time.current.strftime('%Y%m%d-%H%M%S')}.png" + ) + + # 更新城市状态 + city.update!( + last_weather_fetch: Time.current, + last_image_generation: Time.current + ) + rescue => e + Rails.logger.error "Error generating weather art for #{city.name}: #{e.message}" + end +end diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..5cbed47 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,41 @@ +version: '3.8' + +services: + web: + image: songtianlun/today_ai_weather:latest + ports: + - "2222:3000" + environment: + - RAILS_ENV=production + - DATABASE_URL=postgresql://postgres:xxx@db:5432/db + - RAILS_MASTER_KEY=xxx + - REDIS_URL=redis://redis:6379/0 + depends_on: + - db + - redis + + sidekiq: + image: songtianlun/today_ai_weather:latest + command: bundle exec sidekiq + environment: + - RAILS_ENV=production + - DATABASE_URL=postgresql://postgres:xxx@db:5432/db + - RAILS_MASTER_KEY=xxx + - REDIS_URL=redis://redis:6379/0 + depends_on: + - db + - redis + + db: + image: postgres:16 + volumes: + - ./data/pg:/var/lib/postgresql/data + environment: + - POSTGRES_PASSWORD=xxx + - POSTGRES_DB=db + + redis: + image: redis:7-alpine + volumes: + - ./data/redis:/data + command: redis-server --appendonly yes \ No newline at end of file diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb new file mode 100644 index 0000000..6abd674 --- /dev/null +++ b/config/initializers/sidekiq.rb @@ -0,0 +1,9 @@ +require 'sidekiq' +require 'sidekiq-scheduler' + +Sidekiq.configure_server do |config| + config.on(:startup) do + Sidekiq.schedule = YAML.load_file(File.expand_path('../../sidekiq.yml', __FILE__)) + Sidekiq::Scheduler.reload_schedule! + end +end \ No newline at end of file diff --git a/config/routes.rb b/config/routes.rb index fd83a4b..68df5bd 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,3 +1,5 @@ +require 'sidekiq/web' + Rails.application.routes.draw do root "home#index" @@ -17,6 +19,11 @@ Rails.application.routes.draw do get "home/index" devise_for :admin_users, ActiveAdmin::Devise.config ActiveAdmin.routes(self) + + # mount Sidekiq::Web => '/sidekiq' + authenticate :admin_user do + mount Sidekiq::Web => '/sidekiq' + end # Define your application routes per the DSL in https://guides.rubyonrails.org/routing.html # Reveal health status on /up that returns 200 if the app boots with no exceptions, otherwise 500. diff --git a/config/schedule.rb b/config/schedule.rb index ef8b808..1fc9f2a 100644 --- a/config/schedule.rb +++ b/config/schedule.rb @@ -16,8 +16,8 @@ # every 4.days do # runner "AnotherModel.prune_old_records" # end -every 2.hour do - runner "BatchGenerateWeatherArtsJob.perform_later" -end +# every 2.hour do +# runner "BatchGenerateWeatherArtsJob.perform_later" +# end # Learn more: http://github.com/javan/whenever diff --git a/config/sidekiq.yml b/config/sidekiq.yml new file mode 100644 index 0000000..106e43f --- /dev/null +++ b/config/sidekiq.yml @@ -0,0 +1,4 @@ +:schedule: + sample_job: + cron: '0 * * * *' # 每小时执行 + class: BatchGenerateWeatherArtsJob.perform_later \ No newline at end of file