From ce5d09b6214d968d5336dc73abc287fa7dd9f0f3 Mon Sep 17 00:00:00 2001 From: songtianlun Date: Tue, 28 Jan 2025 01:03:06 +0800 Subject: [PATCH] feat: add admin management for various entities - Add menu labels and parents for AdminUser, City, Country, Region, WeatherArt, Ahoy::Event, and Ahoy::Visit. - Introduce a new page for managing Sidekiq jobs, providing functionality to execute or delete scheduled jobs. - Adjust batch job for generating weather art by using Sidekiq for improved performance. - Implement clean-up worker for old Ahoy data and functionalities for refreshing the sitemap. These changes enhance the administration interface by providing better organization and management tools for backend entities. The addition of Sidekiq jobs management further improves system maintenance capabilities. --- app/admin/admin_users.rb | 1 + app/admin/ahoy_events.rb | 3 +- app/admin/ahoy_visits.rb | 3 +- app/admin/cities.rb | 1 + app/admin/countries.rb | 1 + app/admin/regions.rb | 1 + app/admin/sidekiq_jobs.rb | 78 ++++++++++++++++++ app/admin/weather_arts.rb | 1 + app/jobs/application_job.rb | 7 -- app/jobs/batch_generate_weather_arts_job.rb | 35 ++++---- app/jobs/clean_ahoy_data_job.rb | 32 ++++++++ app/jobs/generate_weather_art_job.rb | 89 +++++++++++++-------- app/jobs/refresh_sitemap_job.rb | 58 ++++++++++++++ 13 files changed, 252 insertions(+), 58 deletions(-) create mode 100644 app/admin/sidekiq_jobs.rb delete mode 100644 app/jobs/application_job.rb create mode 100644 app/jobs/clean_ahoy_data_job.rb create mode 100644 app/jobs/refresh_sitemap_job.rb diff --git a/app/admin/admin_users.rb b/app/admin/admin_users.rb index fed0ec1..1e1fd4f 100644 --- a/app/admin/admin_users.rb +++ b/app/admin/admin_users.rb @@ -1,4 +1,5 @@ ActiveAdmin.register AdminUser do + menu label: "AdminUser Manager", parent: "系统管理" permit_params :email, :password, :password_confirmation index do diff --git a/app/admin/ahoy_events.rb b/app/admin/ahoy_events.rb index 35e945b..b1453c1 100644 --- a/app/admin/ahoy_events.rb +++ b/app/admin/ahoy_events.rb @@ -1,4 +1,5 @@ ActiveAdmin.register Ahoy::Event do + menu label: "事件统计", parent: "数据统计" # See permitted parameters documentation: # https://github.com/activeadmin/activeadmin/blob/master/docs/2-resource-customization.md#setting-up-strong-parameters # @@ -13,7 +14,7 @@ ActiveAdmin.register Ahoy::Event do # permitted << :other if params[:action] == 'create' && current_user.admin? # permitted # end - menu priority: 101, label: "事件统计" + # menu priority: 101, label: "事件统计" actions :index diff --git a/app/admin/ahoy_visits.rb b/app/admin/ahoy_visits.rb index 9bf914e..83218f8 100644 --- a/app/admin/ahoy_visits.rb +++ b/app/admin/ahoy_visits.rb @@ -1,4 +1,5 @@ ActiveAdmin.register Ahoy::Visit do + menu label: "访客统计", parent: "数据统计" # See permitted parameters documentation: # https://github.com/activeadmin/activeadmin/blob/master/docs/2-resource-customization.md#setting-up-strong-parameters # @@ -14,7 +15,7 @@ ActiveAdmin.register Ahoy::Visit do # permitted # end - menu priority: 100, label: "访问统计" + # menu priority: 100, label: "访问统计" actions :index diff --git a/app/admin/cities.rb b/app/admin/cities.rb index 013b901..53895cc 100644 --- a/app/admin/cities.rb +++ b/app/admin/cities.rb @@ -1,4 +1,5 @@ ActiveAdmin.register City do + menu label: "City Manager", parent: "系统管理" controller do def find_resource scoped_collection.friendly.find(params[:id]) diff --git a/app/admin/countries.rb b/app/admin/countries.rb index 13d31f9..c6be79f 100644 --- a/app/admin/countries.rb +++ b/app/admin/countries.rb @@ -1,4 +1,5 @@ ActiveAdmin.register Country do + menu label: "Country Manager", parent: "系统管理" controller do def find_resource scoped_collection.friendly.find(params[:id]) diff --git a/app/admin/regions.rb b/app/admin/regions.rb index 494ea4c..3827b06 100644 --- a/app/admin/regions.rb +++ b/app/admin/regions.rb @@ -1,4 +1,5 @@ ActiveAdmin.register Region do + menu label: "Region Manager", parent: "系统管理" # See permitted parameters documentation: # https://github.com/activeadmin/activeadmin/blob/master/docs/2-resource-customization.md#setting-up-strong-parameters # diff --git a/app/admin/sidekiq_jobs.rb b/app/admin/sidekiq_jobs.rb new file mode 100644 index 0000000..3273986 --- /dev/null +++ b/app/admin/sidekiq_jobs.rb @@ -0,0 +1,78 @@ +# app/admin/sidekiq_jobs.rb +ActiveAdmin.register_page "Sidekiq Jobs" do + menu priority: 1, label: "Sidekiq 任务管理" + + content title: "Sidekiq 任务管理" do + columns do + column do + panel "任务统计" do + stats = Sidekiq::Stats.new + table_for [ "统计数据" ] do + column "处理的任务总数" do + stats.processed + end + column "失败的任务总数" do + stats.failed + end + column "等待中的任务数" do + stats.enqueued + end + end + end + end + end + + columns do + column do + panel "计划任务列表" do + table_for Sidekiq::ScheduledSet.new.to_a do + column "任务名称" do |job| + job.item["class"] + end + column "执行时间" do |job| + Time.at(job.at) + end + column "参数" do |job| + job.item["args"].to_s + end + column "操作" do |job| + links = [] + links << link_to("立即执行", execute_admin_sidekiq_jobs_path(jid: job.jid), method: :post) + links << link_to("删除", delete_admin_sidekiq_jobs_path(jid: job.jid), method: :delete) + links.join(" | ").html_safe + end + end + end + end + end + end + + action_item :refresh do + link_to "刷新", admin_sidekiq_jobs_path + end + + # 将 collection_action 改为 page_action + page_action :execute, method: :post do + jid = params[:jid] + job = Sidekiq::ScheduledSet.new.find_job(jid) + if job + job.delete + klass = job.item["class"].constantize + klass.perform_async(*job.item["args"]) + redirect_to admin_sidekiq_jobs_path, notice: "任务已立即执行" + else + redirect_to admin_sidekiq_jobs_path, alert: "任务未找到" + end + end + + page_action :delete, method: :delete do + jid = params[:jid] + job = Sidekiq::ScheduledSet.new.find_job(jid) + if job + job.delete + redirect_to admin_sidekiq_jobs_path, notice: "任务已删除" + else + redirect_to admin_sidekiq_jobs_path, alert: "任务未找到" + end + end +end diff --git a/app/admin/weather_arts.rb b/app/admin/weather_arts.rb index 209e8dc..7f560d5 100644 --- a/app/admin/weather_arts.rb +++ b/app/admin/weather_arts.rb @@ -1,4 +1,5 @@ ActiveAdmin.register WeatherArt do + menu label: "WeatherArt Manager", parent: "系统管理" controller do def find_resource scoped_collection.friendly.find(params[:id]) diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb deleted file mode 100644 index d394c3d..0000000 --- a/app/jobs/application_job.rb +++ /dev/null @@ -1,7 +0,0 @@ -class ApplicationJob < ActiveJob::Base - # Automatically retry jobs that encountered a deadlock - # retry_on ActiveRecord::Deadlocked - - # Most jobs are safe to ignore if the underlying records are no longer available - # discard_on ActiveJob::DeserializationError -end diff --git a/app/jobs/batch_generate_weather_arts_job.rb b/app/jobs/batch_generate_weather_arts_job.rb index 350753b..5d7f16d 100644 --- a/app/jobs/batch_generate_weather_arts_job.rb +++ b/app/jobs/batch_generate_weather_arts_job.rb @@ -1,33 +1,36 @@ -class BatchGenerateWeatherArtsJob < ApplicationJob - queue_as :default +class BatchGenerateWeatherArtsWorker + include Sidekiq::Worker + + GENERATION_INTERVAL = 24.hours + MAX_DURATION = 50.minutes + SLEEP_DURATION = 120.seconds def perform(*args) start_time = Time.current - max_duration = 50.minutes cities_to_process = get_eligible_cities cities_to_process.each do |city| - break if Time.current - start_time > max_duration + break if Time.current - start_time > MAX_DURATION + Rails.logger.info "Generating weather art for #{city.name}" - GenerateWeatherArtJob.perform_now(city) - sleep 1.minute # 确保不超过API限制 + GenerateWeatherArtWorker.perform_async(city.id) + sleep SLEEP_DURATION end end private def get_eligible_cities + cutoff_time = Time.current - GENERATION_INTERVAL + City.active - .where(active: true) - .where("last_weather_fetch IS NULL OR last_weather_fetch < ?", Date.today) - # .select { |city| early_morning_in_timezone?(city.timezone) } + .joins("LEFT JOIN ( + SELECT city_id, MAX(created_at) as last_generation_time + FROM weather_arts + GROUP BY city_id + ) latest_arts ON cities.id = latest_arts.city_id") + .where("latest_arts.last_generation_time IS NULL OR latest_arts.last_generation_time < ?", cutoff_time) + .order(:priority) end - - # def early_morning_in_timezone?(timezone) - # return false if timezone.blank? - - # time = Time.current.in_time_zone(timezone) - # time.hour == 2 - # end end diff --git a/app/jobs/clean_ahoy_data_job.rb b/app/jobs/clean_ahoy_data_job.rb new file mode 100644 index 0000000..b049e43 --- /dev/null +++ b/app/jobs/clean_ahoy_data_job.rb @@ -0,0 +1,32 @@ +# app/workers/clean_ahoy_data_worker.rb +class CleanAhoyDataWorker + include Sidekiq::Worker + + sidekiq_options queue: :default, retry: false + + def perform + cleanup_old_events + cleanup_old_visits + log_cleanup_results + end + + private + + def cleanup_old_events + cutoff_date = 3.months.ago + deleted_events_count = Ahoy::Event.where("time < ?", cutoff_date).delete_all + Rails.logger.info "Deleted #{deleted_events_count} old Ahoy events" + end + + def cleanup_old_visits + cutoff_date = 3.months.ago + deleted_visits_count = Ahoy::Visit.where("started_at < ?", cutoff_date).delete_all + Rails.logger.info "Deleted #{deleted_visits_count} old Ahoy visits" + end + + def log_cleanup_results + Rails.logger.info "Ahoy cleanup completed at #{Time.current}" + Rails.logger.info "Remaining events: #{Ahoy::Event.count}" + Rails.logger.info "Remaining visits: #{Ahoy::Visit.count}" + end +end diff --git a/app/jobs/generate_weather_art_job.rb b/app/jobs/generate_weather_art_job.rb index adbaaa2..0412798 100644 --- a/app/jobs/generate_weather_art_job.rb +++ b/app/jobs/generate_weather_art_job.rb @@ -1,45 +1,68 @@ -class GenerateWeatherArtJob < ApplicationJob - queue_as :default +class GenerateWeatherArtWorker + include Sidekiq::Worker - def perform(*args) - city = args[0] - return if city.last_weather_fetch&.today? + def perform(city_id) + @city = City.find(city_id) - weather_service = WeatherService.new - ai_service = AiService.new - - # 获取天气数据 - weather_data = weather_service.get_weather(city.latitude, city.longitude) + weather_data = fetch_weather_data return unless weather_data - # 生成提示词 - prompt = ai_service.generate_prompt(city, weather_data) + prompt = generate_prompt(weather_data) return unless prompt - # 生成图像 - image_url = ai_service.generate_image(prompt) + image_url = generate_image(prompt) return unless image_url - # 创建天气艺术记录 - weather_art = city.weather_arts.create!( - weather_date: Date.today, - **weather_data, - prompt: prompt - ) + create_weather_art(weather_data, prompt, image_url) + rescue StandardError => e + Rails.logger.error "Error generating weather art for city #{city_id}: #{e.message}" + Rails.logger.error e.backtrace.join("\n") + end - # 下载并附加图像 - tempfile = Down.download(image_url) - weather_art.image.attach( - io: tempfile, - filename: "#{city.country.name}-#{city.name.parameterize}-#{Time.current.strftime('%Y%m%d-%H%M%S')}.png" - ) + private - # 更新城市状态 - city.update!( - last_weather_fetch: Time.current, - last_image_generation: Time.current - ) - rescue => e - Rails.logger.error "Error generating weather art for #{city.name}: #{e.message}" + attr_reader :city + + def fetch_weather_data + WeatherService.new.get_weather(city.latitude, city.longitude) + end + + def generate_prompt(weather_data) + AiService.new.generate_prompt(city, weather_data) + end + + def generate_image(prompt) + AiService.new.generate_image(prompt) + end + + def create_weather_art(weather_data, prompt, image_url) + tempfile = nil + + ActiveRecord::Base.transaction do + weather_art = city.weather_arts.create!( + weather_date: Date.today, + prompt: prompt, + **weather_data + ) + + tempfile = Down.download(image_url) + + weather_art.image.attach( + io: File.open(tempfile.path), + filename: generate_filename, + content_type: "image/png" + ) + + weather_art + end + ensure + if tempfile + tempfile.close + tempfile.unlink + end + end + + def generate_filename + "#{city.country.name}-#{city.name.parameterize}-#{Time.current.strftime('%Y%m%d-%H%M%S')}.png" end end diff --git a/app/jobs/refresh_sitemap_job.rb b/app/jobs/refresh_sitemap_job.rb new file mode 100644 index 0000000..6b768dd --- /dev/null +++ b/app/jobs/refresh_sitemap_job.rb @@ -0,0 +1,58 @@ +class RefreshSitemapWorker + include Sidekiq::Worker + + def perform + host = Rails.env.production? ? "https://todayaiweather.com" : "http://127.0.0.1:3000" + Rails.application.routes.default_url_options[:host] = host + SitemapGenerator::Sitemap.default_host = host + if Rails.env.production? + SitemapGenerator::Sitemap.adapter = SitemapGenerator::AwsSdkAdapter.new( + ENV.fetch("AWS_BUCKET", Rails.application.credentials.dig(:aws, :bucket)), + aws_access_key_id: ENV.fetch("AWS_ACCESS_KEY_ID", Rails.application.credentials.dig(:aws, :access_key_id)), + aws_secret_access_key: ENV.fetch("AWS_SECRET_ACCESS_KEY_ID", Rails.application.credentials.dig(:aws, :secret_access_key)), + aws_region: ENV.fetch("AWS_REGION", "wnam"), + endpoint: ENV.fetch("AWS_ENDPOINT", Rails.application.credentials.dig(:aws, :endpoint)), + ) + else + SitemapGenerator::Sitemap.adapter = SitemapGenerator::AwsSdkAdapter.new( + ENV.fetch("AWS_DEV_BUCKET", Rails.application.credentials.dig(:aws_dev, :bucket)), + aws_access_key_id: ENV.fetch("AWS_DEV_ACCESS_KEY_ID", Rails.application.credentials.dig(:aws_dev, :access_key_id)), + aws_secret_access_key: ENV.fetch("AWS_DEV_SECRET_ACCESS_KEY_ID", Rails.application.credentials.dig(:aws_dev, :secret_access_key)), + aws_region: ENV.fetch("AWS_DEV_REGION", "wnam"), + endpoint: ENV.fetch("AWS_DEV_ENDPOINT", Rails.application.credentials.dig(:aws_dev, :endpoint)), + ) + end + SitemapGenerator::Sitemap.sitemaps_path = "sitemaps/" + + SitemapGenerator::Sitemap.create do + add root_path, changefreq: "daily", priority: 1.0 + add cities_path, changefreq: "daily", priority: 0.9 + add arts_path, changefreq: "daily", priority: 0.9 + + City.find_each do |city| + add city_path(city), + changefreq: "daily", + priority: 0.8, + lastmod: city.updated_at + end + + WeatherArt.includes(:city).find_each do |art| + if art.image.attached? + add city_weather_art_path(art.city, art), + changefreq: "daily", + priority: 0.7, + lastmod: art.updated_at, + images: [ { + loc: url_for(art.image), + title: "#{art.city.name} Weather Art - #{art.weather_date.strftime('%B %d, %Y')}" + } ] + end + end + end + + # SitemapGenerator::Sitemap.ping_search_engines if Rails.env.production? + Rails.logger.info "Sitemap has been generated and uploaded to S3 successfully" + rescue => e + Rails.logger.error "Error refreshing sitemap: #{e.message}" + end +end