feat: add key verification and IndexNow submission worker

- Implement KeyVerificationController to handle ID-based key verification.
- Create SubmitToIndexnowWorker for background processing of URL submissions.
- Add routes for key verification and extend functionality of URL submissions to IndexNow.

This commit enhances the application by allowing for verification of API keys and automatic submission of URLs to multiple endpoints, improving SEO and resource accessibility.

app/controllers/key_verification_controller.rb

@@ -0,0 +1,10 @@
+# app/controllers/key_verification_controller.rb
+class KeyVerificationController < ApplicationController
+  def show
+    if params[:id] == "339ecd3e9cf648c29b767f5673329e48"
+      render plain: "339ecd3e9cf648c29b767f5673329e48"
+    else
+      render status: :not_found, plain: "Not Found"
+    end
+  end
+end

app/workers/submit_to_indexnow_worker.rb

@@ -0,0 +1,151 @@
+# app/workers/submit_to_indexnow_worker.rb
+class SubmitToIndexnowWorker
+  include Sidekiq::Worker
+  require "redis"
+  require "net/http"
+  require "uri"
+  require "json"
+
+  sidekiq_options retry: 3, queue: :indexnow
+
+  INDEXNOW_KEY = "339ecd3e9cf648c29b767f5673329e48"
+  INDEXNOW_ENDPOINTS = [
+    "https://api.indexnow.org/indexnow",
+    "https://www.bing.com/indexnow"
+  ]
+
+  def perform(recent)
+    lock_key = "submit_to_indexnow_lock"
+    lock_ttl = 300 # 锁的生存时间，单位为秒
+
+    recently_updated = recent ? DateTime.now - 1.day : DateTime.now - 356.day
+
+    redis = Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/1"))
+
+    if redis.set(lock_key, Time.current.to_s, nx: true, ex: lock_ttl)
+      begin
+        @host = Rails.env.production? ? "https://todayaiweather.com" : "http://127.0.0.1:3000"
+        Rails.application.routes.default_url_options[:host] = @host
+
+        # 收集所有需要提交的 URLs
+        urls = collect_urls
+
+        # 分批提交 URLs (每批最多 10000 个 URLs，符合 IndexNow 限制)
+        urls.each_slice(100) do |url_batch|
+          submit_urls(url_batch)
+        end
+
+        Rails.logger.info "Successfully submitted #{urls.size} URLs to IndexNow"
+      rescue => e
+        Rails.logger.error "Error submitting URLs to IndexNow: #{e.message}"
+        Rails.logger.error e.backtrace.join("\n")
+        raise e
+      ensure
+        redis.del(lock_key)
+      end
+    else
+      Rails.logger.info "IndexNow submission is already in progress"
+    end
+  end
+
+  private
+
+  def collect_urls
+    urls = []
+    available_locales = I18n.available_locales
+
+    # 添加首页
+    urls << @host
+    available_locales.each do |locale|
+      urls << "#{@host}/#{locale}"
+    end
+
+    # 城市列表页
+    urls << "#{@host}/cities"
+    available_locales.each do |locale|
+      urls << "#{@host}/#{locale}/cities"
+    end
+
+    # 艺术作品列表页
+    urls << "#{@host}/arts"
+    available_locales.each do |locale|
+      urls << "#{@host}/#{locale}/arts"
+    end
+
+    # 城市详情页
+    City.find_each do |city|
+      urls << "#{@host}/cities/#{city.id}"
+      available_locales.each do |locale|
+        urls << "#{@host}/#{locale}/cities/#{city.id}"
+      end
+    end
+
+    # 天气艺术作品页
+    WeatherArt.includes(:city).find_each do |art|
+      if art.image.attached?
+        urls << "#{@host}/cities/#{art.city.id}/weather_arts/#{art.id}"
+        available_locales.each do |locale|
+          urls << "#{@host}/#{locale}/cities/#{art.city.id}/weather_arts/#{art.id}"
+        end
+      end
+    end
+
+    urls.uniq
+  end
+
+  def submit_urls(urls)
+    return if urls.empty?
+
+    # 随机选择一个 IndexNow 端点
+    endpoint = INDEXNOW_ENDPOINTS.sample
+
+    # 准备提交的数据
+    data = {
+      host: URI.parse(@host).host,
+      key: INDEXNOW_KEY,
+      urlList: urls
+    }
+
+    # 发送请求到 IndexNow API
+    uri = URI.parse(endpoint)
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl = true
+
+    request = Net::HTTP::Post.new(uri.path, "Content-Type" => "application/json")
+    request.body = data.to_json
+
+    response = http.request(request)
+
+    unless response.code.to_i == 200
+      Rails.logger.warn "IndexNow submission failed with code #{response.code}: #{response.body}"
+      # 如果第一个端点失败，尝试使用备用端点
+      if endpoint == INDEXNOW_ENDPOINTS.first
+        retry_with_alternate_endpoint(urls)
+      end
+    end
+  end
+
+  def retry_with_alternate_endpoint(urls)
+    # 使用备用端点
+    endpoint = INDEXNOW_ENDPOINTS.last
+
+    data = {
+      host: URI.parse(@host).host,
+      key: INDEXNOW_KEY,
+      urlList: urls
+    }
+
+    uri = URI.parse(endpoint)
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl = true
+
+    request = Net::HTTP::Post.new(uri.path, "Content-Type" => "application/json")
+    request.body = data.to_json
+
+    response = http.request(request)
+
+    unless response.code.to_i == 200
+      Rails.logger.error "All IndexNow submissions failed. Last error: #{response.code}: #{response.body}"
+    end
+  end
+end

config/routes.rb

@@ -44,6 +44,9 @@ Rails.application.routes.draw do
     # Can be used by load balancers and uptime monitors to verify that the app is live.
     get "up" => "rails/health#show", as: :rails_health_check
 
+    # IndexNow key验证
+    get "/:id.txt", to: "key_verification#show", constraints: { id: /\w{32}/ }
+
     # Render dynamic PWA files from app/views/pwa/* (remember to link manifest in application.html.erb)
     # get "manifest" => "rails/pwa#manifest", as: :pwa_manifest
     # get "service-worker" => "rails/pwa#service_worker", as: :pwa_service_worker