today_ai_weather/lib/tasks/sync_geo_data.rake
songtianlun 0e476b546d
Some checks failed
Docker Dev / docker (push) Has been cancelled
CI / scan_ruby (push) Has been cancelled
CI / lint (push) Has been cancelled
CI / test (push) Has been cancelled
Docker Main / docker (push) Has been cancelled
feat: add locking mechanism to batch task worker
- Introduce a Redis-based lock to prevent concurrent execution of
  batch generation tasks.
- Set a TTL of 300 seconds for the lock to ensure it is released
  after a timeout.
- Add logging for situations where a task is already in progress.

This enhancement ensures that batch tasks do not overlap, which can
lead to data inconsistencies and resource contention. The locking
mechanism improves the reliability of the batch processing system.
2025-02-22 15:41:42 +08:00

196 lines
5.7 KiB
Ruby

# lib/tasks/sync_geo_data.rake
require "json"
namespace :geo do
desc "Sync geographical data from JSON"
task sync: :environment do
# 定义 JSON 文件路径
base_path = Rails.root.join("lib", "data")
# 同步顺序很重要:先同步 Regions -> Subregions -> Countries -> States -> Cities
sync_regions(base_path)
sync_subregions(base_path)
sync_countries(base_path)
sync_states(base_path)
sync_cities(base_path)
end
def sync_regions(base_path)
file_path = base_path.join("regions.json")
regions = JSON.parse(File.read(file_path))
sum = regions.count
count = 1
regions.each do |data|
region = Region.find_or_create_by!(id: data["id"]) do |r|
r.name = data["name"]
r.code = data["name"]
end
puts "Sync Regions[#{count}/#{sum}]: [#{region.name}]"
count += 1
region.update!(
name: data["name"],
code: data["name"],
translations: data["translations"].to_json.to_s,
flag: data["flag"] || true,
wiki_data_id: data["wikiDataId"]
)
end
end
def sync_subregions(base_path)
file_path = base_path.join("subregions.json")
subregions = JSON.parse(File.read(file_path))
sum = subregions.count
count = 1
subregions.each do |data|
region = Region.find_by!(id: data["region_id"])
subregion = Subregion.find_or_create_by!(id: data["id"]) do |s|
s.name = data["name"]
s.region_id = region.id
end
puts "Sync Subregions[#{count}/#{sum}]: [#{subregion.name}] region:[#{region.name}]"
count += 1
subregion.update!(
translations: data["translations"].to_json,
flag: data["flag"] || true,
wiki_data_id: data["wikiDataId"]
)
end
end
def sync_countries(base_path)
file_path = base_path.join("countries.json")
countries = JSON.parse(File.read(file_path))
sum = countries.count
count = 1
countries.each do |data|
# 处理 Region
region = if data["region_id"]
Region.find_by(id: data["region_id"])
elsif data["region"]
Region.find_by(name: data["region"])
end
# 处理 Subregion
subregion = nil
if data["subregion_id"].present?
subregion = Subregion.find_by(id: data["subregion_id"])
elsif data["subregion"].present?
subregion = Subregion.find_by(name: data["subregion"])
end
puts "Syncing Country[#{count}/#{sum}] [#{data["name"]}] region: [#{region&.name}] subregion: [#{data["subregion"]}]"
count += 1
# 查找或初始化 Country
country = Country.find_or_create_by!(code: data["iso2"]) do |c|
c.name = data["name"]
end
# 更新 Country 属性
country.update!(
name: data["name"],
iso3: data["iso3"],
numeric_code: data["numeric_code"],
iso2: data["iso2"],
code: data["iso2"],
phonecode: data["phonecode"],
capital: data["capital"],
currency: data["currency"],
currency_name: data["currency_name"],
currency_symbol: data["currency_symbol"],
tld: data["tld"],
native: data["native"],
nationality: data["nationality"],
timezones: data["timezones"].to_json,
translations: data["translations"].to_json,
latitude: data["latitude"],
longitude: data["longitude"],
emoji: data["emoji"],
emoji_u: data["emojiU"],
flag: data["flag"] || true,
wiki_data_id: data["wikiDataId"]
)
country.update!(region: region) if region != nil
country.update!(subregion: subregion) if subregion != nil
end
end
def sync_states(base_path)
file_path = base_path.join("states.json")
states = JSON.parse(File.read(file_path))
sum = states.count
count = 1
states.each do |data|
puts "Syncing State[#{count}/#{sum}] [#{data["name"]}] country:[#{data["country_name"]}]"
count += 1
country =
data["country_code"] ?
Country.find_by!(code: data["country_code"]) :
Country.find_by!(name: data["country_name"])
state = State.find_or_create_by!(name: data["name"]) do |s|
s.country_id = country.id
s.country_code = country.code
end
state.update!(
country_code: country.code,
fips_code: data["fips_code"],
iso2: data["iso2"],
code: data["state_code"],
state_type: data["type"],
level: data["level"],
parent_id: data["parent_id"],
latitude: data["latitude"],
longitude: data["longitude"],
flag: data["flag"] || true,
wiki_data_id: data["wikiDataId"]
)
end
end
def sync_cities(base_path)
file_path = base_path.join("cities.json")
cities = JSON.parse(File.read(file_path))
sum = cities.count
count = 1
cities.each do |data|
country = Country.find_by!(code: data["country_code"])
state = State.find_by(code: data["state_code"])
city = City.find_or_create_by!(name: data["name"]) do |c|
c.country_id = country.id,
c.country_code = country.code,
c.latitude = data["latitude"],
c.longitude = data["longitude"]
end
puts "Syncing City[#{count}/#{sum}] [#{data["name"]}] Country:[#{country&.name}] "
count += 1
city.update!(
flag: data["flag"] || true,
wiki_data_id: data["wikiDataId"]
)
if country != nil
city.update(
country: country,
country_code: country.code,
)
end
city.update!(state: state) unless state == nil
city.update!(state_code: state.code) unless state == nil
city.update!(active: false) if city.active == nil
city.update!(priority: 10) if city.priority == nil
end
end
end