BrunoGao
904b0698ab
- 新增持续学习系统完整架构 - 添加AI训练自动化调度器和模型版本管理 - 优化Docker配置支持实时系统 - 新增数据源管理系统集成 - 改进WebSocket和API端口配置 - 添加系统监控和健康检查功能
398 lines
15 KiB
Python
398 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
离线数据初始化器
|
|
不依赖TuShare API，使用预设数据快速初始化数据库
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import logging
|
|
import pymysql
|
|
from datetime import datetime, timedelta
|
|
import pandas as pd
|
|
import numpy as np
|
|
from config_manager import get_config
|
|
|
|
class OfflineDataInitializer:
|
|
"""离线数据初始化器"""
|
|
|
|
def __init__(self):
|
|
self.config = get_config()
|
|
self.setup_logging()
|
|
|
|
# 数据库连接参数
|
|
self.db_config = {
|
|
'host': self.config.database.host,
|
|
'port': self.config.database.port,
|
|
'user': self.config.database.user,
|
|
'password': self.config.database.password,
|
|
'database': self.config.database.database,
|
|
'charset': self.config.database.charset
|
|
}
|
|
|
|
self.logger.info("📊 离线数据初始化器已启动")
|
|
|
|
def setup_logging(self):
|
|
"""设置日志"""
|
|
os.makedirs('logs', exist_ok=True)
|
|
|
|
log_file = f'logs/offline_init_{datetime.now().strftime("%Y%m%d_%H%M%S")}.log'
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format='%(asctime)s - %(levelname)s - %(message)s',
|
|
handlers=[
|
|
logging.FileHandler(log_file, encoding='utf-8'),
|
|
logging.StreamHandler()
|
|
]
|
|
)
|
|
|
|
self.logger = logging.getLogger(__name__)
|
|
|
|
def create_sample_stock_data(self):
|
|
"""创建示例股票数据"""
|
|
# A股主要股票列表
|
|
sample_stocks = [
|
|
('000001.SZ', '平安银行', '银行', '金融', 'SZ', 'stock', '深圳'),
|
|
('000002.SZ', '万科A', '房地产开发', '房地产', 'SZ', 'stock', '深圳'),
|
|
('000858.SZ', '五粮液', '白酒', '食品饮料', 'SZ', 'stock', '四川'),
|
|
('002415.SZ', '海康威视', '安防设备', '电子', 'SZ', 'stock', '浙江'),
|
|
('600000.SH', '浦发银行', '银行', '金融', 'SH', 'stock', '上海'),
|
|
('600036.SH', '招商银行', '银行', '金融', 'SH', 'stock', '上海'),
|
|
('600519.SH', '贵州茅台', '白酒', '食品饮料', 'SH', 'stock', '贵州'),
|
|
('600887.SH', '伊利股份', '乳品', '食品饮料', 'SH', 'stock', '内蒙古'),
|
|
('002230.SZ', '科大讯飞', '人工智能', '计算机', 'SZ', 'stock', '安徽'),
|
|
('002594.SZ', '比亚迪', '新能源汽车', '汽车', 'SZ', 'stock', '广东'),
|
|
('300015.SZ', '爱尔眼科', '医疗服务', '医药生物', 'SZ', 'stock', '湖南'),
|
|
('300059.SZ', '东方财富', '证券', '非银金融', 'SZ', 'stock', '上海'),
|
|
('300142.SZ', '沃森生物', '疫苗', '医药生物', 'SZ', 'stock', '云南'),
|
|
('300760.SZ', '迈瑞医疗', '医疗器械', '医药生物', 'SZ', 'stock', '广东'),
|
|
('002304.SZ', '洋河股份', '白酒', '食品饮料', 'SZ', 'stock', '江苏'),
|
|
]
|
|
|
|
# 创建DataFrame
|
|
stock_df = pd.DataFrame(sample_stocks, columns=[
|
|
'stock_code', 'stock_name', 'industry', 'sector', 'market', 'stock_type', 'area'
|
|
])
|
|
|
|
# 添加上市日期
|
|
stock_df['list_date'] = pd.date_range(start='2010-01-01', periods=len(stock_df), freq='30D')
|
|
stock_df['delist_date'] = None
|
|
stock_df['is_active'] = 1
|
|
stock_df['created_at'] = datetime.now()
|
|
stock_df['updated_at'] = datetime.now()
|
|
|
|
return stock_df
|
|
|
|
def generate_daily_data(self, stock_code: str, days: int = 1000):
|
|
"""生成日线数据"""
|
|
# 基础价格和波动
|
|
base_price = np.random.uniform(10, 100)
|
|
dates = pd.date_range(end=datetime.now().date(), periods=days, freq='D')
|
|
|
|
# 生成价格序列（随机游走）
|
|
returns = np.random.normal(0.001, 0.02, days) # 平均0.1%收益，2%波动
|
|
prices = [base_price]
|
|
|
|
for i in range(1, days):
|
|
new_price = prices[-1] * (1 + returns[i])
|
|
prices.append(max(new_price, 0.01)) # 确保价格为正
|
|
|
|
# 创建OHLC数据
|
|
daily_data = []
|
|
for i, date in enumerate(dates):
|
|
if date.weekday() >= 5: # 跳过周末
|
|
continue
|
|
|
|
price = prices[i]
|
|
volatility = np.random.uniform(0.005, 0.03)
|
|
|
|
high = price * (1 + volatility/2)
|
|
low = price * (1 - volatility/2)
|
|
open_price = price * np.random.uniform(0.98, 1.02)
|
|
close_price = price
|
|
|
|
volume = np.random.randint(1000000, 50000000)
|
|
amount = volume * close_price
|
|
|
|
# 计算涨跌幅
|
|
if i > 0:
|
|
pre_close = prices[i-1]
|
|
change = close_price - pre_close
|
|
pct_chg = (change / pre_close) * 100
|
|
else:
|
|
pre_close = close_price
|
|
change = 0
|
|
pct_chg = 0
|
|
|
|
daily_data.append({
|
|
'stock_code': stock_code,
|
|
'trade_date': date,
|
|
'open': round(open_price, 2),
|
|
'high': round(high, 2),
|
|
'low': round(low, 2),
|
|
'close': round(close_price, 2),
|
|
'pre_close': round(pre_close, 2),
|
|
'change': round(change, 2),
|
|
'pct_chg': round(pct_chg, 2),
|
|
'volume': volume,
|
|
'amount': round(amount, 2),
|
|
'created_at': datetime.now()
|
|
})
|
|
|
|
return pd.DataFrame(daily_data)
|
|
|
|
def init_stock_basic(self):
|
|
"""初始化股票基本信息"""
|
|
self.logger.info("📋 初始化股票基本信息...")
|
|
|
|
try:
|
|
conn = pymysql.connect(**self.db_config)
|
|
cursor = conn.cursor()
|
|
|
|
# 清空现有数据
|
|
cursor.execute("TRUNCATE TABLE stock_basic")
|
|
|
|
# 获取示例数据
|
|
stock_df = self.create_sample_stock_data()
|
|
|
|
# 插入数据
|
|
insert_sql = """
|
|
INSERT INTO stock_basic (
|
|
stock_code, stock_name, industry, sector, market,
|
|
stock_type, area, list_date, delist_date, is_active,
|
|
created_at, updated_at
|
|
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
|
"""
|
|
|
|
for _, row in stock_df.iterrows():
|
|
cursor.execute(insert_sql, (
|
|
row['stock_code'], row['stock_name'], row['industry'],
|
|
row['sector'], row['market'], row['stock_type'],
|
|
row['area'], row['list_date'], row['delist_date'],
|
|
row['is_active'], row['created_at'], row['updated_at']
|
|
))
|
|
|
|
conn.commit()
|
|
cursor.close()
|
|
conn.close()
|
|
|
|
self.logger.info(f"✅ 股票基本信息初始化完成: {len(stock_df)} 只股票")
|
|
return True
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"❌ 股票基本信息初始化失败: {e}")
|
|
return False
|
|
|
|
def init_daily_data(self):
|
|
"""初始化日线数据"""
|
|
self.logger.info("📈 初始化日线数据...")
|
|
|
|
try:
|
|
conn = pymysql.connect(**self.db_config)
|
|
cursor = conn.cursor()
|
|
|
|
# 获取股票列表
|
|
cursor.execute("SELECT stock_code, stock_name FROM stock_basic WHERE is_active = 1")
|
|
stocks = cursor.fetchall()
|
|
|
|
# 清空现有数据
|
|
cursor.execute("TRUNCATE TABLE stock_daily")
|
|
|
|
total_records = 0
|
|
|
|
for stock_code, stock_name in stocks:
|
|
self.logger.info(f" 📊 生成 {stock_name}({stock_code}) 日线数据...")
|
|
|
|
# 生成5年数据
|
|
daily_df = self.generate_daily_data(stock_code, days=1200)
|
|
|
|
# 插入数据
|
|
insert_sql = """
|
|
INSERT INTO stock_daily (
|
|
stock_code, trade_date, open, high, low, close,
|
|
pre_close, `change`, pct_chg, volume, amount, created_at
|
|
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
|
"""
|
|
|
|
for _, row in daily_df.iterrows():
|
|
cursor.execute(insert_sql, (
|
|
row['stock_code'], row['trade_date'], row['open'],
|
|
row['high'], row['low'], row['close'], row['pre_close'],
|
|
row['change'], row['pct_chg'], row['volume'],
|
|
row['amount'], row['created_at']
|
|
))
|
|
|
|
total_records += len(daily_df)
|
|
|
|
# 每处理一只股票提交一次
|
|
conn.commit()
|
|
self.logger.info(f" ✅ {stock_name} 完成: {len(daily_df)} 条记录")
|
|
|
|
cursor.close()
|
|
conn.close()
|
|
|
|
self.logger.info(f"✅ 日线数据初始化完成: {total_records} 条记录")
|
|
return True
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"❌ 日线数据初始化失败: {e}")
|
|
return False
|
|
|
|
def create_additional_tables(self):
|
|
"""创建额外的数据表"""
|
|
self.logger.info("🔧 创建额外数据表...")
|
|
|
|
try:
|
|
conn = pymysql.connect(**self.db_config)
|
|
cursor = conn.cursor()
|
|
|
|
# 技术指标表
|
|
cursor.execute("""
|
|
CREATE TABLE IF NOT EXISTS technical_indicators (
|
|
id BIGINT AUTO_INCREMENT PRIMARY KEY,
|
|
stock_code VARCHAR(20) NOT NULL,
|
|
trade_date DATE NOT NULL,
|
|
ma5 DECIMAL(10,3),
|
|
ma10 DECIMAL(10,3),
|
|
ma20 DECIMAL(10,3),
|
|
ma60 DECIMAL(10,3),
|
|
rsi DECIMAL(8,4),
|
|
macd DECIMAL(8,4),
|
|
macd_signal DECIMAL(8,4),
|
|
macd_hist DECIMAL(8,4),
|
|
bb_upper DECIMAL(10,3),
|
|
bb_middle DECIMAL(10,3),
|
|
bb_lower DECIMAL(10,3),
|
|
kdj_k DECIMAL(8,4),
|
|
kdj_d DECIMAL(8,4),
|
|
kdj_j DECIMAL(8,4),
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
UNIQUE KEY uk_code_date (stock_code, trade_date),
|
|
INDEX idx_code (stock_code),
|
|
INDEX idx_date (trade_date)
|
|
) COMMENT='技术指标数据表'
|
|
""")
|
|
|
|
# 财务数据表
|
|
cursor.execute("""
|
|
CREATE TABLE IF NOT EXISTS financial_data (
|
|
id BIGINT AUTO_INCREMENT PRIMARY KEY,
|
|
stock_code VARCHAR(20) NOT NULL,
|
|
report_date DATE NOT NULL,
|
|
total_revenue DECIMAL(20,2),
|
|
net_profit DECIMAL(20,2),
|
|
total_assets DECIMAL(20,2),
|
|
total_equity DECIMAL(20,2),
|
|
roe DECIMAL(8,4),
|
|
roa DECIMAL(8,4),
|
|
eps DECIMAL(8,4),
|
|
bps DECIMAL(8,4),
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
UNIQUE KEY uk_code_date (stock_code, report_date),
|
|
INDEX idx_code (stock_code),
|
|
INDEX idx_date (report_date)
|
|
) COMMENT='财务数据表'
|
|
""")
|
|
|
|
# 股票推荐表
|
|
cursor.execute("""
|
|
CREATE TABLE IF NOT EXISTS stock_recommendations (
|
|
id BIGINT AUTO_INCREMENT PRIMARY KEY,
|
|
stock_code VARCHAR(20) NOT NULL,
|
|
recommendation_type ENUM('BUY', 'SELL', 'HOLD') NOT NULL,
|
|
confidence_score DECIMAL(5,4),
|
|
target_price DECIMAL(10,3),
|
|
analysis_text TEXT,
|
|
strategy_type VARCHAR(50),
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
valid_until TIMESTAMP,
|
|
status ENUM('ACTIVE', 'EXPIRED', 'EXECUTED') DEFAULT 'ACTIVE',
|
|
INDEX idx_code (stock_code),
|
|
INDEX idx_type (recommendation_type),
|
|
INDEX idx_status (status),
|
|
INDEX idx_created (created_at)
|
|
) COMMENT='股票推荐表'
|
|
""")
|
|
|
|
conn.commit()
|
|
cursor.close()
|
|
conn.close()
|
|
|
|
self.logger.info("✅ 额外数据表创建完成")
|
|
return True
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"❌ 创建额外数据表失败: {e}")
|
|
return False
|
|
|
|
def run_initialization(self):
|
|
"""运行完整初始化"""
|
|
self.logger.info("🚀 开始离线数据初始化...")
|
|
|
|
success_count = 0
|
|
|
|
# 1. 创建额外表
|
|
if self.create_additional_tables():
|
|
success_count += 1
|
|
|
|
# 2. 初始化股票基本信息
|
|
if self.init_stock_basic():
|
|
success_count += 1
|
|
|
|
# 3. 初始化日线数据
|
|
if self.init_daily_data():
|
|
success_count += 1
|
|
|
|
# 生成报告
|
|
self.generate_report(success_count)
|
|
|
|
return success_count == 3
|
|
|
|
def generate_report(self, success_count: int):
|
|
"""生成初始化报告"""
|
|
report = {
|
|
'initialization_time': datetime.now().isoformat(),
|
|
'success_modules': success_count,
|
|
'total_modules': 3,
|
|
'success_rate': success_count / 3 * 100,
|
|
'initialized_data': [
|
|
'股票基本信息',
|
|
'5年日线数据',
|
|
'技术指标表结构',
|
|
'财务数据表结构',
|
|
'推荐数据表结构'
|
|
]
|
|
}
|
|
|
|
self.logger.info("🎉 离线数据初始化完成！")
|
|
self.logger.info(f"📊 成功模块: {success_count}/3")
|
|
self.logger.info(f"✅ 成功率: {success_count/3*100:.1f}%")
|
|
|
|
if success_count == 3:
|
|
self.logger.info("🎯 所有模块初始化成功，系统可以正常使用")
|
|
else:
|
|
self.logger.warning("⚠️ 部分模块初始化失败，请检查错误日志")
|
|
|
|
def main():
|
|
"""主函数"""
|
|
try:
|
|
initializer = OfflineDataInitializer()
|
|
success = initializer.run_initialization()
|
|
|
|
if success:
|
|
print("\n✅ 离线数据初始化成功！")
|
|
print("📊 系统已准备就绪，可以开始使用")
|
|
else:
|
|
print("\n❌ 离线数据初始化失败！")
|
|
print("📋 请检查日志文件了解详细错误信息")
|
|
|
|
except KeyboardInterrupt:
|
|
print("\n⚠️ 用户中断操作")
|
|
except Exception as e:
|
|
print(f"❌ 初始化失败: {e}")
|
|
import traceback
|
|
traceback.print_exc()
|
|
|
|
if __name__ == "__main__":
|
|
main() |