commit d340cf609da00c1ced36c29cb09b6c151bb34381 Author: shellway-pc <413209390@qq.com> Date: Thu Jul 30 22:11:25 2026 +0800 第一次提交，数据库创建，初始化项目 diff --git a/quantitative_data/.ipynb_checkpoints/config-checkpoint.py b/quantitative_data/.ipynb_checkpoints/config-checkpoint.py new file mode 100644 index 0000000..d1f9638 --- /dev/null +++ b/quantitative_data/.ipynb_checkpoints/config-checkpoint.py @@ -0,0 +1,19 @@ +""" +量化数据库配置文件 +""" +# PostgreSQL Docker 连接配置 +DB_CONFIG = { + "host": "192.168.27.15", + "port": 12345, + "database": "quant_db", + "user": "postgres", + "password": "postgres", # 请修改为实际密码 +} + +# Tushare API Token +TUSHARE_TOKEN = "你的token" # 请替换为你的tushare token + +# 批量导入参数 +BATCH_SIZE = 5000 # 每批次插入行数 +START_DATE = "2010-01-01" # 数据起始日期 +END_DATE = "2025-12-31" # 数据结束日期 \ No newline at end of file diff --git a/quantitative_data/.ipynb_checkpoints/importer-checkpoint.py b/quantitative_data/.ipynb_checkpoints/importer-checkpoint.py new file mode 100644 index 0000000..5d0c912 --- /dev/null +++ b/quantitative_data/.ipynb_checkpoints/importer-checkpoint.py @@ -0,0 +1,983 @@ +""" +量化数据导入核心模块 +连接 Docker PostgreSQL (192.168.27.15:12345) +从 Tushare 拉取数据并批量导入 +""" +import time +import logging +from datetime import datetime, timedelta +from typing import Optional, List, Dict + +import tushare as ts +import pandas as pd +import psycopg2 +from psycopg2 import sql +from psycopg2.extras import execute_values +from sqlalchemy import create_engine + +from config import DB_CONFIG, TUSHARE_TOKEN, BATCH_SIZE, START_DATE, END_DATE + +# ============================================================ +# 日志配置 +# ============================================================ +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + handlers=[ + logging.FileHandler("import_data.log", encoding="utf-8"), + logging.StreamHandler(), + ], +) +logger = logging.getLogger(__name__) + +# ============================================================ +# 初始化连接 +# ============================================================ + +# psycopg2 原生连接 (用于执行 DDL / 精细控制) +def get_pg_connection(): + """获取 psycopg2 原生连接""" + return psycopg2.connect( + host=DB_CONFIG["host"], + port=DB_CONFIG["port"], + database=DB_CONFIG["database"], + user=DB_CONFIG["user"], + password=DB_CONFIG["password"], + ) + + +# SQLAlchemy 引擎 (用于 DataFrame.to_sql) +_sqlalchemy_engine = None + + +def get_sqlalchemy_engine(): + """获取 SQLAlchemy 引擎 (单例)""" + global _sqlalchemy_engine + if _sqlalchemy_engine is None: + db_url = ( + f"postgresql://{DB_CONFIG['user']}:{DB_CONFIG['password']}" + f"@{DB_CONFIG['host']}:{DB_CONFIG['port']}/{DB_CONFIG['database']}" + ) + _sqlalchemy_engine = create_engine(db_url, pool_size=5, max_overflow=10) + return _sqlalchemy_engine + + +# Tushare Pro API (单例) +_ts_pro = None + + +def get_ts_pro(): + """获取 Tushare Pro API 实例 (单例)""" + global _ts_pro + if _ts_pro is None: + ts.set_token(TUSHARE_TOKEN) + _ts_pro = ts.pro_api() + logger.info("Tushare Pro API 初始化完成") + return _ts_pro + + +# ============================================================ +# 通用导入工具函数 +# ============================================================ + +def normalize_columns(df: pd.DataFrame) -> pd.DataFrame: + """ + 标准化列名：Tushare 返回的列名可能有大小写差异，统一转小写 + """ + df.columns = [c.lower() for c in df.columns] + return df + + +def safe_float(val): + """安全转换为浮点数，NaN -> None""" + try: + if pd.isna(val): + return None + return float(val) + except (ValueError, TypeError): + return None + + +def batch_insert(table_name: str, df: pd.DataFrame, conn, conflict_columns: List[str]): + """ + 使用 execute_values 批量 UPSERT (INSERT ... ON CONFLICT) + - table_name: 目标表名 + - df: 待导入 DataFrame + - conn: psycopg2 连接 + - conflict_columns: 冲突列 (唯一约束列)，冲突时更新其他列 + """ + if df.empty: + logger.warning(f" {table_name}: 空数据，跳过") + return 0 + + columns = list(df.columns) + rows = [tuple(row) for row in df.itertuples(index=False)] + + # 构建 ON CONFLICT 子句 + conflict_str = ", ".join(conflict_columns) + # 构建 UPDATE SET 子句 (排除冲突列) + update_cols = [c for c in columns if c not in conflict_columns] + if not update_cols: + # 没有需要更新的列，使用 DO NOTHING + upsert_sql = sql.SQL( + "INSERT INTO {table} ({cols}) VALUES %s " + "ON CONFLICT ({conflict}) DO NOTHING" + ).format( + table=sql.Identifier(table_name), + cols=sql.SQL(", ").join(map(sql.Identifier, columns)), + conflict=sql.SQL(conflict_str), + ) + else: + update_set = sql.SQL(", ").join( + sql.SQL("{col} = EXCLUDED.{col}").format(col=sql.Identifier(c)) + for c in update_cols + ) + upsert_sql = sql.SQL( + "INSERT INTO {table} ({cols}) VALUES %s " + "ON CONFLICT ({conflict}) DO UPDATE SET {update_set}" + ).format( + table=sql.Identifier(table_name), + cols=sql.SQL(", ").join(map(sql.Identifier, columns)), + conflict=sql.SQL(conflict_str), + update_set=update_set, + ) + + cursor = conn.cursor() + try: + execute_values(cursor, upsert_sql.as_string(cursor), rows, page_size=BATCH_SIZE) + conn.commit() + logger.info(f" {table_name}: 成功导入 {len(rows)} 条记录") + return len(rows) + except Exception as e: + conn.rollback() + logger.error(f" {table_name}: 批量导入失败 - {e}") + raise + finally: + cursor.close() + + +def fetch_with_retry(fetch_func, max_retries: int = 3, delay: float = 2.0): + """ + 带重试的数据获取装饰器 + - fetch_func: 数据获取函数 (返回 DataFrame) + - max_retries: 最大重试次数 + - delay: 重试间隔(秒) + """ + for attempt in range(max_retries): + try: + result = fetch_func() + if result is not None and not result.empty: + return result + logger.warning(f" 第 {attempt+1} 次获取返回空数据，重试...") + except Exception as e: + logger.warning(f" 第 {attempt+1} 次获取失败: {e}") + if attempt < max_retries - 1: + time.sleep(delay * (attempt + 1)) # 递增延迟 + return pd.DataFrame() + + +# ============================================================ +# 1. 导入股票基本信息 +# ============================================================ + +def import_stock_basic(): + """ + 导入股票基本信息 (stock_basic) + Tushare: stock_basic + """ + logger.info("=" * 60) + logger.info("[1/7] 导入股票基本信息 (stock_basic) ...") + + pro = get_ts_pro() + conn = get_pg_connection() + + try: + # 获取全量股票基本信息 + df = pro.stock_basic( + exchange="", + list_status="L", + fields="ts_code,symbol,name,area,industry,market,list_status,list_date,is_hs,act_name,act_ent_type", + ) + if df is None or df.empty: + logger.warning("未获取到股票基本信息") + return + + df = normalize_columns(df) + + # 转换日期格式 + if "list_date" in df.columns: + df["list_date"] = pd.to_datetime(df["list_date"], format="%Y%m%d", errors="coerce") + + logger.info(f" 获取到 {len(df)} 条股票基本信息") + + # 批量导入 + conflict_cols = ["ts_code"] + batch_insert("stock_basic", df, conn, conflict_cols) + + # 也尝试获取退市的股票 + try: + df_d = pro.stock_basic( + exchange="", + list_status="D", + fields="ts_code,symbol,name,area,industry,market,list_status,list_date,is_hs", + ) + if df_d is not None and not df_d.empty: + df_d = normalize_columns(df_d) + if "list_date" in df_d.columns: + df_d["list_date"] = pd.to_datetime( + df_d["list_date"], format="%Y%m%d", errors="coerce" + ) + batch_insert("stock_basic", df_d, conn, conflict_cols) + logger.info(f" 额外导入 {len(df_d)} 条退市股票信息") + except Exception as e: + logger.warning(f" 获取退市股票信息失败: {e}") + + except Exception as e: + logger.error(f" 导入股票基本信息失败: {e}") + raise + finally: + conn.close() + + +# ============================================================ +# 2. 导入交易日历 +# ============================================================ + +def import_trade_cal(start_date: str = None, end_date: str = None): + """ + 导入交易日历 (trade_cal) + Tushare: trade_cal + """ + if start_date is None: + start_date = START_DATE + if end_date is None: + end_date = END_DATE + + logger.info("=" * 60) + logger.info(f"[2/7] 导入交易日历 (trade_cal): {start_date} ~ {end_date}") + + pro = get_ts_pro() + conn = get_pg_connection() + + try: + # 上交所 + for exchange, ex_name in [("SSE", "上交所"), ("SZSE", "深交所")]: + df = pro.trade_cal( + exchange=exchange, + start_date=start_date.replace("-", ""), + end_date=end_date.replace("-", ""), + ) + if df is None or df.empty: + logger.warning(f" {ex_name} 交易日历为空") + continue + + df = normalize_columns(df) + + # 转换日期 + for col in ["cal_date", "pretrade_date"]: + if col in df.columns: + df[col] = pd.to_datetime(df[col], format="%Y%m%d", errors="coerce") + + # 重命名 is_open (Tushare 返回 0/1 整数) + if "is_open" in df.columns: + df["is_open"] = df["is_open"].astype(int) + + conflict_cols = ["exchange", "cal_date"] + batch_insert("trade_cal", df, conn, conflict_cols) + logger.info(f" {ex_name}: {len(df)} 条交易日历") + + except Exception as e: + logger.error(f" 导入交易日历失败: {e}") + raise + finally: + conn.close() + + time.sleep(0.5) # API 频率限制 + + +# ============================================================ +# 3. 导入日线行情 (核心表) +# ============================================================ + +def import_daily_for_stock(ts_code: str, start_date: str, end_date: str, conn) -> int: + """ + 导入单只股票的日线行情 + 返回导入的记录数 + """ + pro = get_ts_pro() + + def fetch(): + return pro.daily( + ts_code=ts_code, + start_date=start_date.replace("-", ""), + end_date=end_date.replace("-", ""), + ) + + df = fetch_with_retry(fetch, max_retries=2) + if df is None or df.empty: + return 0 + + df = normalize_columns(df) + + # 转换日期 + if "trade_date" in df.columns: + df["trade_date"] = pd.to_datetime(df["trade_date"], format="%Y%m%d", errors="coerce") + + # 数值列处理 NaN + numeric_cols = [ + "open", "high", "low", "close", "pre_close", "change", "pct_chg", + "vol", "amount", + ] + for col in numeric_cols: + if col in df.columns: + df[col] = pd.to_numeric(df[col], errors="coerce") + + # 额外列 (Tushare Pro 不同版本返回字段可能不同) + for col in ["turnover_rate", "volume_ratio", "ma5", "ma10", "ma20", "ma_v_5", "ma_v_10", "ma_v_20"]: + if col not in df.columns: + df[col] = None + + conflict_cols = ["ts_code", "trade_date"] + return batch_insert("daily", df, conn, conflict_cols) + + +def import_daily_batch( + stock_list: List[str], + start_date: str = None, + end_date: str = None, + sleep_interval: float = 0.3, +): + """ + 批量导入多只股票的日线行情 + - stock_list: 股票代码列表 + - sleep_interval: API 调用间隔 (避免频率限制) + """ + if start_date is None: + start_date = START_DATE + if end_date is None: + end_date = END_DATE + + total = len(stock_list) + logger.info("=" * 60) + logger.info( + f"[3/7] 导入日线行情 (daily): {start_date} ~ {end_date}, " + f"共 {total} 只股票" + ) + + conn = get_pg_connection() + success_count = 0 + fail_list = [] + + for i, ts_code in enumerate(stock_list, 1): + try: + n = import_daily_for_stock(ts_code, start_date, end_date, conn) + if n > 0: + success_count += 1 + if i % 50 == 0 or i == total: + logger.info(f" 进度: {i}/{total} 成功={success_count} 失败={len(fail_list)}") + except Exception as e: + logger.error(f" [{ts_code}] 导入失败: {e}") + fail_list.append(ts_code) + conn.rollback() + + time.sleep(sleep_interval) # API 频率控制 + + conn.close() + logger.info(f" 日线行情导入完成: 成功 {success_count}/{total}") + + if fail_list: + logger.warning(f" 失败列表({len(fail_list)}): {fail_list[:20]}...") + return fail_list + + +def import_daily_by_year( + stock_list: List[str], + start_year: int = 2010, + end_year: int = 2025, +): + """ + 按年份逐批导入日线行情 (断点续传友好) + 适合大数据量导入，每年每只股票可单独重试 + """ + logger.info("=" * 60) + logger.info( + f"[3/7] 按年导入日线行情: {start_year} ~ {end_year}, " + f"共 {len(stock_list)} 只股票" + ) + + total_imported = 0 + for year in range(start_year, end_year + 1): + year_start = f"{year}-01-01" + year_end = f"{year}-12-31" + logger.info(f"--- 导入 {year} 年日线行情 ---") + fail_list = import_daily_batch( + stock_list, + start_date=year_start, + end_date=year_end, + sleep_interval=0.2, + ) + total_imported += 1 + logger.info(f" {year} 年完成\n") + + logger.info(f" 所有年份日线行情导入完成!") + + +# ============================================================ +# 4. 导入每日指标 (daily_basic) - 估值/基本面 +# ============================================================ + +def import_daily_basic( + ts_code: str = None, + trade_date: str = None, + start_date: str = None, + end_date: str = None, + conn=None, +): + """ + 导入每日指标 (daily_basic) + Tushare: daily_basic + - 支持按单只股票导入 + - 支持按日期范围全市场导入 + """ + if start_date is None: + start_date = START_DATE + if end_date is None: + end_date = END_DATE + + pro = get_ts_pro() + own_conn = conn is None + if own_conn: + conn = get_pg_connection() + + try: + # 构建参数 + kwargs = { + "start_date": start_date.replace("-", ""), + "end_date": end_date.replace("-", ""), + } + if ts_code: + kwargs["ts_code"] = ts_code + if trade_date: + kwargs["trade_date"] = trade_date.replace("-", "") + + def fetch(): + return pro.daily_basic(**kwargs) + + df = fetch_with_retry(fetch, max_retries=2) + if df is None or df.empty: + return 0 + + df = normalize_columns(df) + + if "trade_date" in df.columns: + df["trade_date"] = pd.to_datetime(df["trade_date"], format="%Y%m%d", errors="coerce") + + numeric_cols = df.select_dtypes(include=["number"]).columns.tolist() + for col in numeric_cols: + df[col] = pd.to_numeric(df[col], errors="coerce") + + conflict_cols = ["ts_code", "trade_date"] + return batch_insert("daily_basic", df, conn, conflict_cols) + + finally: + if own_conn: + conn.close() + + +def import_daily_basic_by_date( + start_date: str = None, + end_date: str = None, +): + """ + 按日期批量导入每日指标 (全市场) + Tushare daily_basic 接口可按交易日获取全市场数据，比较高效 + """ + if start_date is None: + start_date = START_DATE + if end_date is None: + end_date = END_DATE + + logger.info("=" * 60) + logger.info(f"[4/7] 导入每日指标 (daily_basic): {start_date} ~ {end_date}") + + # 获取交易日列表 + conn = get_pg_connection() + try: + cursor = conn.cursor() + cursor.execute( + """ + SELECT DISTINCT cal_date FROM trade_cal + WHERE is_open = 1 + AND cal_date >= %s AND cal_date <= %s + ORDER BY cal_date + """, + (start_date, end_date), + ) + trade_dates = [row[0].strftime("%Y%m%d") for row in cursor.fetchall()] + cursor.close() + finally: + conn.close() + + total = len(trade_dates) + logger.info(f" 共 {total} 个交易日") + + conn = get_pg_connection() + for i, td in enumerate(trade_dates, 1): + try: + pro = get_ts_pro() + df = pro.daily_basic(trade_date=td) + if df is not None and not df.empty: + df = normalize_columns(df) + if "trade_date" in df.columns: + df["trade_date"] = pd.to_datetime(df["trade_date"], format="%Y%m%d", errors="coerce") + conflict_cols = ["ts_code", "trade_date"] + batch_insert("daily_basic", df, conn, conflict_cols) + except Exception as e: + logger.warning(f" [{td}] 导入失败: {e}") + conn.rollback() + + if i % 20 == 0 or i == total: + logger.info(f" 进度: {i}/{total}") + time.sleep(0.3) + + conn.close() + logger.info(" 每日指标导入完成") + + +# ============================================================ +# 5. 导入复权因子 +# ============================================================ + +def import_adj_factor( + ts_code: str = None, + start_date: str = None, + end_date: str = None, + conn=None, +): + """ + 导入复权因子 (adj_factor) + Tushare: adj_factor + """ + if start_date is None: + start_date = START_DATE + if end_date is None: + end_date = END_DATE + + pro = get_ts_pro() + own_conn = conn is None + if own_conn: + conn = get_pg_connection() + + try: + kwargs = { + "start_date": start_date.replace("-", ""), + "end_date": end_date.replace("-", ""), + } + if ts_code: + kwargs["ts_code"] = ts_code + + def fetch(): + return pro.adj_factor(**kwargs) + + df = fetch_with_retry(fetch, max_retries=2) + if df is None or df.empty: + return 0 + + df = normalize_columns(df) + + if "trade_date" in df.columns: + df["trade_date"] = pd.to_datetime(df["trade_date"], format="%Y%m%d", errors="coerce") + + conflict_cols = ["ts_code", "trade_date"] + return batch_insert("adj_factor", df, conn, conflict_cols) + + finally: + if own_conn: + conn.close() + + +def import_adj_factor_batch( + stock_list: List[str], + start_date: str = None, + end_date: str = None, +): + """批量导入复权因子""" + if start_date is None: + start_date = START_DATE + if end_date is None: + end_date = END_DATE + + total = len(stock_list) + logger.info("=" * 60) + logger.info(f"[5/7] 导入复权因子 (adj_factor): {total} 只股票") + + conn = get_pg_connection() + for i, ts_code in enumerate(stock_list, 1): + try: + import_adj_factor(ts_code=ts_code, start_date=start_date, end_date=end_date, conn=conn) + except Exception as e: + logger.warning(f" [{ts_code}] 复权因子导入失败: {e}") + conn.rollback() + + if i % 100 == 0 or i == total: + logger.info(f" 进度: {i}/{total}") + time.sleep(0.25) + + conn.close() + logger.info(" 复权因子导入完成") + + +# ============================================================ +# 6. 导入财务数据 (利润表、资产负债表、现金流量表、财务指标) +# ============================================================ + +def import_financial_statements( + stock_list: List[str], + start_date: str = None, + end_date: str = None, +): + """ + 按股票批量导入三大报表 + 财务指标 + - income: 利润表 + - balancesheet: 资产负债表 + - cashflow: 现金流量表 + - fina_indicator: 财务指标 + """ + if start_date is None: + start_date = START_DATE + if end_date is None: + end_date = END_DATE + + total = len(stock_list) + logger.info("=" * 60) + logger.info( + f"[6/7] 导入财务数据: {start_date} ~ {end_date}, " + f"共 {total} 只股票" + ) + + period_start = start_date.replace("-", "") + period_end = end_date.replace("-", "") + pro = get_ts_pro() + conn = get_pg_connection() + + for i, ts_code in enumerate(stock_list, 1): + for table_name, fetch_method in [ + ("income", pro.income), + ("balancesheet", pro.balancesheet), + ("cashflow", pro.cashflow), + ("fina_indicator", pro.fina_indicator), + ]: + try: + if table_name == "fina_indicator": + # fina_indicator 参数略有不同 + df = fetch_method( + ts_code=ts_code, + start_date=period_start, + end_date=period_end, + ) + else: + df = fetch_method( + ts_code=ts_code, + start_date=period_start, + end_date=period_end, + ) + + if df is None or df.empty: + continue + + df = normalize_columns(df) + + # 转换日期列 + for col in ["ann_date", "f_ann_date", "end_date"]: + if col in df.columns: + df[col] = pd.to_datetime(df[col], format="%Y%m%d", errors="coerce") + + if table_name == "fina_indicator": + conflict_cols = ["ts_code", "end_date"] + else: + conflict_cols = ["ts_code", "end_date", "report_type"] + + batch_insert(table_name, df, conn, conflict_cols) + + except Exception as e: + logger.debug(f" [{ts_code}] {table_name}: {e}") + conn.rollback() + + if i % 50 == 0 or i == total: + logger.info(f" 财务数据进度: {i}/{total}") + time.sleep(0.3) + + conn.close() + logger.info(" 财务数据导入完成") + + +# ============================================================ +# 7. 导入指数日线行情 +# ============================================================ + +def import_index_daily( + index_codes: List[str] = None, + start_date: str = None, + end_date: str = None, +): + """ + 导入指数日线行情 (index_daily) + 默认导入主要指数：上证指数、深证成指、沪深300、中证500、创业板指、科创50 + """ + if index_codes is None: + index_codes = [ + "000001.SH", # 上证指数 + "399001.SZ", # 深证成指 + "000300.SH", # 沪深300 + "000905.SH", # 中证500 + "399006.SZ", # 创业板指 + "000688.SH", # 科创50 + ] + if start_date is None: + start_date = START_DATE + if end_date is None: + end_date = END_DATE + + logger.info("=" * 60) + logger.info(f"[7/7] 导入指数日线行情: {len(index_codes)} 个指数") + + pro = get_ts_pro() + conn = get_pg_connection() + + for idx_code in index_codes: + try: + def fetch(): + return pro.index_daily( + ts_code=idx_code, + start_date=start_date.replace("-", ""), + end_date=end_date.replace("-", ""), + ) + + df = fetch_with_retry(fetch, max_retries=2) + if df is not None and not df.empty: + df = normalize_columns(df) + if "trade_date" in df.columns: + df["trade_date"] = pd.to_datetime( + df["trade_date"], format="%Y%m%d", errors="coerce" + ) + conflict_cols = ["ts_code", "trade_date"] + n = batch_insert("index_daily", df, conn, conflict_cols) + logger.info(f" {idx_code}: {n} 条") + else: + logger.warning(f" {idx_code}: 无数据") + except Exception as e: + logger.error(f" {idx_code}: {e}") + conn.rollback() + time.sleep(0.3) + + conn.close() + logger.info(" 指数日线行情导入完成") + + +# ============================================================ +# 8. 初始化数据库 Schema +# ============================================================ + +def init_database(): + """ + 执行 DDL，创建所有表结构 + """ + logger.info("=" * 60) + logger.info("初始化数据库 Schema ...") + + # 尝试创建数据库 (如果不存在) + try: + admin_conn = psycopg2.connect( + host=DB_CONFIG["host"], + port=DB_CONFIG["port"], + database="postgres", # 连接默认 postgres 库来创建新库 + user=DB_CONFIG["user"], + password=DB_CONFIG["password"], + ) + admin_conn.autocommit = True + cursor = admin_conn.cursor() + + # 检查数据库是否存在 + cursor.execute( + "SELECT 1 FROM pg_database WHERE datname = %s", + (DB_CONFIG["database"],), + ) + if cursor.fetchone() is None: + cursor.execute( + sql.SQL("CREATE DATABASE {} ENCODING 'UTF8'").format( + sql.Identifier(DB_CONFIG["database"]) + ) + ) + logger.info(f" 数据库 {DB_CONFIG['database']} 创建成功") + else: + logger.info(f" 数据库 {DB_CONFIG['database']} 已存在") + + cursor.close() + admin_conn.close() + except Exception as e: + logger.warning(f" 创建数据库步骤跳过 (可能无权限): {e}") + + # 执行 DDL + conn = get_pg_connection() + try: + with open("schema.sql", "r", encoding="utf-8") as f: + ddl_sql = f.read() + + # 按分号分割，逐条执行 (忽略被注释掉的分区表DDL) + statements = [s.strip() for s in ddl_sql.split(";") if s.strip() + and not s.strip().startswith("--")] + cursor = conn.cursor() + for stmt in statements: + if stmt and not stmt.startswith("--"): + try: + cursor.execute(stmt) + except Exception as e: + logger.debug(f" SQL 跳过: {str(e)[:100]}") + + conn.commit() + cursor.close() + logger.info(" Schema 初始化完成") + except Exception as e: + conn.rollback() + logger.error(f" Schema 初始化失败: {e}") + raise + finally: + conn.close() + + +# ============================================================ +# 9. 获取所有股票列表 (辅助) +# ============================================================ + +def get_all_stock_codes(include_delisted: bool = False) -> List[str]: + """ + 从 Tushare 获取所有 A 股股票代码列表 + """ + pro = get_ts_pro() + codes = [] + + for status, label in [("L", "上市"), ("D", "退市"), ("P", "暂停")]: + if status != "L" and not include_delisted: + continue + try: + df = pro.stock_basic( + exchange="", + list_status=status, + fields="ts_code", + ) + if df is not None and not df.empty: + codes.extend(df["ts_code"].tolist()) + except Exception as e: + logger.warning(f" 获取 {label} 股票列表失败: {e}") + + logger.info(f" 获取到 {len(codes)} 只股票代码") + return codes + + +def get_stock_codes_from_db(conn=None) -> List[str]: + """ + 从已导入的 stock_basic 表获取股票代码列表 + """ + own_conn = conn is None + if own_conn: + conn = get_pg_connection() + + try: + cursor = conn.cursor() + cursor.execute("SELECT ts_code FROM stock_basic WHERE list_status = 'L' ORDER BY ts_code") + codes = [row[0] for row in cursor.fetchall()] + cursor.close() + return codes + finally: + if own_conn: + conn.close() + + +# ============================================================ +# 10. 一键全量导入 +# ============================================================ + +def full_import( + start_date: str = None, + end_date: str = None, + import_financials: bool = True, + stock_codes: List[str] = None, +): + """ + 一键全量导入： + 1. 初始化 Schema + 2. 股票基本信息 + 3. 交易日历 + 4. 日线行情 + 5. 每日指标(估值) + 6. 复权因子 + 7. 财务数据 (可选) + 8. 指数日线行情 + + 参数: + - start_date, end_date: 数据范围 + - import_financials: 是否导入财务数据 (耗时较长) + - stock_codes: 指定股票列表，不传则全量导入 + """ + if start_date is None: + start_date = START_DATE + if end_date is None: + end_date = END_DATE + + start_time = datetime.now() + logger.info("=" * 70) + logger.info(f" 开始全量数据导入: {start_date} ~ {end_date}") + logger.info(f" PostgreSQL: {DB_CONFIG['host']}:{DB_CONFIG['port']}/{DB_CONFIG['database']}") + logger.info("=" * 70) + + # Step 0: 初始化 Schema + init_database() + + # Step 1: 股票基本信息 + import_stock_basic() + + # Step 2: 交易日历 + import_trade_cal(start_date, end_date) + + # 获取股票列表 + if stock_codes is None: + stock_codes = get_stock_codes_from_db() + if not stock_codes: + logger.error("无法获取股票列表，请先导入 stock_basic") + return + + # Step 3: 日线行情 (按年导入) + import_daily_by_year( + stock_codes, + start_year=int(start_date[:4]), + end_year=int(end_date[:4]), + ) + + # Step 4: 每日指标 (按日期导入) + import_daily_basic_by_date(start_date, end_date) + + # Step 5: 复权因子 + import_adj_factor_batch(stock_codes, start_date, end_date) + + # Step 6: 财务数据 + if import_financials: + import_financial_statements(stock_codes, start_date, end_date) + + # Step 7: 指数日线行情 + import_index_daily(start_date=start_date, end_date=end_date) + + elapsed = datetime.now() - start_time + logger.info("=" * 70) + logger.info(f" 全量数据导入完成! 总耗时: {elapsed}") + logger.info("=" * 70) + + +if __name__ == "__main__": + # 测试连接 + try: + conn = get_pg_connection() + logger.info(f"成功连接到 PostgreSQL: {DB_CONFIG['host']}:{DB_CONFIG['port']}") + conn.close() + except Exception as e: + logger.error(f"无法连接到 PostgreSQL: {e}") + logger.error("请确认 Docker 容器已启动，且 config.py 中的连接参数正确") \ No newline at end of file diff --git a/quantitative_data/.ipynb_checkpoints/requirements-checkpoint.txt b/quantitative_data/.ipynb_checkpoints/requirements-checkpoint.txt new file mode 100644 index 0000000..d8b1d64 --- /dev/null +++ b/quantitative_data/.ipynb_checkpoints/requirements-checkpoint.txt @@ -0,0 +1,4 @@ +tushare>=1.4.0 +pandas>=1.5.0 +psycopg2-binary>=2.9.0 +sqlalchemy>=2.0.0 \ No newline at end of file diff --git a/quantitative_data/.ipynb_checkpoints/schema-checkpoint.sql b/quantitative_data/.ipynb_checkpoints/schema-checkpoint.sql new file mode 100644 index 0000000..e19c387 --- /dev/null +++ b/quantitative_data/.ipynb_checkpoints/schema-checkpoint.sql @@ -0,0 +1,576 @@ +-- ============================================================ +-- 量化投资数据库 Schema +-- PostgreSQL (Docker: 192.168.27.15:12345) +-- 数据来源：Tushare +-- ============================================================ + +-- 先创建数据库（需要超级用户权限，在 psql 中手动执行）： +-- CREATE DATABASE quant_db ENCODING 'UTF8'; + +-- ============================================================ +-- 1. 股票基本信息表 (stock_basic) +-- tushare接口: stock_basic +-- ============================================================ +CREATE TABLE IF NOT EXISTS stock_basic ( + ts_code VARCHAR(20) PRIMARY KEY, -- 股票代码 (如 000001.SZ) + symbol VARCHAR(10) NOT NULL, -- 股票代码 (如 000001) + name VARCHAR(50) NOT NULL, -- 股票名称 + area VARCHAR(30), -- 地区 + industry VARCHAR(50), -- 行业 + market VARCHAR(10), -- 市场类型 (主板/中小板/创业板/科创板) + list_date DATE, -- 上市日期 + list_status CHAR(1) DEFAULT 'L', -- 上市状态 L上市 D退市 P暂停上市 + is_hs VARCHAR(5), -- 是否沪深港通标的 + act_name VARCHAR(100), -- 实控人名称 + act_ent_type VARCHAR(50), -- 实控人企业性质 + updated_at TIMESTAMP DEFAULT NOW() -- 记录更新时间 +); + +COMMENT ON TABLE stock_basic IS '股票基本信息表 (Tushare: stock_basic)'; + +-- 索引 +CREATE INDEX IF NOT EXISTS idx_stock_basic_industry ON stock_basic(industry); +CREATE INDEX IF NOT EXISTS idx_stock_basic_area ON stock_basic(area); +CREATE INDEX IF NOT EXISTS idx_stock_basic_list_status ON stock_basic(list_status); + + +-- ============================================================ +-- 2. 日线行情表 (daily) - 按年份分区 +-- tushare接口: daily / pro_bar +-- ============================================================ +CREATE TABLE IF NOT EXISTS daily ( + id BIGSERIAL, -- 自增主键 + ts_code VARCHAR(20) NOT NULL, -- 股票代码 + trade_date DATE NOT NULL, -- 交易日期 + open NUMERIC(12,3), -- 开盘价 + high NUMERIC(12,3), -- 最高价 + low NUMERIC(12,3), -- 最低价 + close NUMERIC(12,3), -- 收盘价 + pre_close NUMERIC(12,3), -- 昨收价 + change NUMERIC(12,3), -- 涨跌额 + pct_chg NUMERIC(10,4), -- 涨跌幅 (%) + vol NUMERIC(20,2), -- 成交量 (手) + amount NUMERIC(20,4), -- 成交额 (千元) + turnover_rate NUMERIC(10,4), -- 换手率 (%) + volume_ratio NUMERIC(10,4), -- 量比 + ma5 NUMERIC(12,3), -- 5日均价 + ma10 NUMERIC(12,3), -- 10日均价 + ma20 NUMERIC(12,3), -- 20日均价 + ma_v_5 NUMERIC(20,2), -- 5日均量 + ma_v_10 NUMERIC(20,2), -- 10日均量 + ma_v_20 NUMERIC(20,2), -- 20日均量 + updated_at TIMESTAMP DEFAULT NOW(), + + PRIMARY KEY (ts_code, trade_date) +); + +COMMENT ON TABLE daily IS '日线行情表 (Tushare: daily)'; + +-- 核心索引 +CREATE INDEX IF NOT EXISTS idx_daily_trade_date ON daily(trade_date); +CREATE INDEX IF NOT EXISTS idx_daily_ts_code ON daily(ts_code); +CREATE INDEX IF NOT EXISTS idx_daily_ts_code_date ON daily(ts_code, trade_date); + + +-- ============================================================ +-- 3. 日线行情表 - 按年份分区子表（优化查询性能） +-- PostgreSQL 原生分区表 (PG 10+) +-- 如果版本不支持，可以跳过，直接使用 daily 表 +-- ============================================================ + +-- 仅 PG 10+ 支持以下语法，如果报错请删除分区表相关语句 +-- CREATE TABLE daily_partitioned ( +-- id BIGSERIAL, +-- ts_code VARCHAR(20) NOT NULL, +-- trade_date DATE NOT NULL, +-- open NUMERIC(12,3), +-- high NUMERIC(12,3), +-- low NUMERIC(12,3), +-- close NUMERIC(12,3), +-- pre_close NUMERIC(12,3), +-- change NUMERIC(12,3), +-- pct_chg NUMERIC(10,4), +-- vol NUMERIC(20,2), +-- amount NUMERIC(20,4), +-- turnover_rate NUMERIC(10,4), +-- volume_ratio NUMERIC(10,4), +-- ma5 NUMERIC(12,3), +-- ma10 NUMERIC(12,3), +-- ma20 NUMERIC(12,3), +-- ma_v_5 NUMERIC(20,2), +-- ma_v_10 NUMERIC(20,2), +-- ma_v_20 NUMERIC(20,2), +-- updated_at TIMESTAMP DEFAULT NOW(), +-- PRIMARY KEY (ts_code, trade_date, trade_date) -- 分区键必须包含在主键中 +-- ) PARTITION BY RANGE (trade_date); + + +-- ============================================================ +-- 4. 复权因子表 (adj_factor) +-- tushare接口: adj_factor +-- 用于计算前复权/后复权价格 +-- ============================================================ +CREATE TABLE IF NOT EXISTS adj_factor ( + ts_code VARCHAR(20) NOT NULL, + trade_date DATE NOT NULL, + adj_factor NUMERIC(16,8), -- 复权因子 + updated_at TIMESTAMP DEFAULT NOW(), + + PRIMARY KEY (ts_code, trade_date) +); + +COMMENT ON TABLE adj_factor IS '复权因子表 (Tushare: adj_factor)'; + + +-- ============================================================ +-- 5. 个股资金流向表 (moneyflow) +-- tushare接口: moneyflow +-- ============================================================ +CREATE TABLE IF NOT EXISTS moneyflow ( + ts_code VARCHAR(20) NOT NULL, + trade_date DATE NOT NULL, + buy_sm_vol NUMERIC(20,2), -- 小单买入量(手) + buy_sm_amount NUMERIC(20,4), -- 小单买入金额(万元) + sell_sm_vol NUMERIC(20,2), -- 小单卖出量(手) + sell_sm_amount NUMERIC(20,4), -- 小单卖出金额(万元) + buy_md_vol NUMERIC(20,2), -- 中单买入量(手) + buy_md_amount NUMERIC(20,4), -- 中单买入金额(万元) + sell_md_vol NUMERIC(20,2), -- 中单卖出量(手) + sell_md_amount NUMERIC(20,4), -- 中单卖出金额(万元) + buy_lg_vol NUMERIC(20,2), -- 大单买入量(手) + buy_lg_amount NUMERIC(20,4), -- 大单买入金额(万元) + sell_lg_vol NUMERIC(20,2), -- 大单卖出量(手) + sell_lg_amount NUMERIC(20,4), -- 大单卖出金额(万元) + buy_elg_vol NUMERIC(20,2), -- 特大单买入量(手) + buy_elg_amount NUMERIC(20,4), -- 特大单买入金额(万元) + sell_elg_vol NUMERIC(20,2), -- 特大单卖出量(手) + sell_elg_amount NUMERIC(20,4), -- 特大单卖出金额(万元) + net_mf_vol NUMERIC(20,2), -- 净流入量(手) + net_mf_amount NUMERIC(20,4), -- 净流入额(万元) + updated_at TIMESTAMP DEFAULT NOW(), + + PRIMARY KEY (ts_code, trade_date) +); + +COMMENT ON TABLE moneyflow IS '个股资金流向表 (Tushare: moneyflow)'; + + +-- ============================================================ +-- 6. 每日指标表 (daily_basic) - 包含基本面/估值数据 +-- tushare接口: daily_basic +-- ============================================================ +CREATE TABLE IF NOT EXISTS daily_basic ( + ts_code VARCHAR(20) NOT NULL, + trade_date DATE NOT NULL, + close NUMERIC(12,3), -- 收盘价 + turnover_rate NUMERIC(10,4), -- 换手率(%) + turnover_rate_f NUMERIC(10,4), -- 自由流通股换手率(%) + volume_ratio NUMERIC(10,4), -- 量比 + pe NUMERIC(16,4), -- 市盈率(PE) + pe_ttm NUMERIC(16,4), -- 市盈率(TTM) + pb NUMERIC(16,4), -- 市净率(PB) + ps NUMERIC(16,4), -- 市销率(PS) + ps_ttm NUMERIC(16,4), -- 市销率(TTM) + dv_ratio NUMERIC(10,4), -- 股息率(%) + dv_ttm NUMERIC(10,4), -- 股息率(TTM)(%) + total_share NUMERIC(20,4), -- 总股本(万股) + float_share NUMERIC(20,4), -- 流通股本(万股) + free_share NUMERIC(20,4), -- 自由流通股本(万股) + total_mv NUMERIC(20,4), -- 总市值(万元) + circ_mv NUMERIC(20,4), -- 流通市值(万元) + updated_at TIMESTAMP DEFAULT NOW(), + + PRIMARY KEY (ts_code, trade_date) +); + +COMMENT ON TABLE daily_basic IS '每日指标表/基本面估值表 (Tushare: daily_basic)'; + + +-- ============================================================ +-- 7. 利润表 (income) +-- tushare接口: income +-- ============================================================ +CREATE TABLE IF NOT EXISTS income ( + ts_code VARCHAR(20) NOT NULL, + ann_date DATE, -- 公告日期 + f_ann_date DATE, -- 实际公告日期 + end_date DATE NOT NULL, -- 报告期 + report_type VARCHAR(10), -- 报告类型 + comp_type CHAR(1), -- 公司类型 + end_type VARCHAR(10), -- 报告期类型 + basic_eps NUMERIC(16,4), -- 基本每股收益 + diluted_eps NUMERIC(16,4), -- 稀释每股收益 + total_revenue NUMERIC(20,4), -- 营业总收入(元) + revenue NUMERIC(20,4), -- 营业收入(元) + int_income NUMERIC(20,4), -- 利息收入 + prem_earned NUMERIC(20,4), -- 已赚保费 + comm_income NUMERIC(20,4), -- 手续费及佣金收入 + n_commis_income NUMERIC(20,4), -- 手续费及佣金净收入 + n_oth_income NUMERIC(20,4), -- 其他经营净收益 + n_oth_b_income NUMERIC(20,4), -- 加:其他业务净收益 + prem_income NUMERIC(20,4), -- 保险业务收入 + out_prem NUMERIC(20,4), -- 减:分出保费 + une_prem_reser NUMERIC(20,4), -- 提取未到期责任准备金 + reins_income NUMERIC(20,4), -- 其中:分保费收入 + n_sec_tb_income NUMERIC(20,4), -- 代理买卖证券业务净收入 + n_sec_uw_income NUMERIC(20,4), -- 证券承销业务净收入 + n_asset_mg_income NUMERIC(20,4), -- 受托客户资产管理业务净收入 + oth_b_income NUMERIC(20,4), -- 其他业务收入 + fv_value_chg NUMERIC(20,4), -- 加:公允价值变动净收益 + invest_income NUMERIC(20,4), -- 加:投资净收益 + ass_invest_income NUMERIC(20,4), -- 其中:对联营企业和合营企业的投资收益 + forex_gain NUMERIC(20,4), -- 加:汇兑净收益 + total_cogs NUMERIC(20,4), -- 营业总成本 + oper_cost NUMERIC(20,4), -- 减:营业成本 + int_exp NUMERIC(20,4), -- 减:利息支出 + comm_exp NUMERIC(20,4), -- 减:手续费及佣金支出 + biz_tax_surchg NUMERIC(20,4), -- 减:营业税金及附加 + sell_exp NUMERIC(20,4), -- 减:销售费用 + admin_exp NUMERIC(20,4), -- 减:管理费用 + fin_exp NUMERIC(20,4), -- 减:财务费用 + assets_impair_loss NUMERIC(20,4), -- 减:资产减值损失 + prem_refund NUMERIC(20,4), -- 退保金 + compens_payout NUMERIC(20,4), -- 赔付总支出 + reser_insur_liab NUMERIC(20,4), -- 提取保险责任准备金 + div_payt NUMERIC(20,4), -- 保户红利支出 + reins_exp NUMERIC(20,4), -- 分保费用 + oper_exp NUMERIC(20,4), -- 营业支出 + compens_payout_refu NUMERIC(20,4), -- 减:摊回赔付支出 + insur_reser_refu NUMERIC(20,4), -- 减:摊回保险责任准备金 + reins_cost_refund NUMERIC(20,4), -- 减:摊回分保费用 + other_bus_cost NUMERIC(20,4), -- 其他业务成本 + operate_profit NUMERIC(20,4), -- 营业利润 + non_oper_income NUMERIC(20,4), -- 加:营业外收入 + non_oper_exp NUMERIC(20,4), -- 减:营业外支出 + nca_disploss NUMERIC(20,4), -- 其中:减:非流动资产处置净损失 + total_profit NUMERIC(20,4), -- 利润总额 + income_tax NUMERIC(20,4), -- 所得税费用 + n_income NUMERIC(20,4), -- 净利润(含少数股东损益) + n_income_attr_p NUMERIC(20,4), -- 净利润(不含少数股东损益) + minority_gain NUMERIC(20,4), -- 少数股东损益 + oth_compre_income NUMERIC(20,4), -- 其他综合收益 + total_compre_income NUMERIC(20,4), -- 综合收益总额 + comp_pre_inc_attr_p NUMERIC(20,4), -- 归属于母公司(或股东)的综合收益总额 + comp_inc_attr_m_s NUMERIC(20,4), -- 归属于少数股东的综合收益总额 + ebit NUMERIC(20,4), -- 息税前利润 + ebitda NUMERIC(20,4), -- 息税折旧摊销前利润 + insurance_exp NUMERIC(20,4), -- 保险业务支出 + undist_profit NUMERIC(20,4), -- 年初未分配利润 + distable_profit NUMERIC(20,4), -- 可分配利润 + updated_at TIMESTAMP DEFAULT NOW(), + + PRIMARY KEY (ts_code, end_date, report_type) +); + +COMMENT ON TABLE income IS '利润表 (Tushare: income)'; + + +-- ============================================================ +-- 8. 资产负债表 (balancesheet) +-- tushare接口: balancesheet +-- ============================================================ +CREATE TABLE IF NOT EXISTS balancesheet ( + ts_code VARCHAR(20) NOT NULL, + ann_date DATE, + f_ann_date DATE, + end_date DATE NOT NULL, -- 报告期 + report_type VARCHAR(10), + comp_type CHAR(1), + end_type VARCHAR(10), + + -- 资产 + total_hldr_eqy_exc_min_int NUMERIC(20,4), -- 股东权益合计(不含少数股东权益) + total_hldr_eqy_inc_min_int NUMERIC(20,4), -- 股东权益合计(含少数股东权益) + total_assets NUMERIC(20,4), -- 资产总计 + total_cur_assets NUMERIC(20,4), -- 流动资产合计 + total_nca NUMERIC(20,4), -- 非流动资产合计 + money_cap NUMERIC(20,4), -- 货币资金 + trad_asset NUMERIC(20,4), -- 交易性金融资产 + notes_receiv NUMERIC(20,4), -- 应收票据 + account_receiv NUMERIC(20,4), -- 应收账款 + oth_receiv NUMERIC(20,4), -- 其他应收款 + prepayment NUMERIC(20,4), -- 预付款项 + div_receiv NUMERIC(20,4), -- 应收股利 + int_receiv NUMERIC(20,4), -- 应收利息 + inventories NUMERIC(20,4), -- 存货 + amor_exp NUMERIC(20,4), -- 待摊费用 + nca_within_1y NUMERIC(20,4), -- 一年内到期的非流动资产 + sett_rsrv NUMERIC(20,4), -- 结算备付金 + loanto_oth_bank_fi NUMERIC(20,4), -- 拆出资金 + premium_receiv NUMERIC(20,4), -- 应收保费 + reinsur_receiv NUMERIC(20,4), -- 应收分保账款 + reinsur_res_receiv NUMERIC(20,4), -- 应收分保合同准备金 + pur_resale_fa NUMERIC(20,4), -- 买入返售金融资产 + oth_cur_assets NUMERIC(20,4), -- 其他流动资产 + + -- 非流动资产 + fa_avail_for_sale NUMERIC(20,4), -- 可供出售金融资产 + htm_invest NUMERIC(20,4), -- 持有至到期投资 + lt_eqt_invest NUMERIC(20,4), -- 长期股权投资 + invest_real_estate NUMERIC(20,4), -- 投资性房地产 + time_deposits NUMERIC(20,4), -- 定期存款 + oth_assets NUMERIC(20,4), -- 其他资产 + lt_rec NUMERIC(20,4), -- 长期应收款 + fix_assets NUMERIC(20,4), -- 固定资产 + cip NUMERIC(20,4), -- 在建工程 + const_materials NUMERIC(20,4), -- 工程物资 + fixed_assets_disp NUMERIC(20,4), -- 固定资产清理 + produc_bio_assets NUMERIC(20,4), -- 生产性生物资产 + oil_and_gas_assets NUMERIC(20,4), -- 油气资产 + intan_assets NUMERIC(20,4), -- 无形资产 + r_and_d NUMERIC(20,4), -- 研发支出 + goodwill NUMERIC(20,4), -- 商誉 + lt_amor_exp NUMERIC(20,4), -- 长期待摊费用 + defer_tax_assets NUMERIC(20,4), -- 递延所得税资产 + decr_in_disbur NUMERIC(20,4), -- 待处理流动资产损益 + oth_nca NUMERIC(20,4), -- 其他非流动资产 + + -- 负债 + total_liab NUMERIC(20,4), -- 负债合计 + total_cur_liab NUMERIC(20,4), -- 流动负债合计 + total_ncl NUMERIC(20,4), -- 非流动负债合计 + short_borr NUMERIC(20,4), -- 短期借款 + lt_borr NUMERIC(20,4), -- 长期借款 + st_notes_payable NUMERIC(20,4), -- 应付票据 + acct_payable NUMERIC(20,4), -- 应付账款 + adv_receipts NUMERIC(20,4), -- 预收款项 + bonds_payable NUMERIC(20,4), -- 应付债券 + oth_payable NUMERIC(20,4), -- 其他应付款 + div_payable NUMERIC(20,4), -- 应付股利 + int_payable NUMERIC(20,4), -- 应付利息 + lt_payable NUMERIC(20,4), -- 长期应付款 + special_payable NUMERIC(20,4), -- 专项应付款 + updated_at TIMESTAMP DEFAULT NOW(), + + PRIMARY KEY (ts_code, end_date, report_type) +); + +COMMENT ON TABLE balancesheet IS '资产负债表 (Tushare: balancesheet)'; + + +-- ============================================================ +-- 9. 现金流量表 (cashflow) +-- tushare接口: cashflow +-- ============================================================ +CREATE TABLE IF NOT EXISTS cashflow ( + ts_code VARCHAR(20) NOT NULL, + ann_date DATE, + f_ann_date DATE, + end_date DATE NOT NULL, -- 报告期 + report_type VARCHAR(10), + comp_type CHAR(1), + end_type VARCHAR(10), + + -- 经营活动 + net_profit NUMERIC(20,4), -- 净利润 + fin_exp NUMERIC(20,4), -- 财务费用 + c_fr_sale_sg NUMERIC(20,4), -- 销售商品、提供劳务收到的现金 + recp_tax_rends NUMERIC(20,4), -- 收到的税费返还 + n_depos_incr_fi NUMERIC(20,4), -- 客户存款和同业存放款项净增加额 + n_incr_loans_cb NUMERIC(20,4), -- 向中央银行借款净增加额 + n_incr_borr_oth_fi NUMERIC(20,4), -- 向其他金融机构拆入资金净增加额 + prem_fr_orig_contr NUMERIC(20,4), -- 收到原保险合同保费取得的现金 + n_incr_insured_dep NUMERIC(20,4), -- 保户储金净增加额 + n_reinsur_prem NUMERIC(20,4), -- 收到再保业务现金净额 + n_incr_disp_tfa NUMERIC(20,4), -- 处置交易性金融资产净增加额 + ifc_cash_incr NUMERIC(20,4), -- 收取利息、手续费及佣金的现金 + n_incr_disp_faas NUMERIC(20,4), -- 处置可供出售金融资产净增加额 + n_incr_loans_oth_bank NUMERIC(20,4), -- 拆入资金净增加额 + dcr_cash_paym NUMERIC(20,4), -- 回购业务资金净增加额 + oth_cash_recp_ral_oper_act NUMERIC(20,4), -- 收到其他与经营活动有关的现金 + st_cash_out_act NUMERIC(20,4), -- 经营活动现金流出小计 + n_cashflow_act NUMERIC(20,4), -- 经营活动产生的现金流量净额 + + -- 投资活动 + c_inf_fr_invest_act NUMERIC(20,4), -- 投资活动现金流入小计 + c_out_invest_act NUMERIC(20,4), -- 投资活动现金流出小计 + n_cashflow_inv_act NUMERIC(20,4), -- 投资活动产生的现金流量净额 + + -- 筹资活动 + c_inf_fr_fin_act NUMERIC(20,4), -- 筹资活动现金流入小计 + st_cash_out_fin_act NUMERIC(20,4), -- 筹资活动现金流出小计 + n_cash_flows_fin_act NUMERIC(20,4), -- 筹资活动产生的现金流量净额 + eff_fx_flu_cash NUMERIC(20,4), -- 汇率变动对现金的影响 + n_incr_cash_cash_equ NUMERIC(20,4), -- 现金及现金等价物净增加额 + c_cash_equ_beg_period NUMERIC(20,4), -- 期初现金及现金等价物余额 + c_cash_equ_end_period NUMERIC(20,4), -- 期末现金及现金等价物余额 + updated_at TIMESTAMP DEFAULT NOW(), + + PRIMARY KEY (ts_code, end_date, report_type) +); + +COMMENT ON TABLE cashflow IS '现金流量表 (Tushare: cashflow)'; + + +-- ============================================================ +-- 10. 财务指标表 (fina_indicator) - 核心基本面指标 +-- tushare接口: fina_indicator +-- ============================================================ +CREATE TABLE IF NOT EXISTS fina_indicator ( + ts_code VARCHAR(20) NOT NULL, + ann_date DATE, + end_date DATE NOT NULL, -- 报告期 + -- 每股指标 + eps NUMERIC(16,4), -- 每股收益 + dt_eps NUMERIC(16,4), -- 稀释每股收益 + total_revenue_ps NUMERIC(16,4), -- 每股营业总收入 + revenue_ps NUMERIC(16,4), -- 每股营业收入 + capital_rese_ps NUMERIC(16,4), -- 每股资本公积 + surplus_rese_ps NUMERIC(16,4), -- 每股盈余公积 + undist_profit_ps NUMERIC(16,4), -- 每股未分配利润 + extra_item NUMERIC(16,4), -- 非经常性损益 + profit_dedt NUMERIC(16,4), -- 扣除非经常性损益后的净利润(扣非净利润) + -- 成长能力 + gross_margin NUMERIC(16,4), -- 毛利 + current_ratio NUMERIC(16,4), -- 流动比率 + quick_ratio NUMERIC(16,4), -- 速动比率 + cash_ratio NUMERIC(16,4), -- 保守速动比率 + invturn_days NUMERIC(16,4), -- 存货周转天数 + arturn_days NUMERIC(16,4), -- 应收账款周转天数 + inv_turn NUMERIC(16,4), -- 存货周转率 + ar_turn NUMERIC(16,4), -- 应收账款周转率 + assets_turn NUMERIC(16,4), -- 总资产周转率 + -- 盈利能力 + roe NUMERIC(16,4), -- 净资产收益率 + roe_dt NUMERIC(16,4), -- 净资产收益率(扣除非经常损益) + roa NUMERIC(16,4), -- 总资产净利率 + roa2 NUMERIC(16,4), -- 总资产报酬率 + roic NUMERIC(16,4), -- 投入资本回报率 + -- 杠杆/偿债 + longdeb_to_debt NUMERIC(16,4), -- 长期借款与资产总计之比 + debt_to_assets NUMERIC(16,4), -- 资产负债率 + ncfm NUMERIC(16,4), -- 销售商品提供劳务收到的现金/营业收入 + -- 收益质量 + profit_to_op NUMERIC(16,4), -- 经营活动净收益/利润总额 + cfo_to_or NUMERIC(16,4), -- 经营活动现金流净额/营业收入 + ocf_to_or NUMERIC(16,4), -- 经营现金净流量/营业总收入 + -- 杜邦分析 + dupont_roe NUMERIC(16,4), -- 净资产收益率(杜邦) + dupont_assets_turn NUMERIC(16,4), -- 资产周转率(杜邦) + dupont_equity_multi NUMERIC(16,4), -- 权益乘数(杜邦) + dupont_net_profit_margin NUMERIC(16,4), -- 销售净利率(杜邦) + -- 分红 + div_rec NUMERIC(16,4), -- 每股股利(税前) + -- 其他 + ocfps NUMERIC(16,4), -- 每股经营活动产生的现金流量净额 + cfps NUMERIC(16,4), -- 每股现金流量净额 + ebit_ps NUMERIC(16,4), -- 每股息税前利润 + fcff_ps NUMERIC(16,4), -- 每股企业自由现金流量 + fcfe_ps NUMERIC(16,4), -- 每股股东自由现金流量 + updated_at TIMESTAMP DEFAULT NOW(), + + PRIMARY KEY (ts_code, end_date) +); + +COMMENT ON TABLE fina_indicator IS '财务指标表 (Tushare: fina_indicator)'; + + +-- ============================================================ +-- 11. 交易日历表 (trade_cal) +-- tushare接口: trade_cal +-- ============================================================ +CREATE TABLE IF NOT EXISTS trade_cal ( + exchange VARCHAR(10) NOT NULL, -- 交易所 SSE上交所 SZSE深交所 + cal_date DATE NOT NULL, -- 日历日期 + is_open INTEGER NOT NULL, -- 是否交易 0休市 1交易 + pretrade_date DATE, -- 上一个交易日 + updated_at TIMESTAMP DEFAULT NOW(), + + PRIMARY KEY (exchange, cal_date) +); + +COMMENT ON TABLE trade_cal IS '交易日历表 (Tushare: trade_cal)'; + + +-- ============================================================ +-- 12. 指数日线行情表 (index_daily) +-- tushare接口: index_daily +-- 用于记录主要指数(沪深300、中证500等)的日线行情 +-- ============================================================ +CREATE TABLE IF NOT EXISTS index_daily ( + ts_code VARCHAR(20) NOT NULL, -- 指数代码 (如 000300.SH) + trade_date DATE NOT NULL, + close NUMERIC(12,4), + open NUMERIC(12,4), + high NUMERIC(12,4), + low NUMERIC(12,4), + pre_close NUMERIC(12,4), + change NUMERIC(12,4), + pct_chg NUMERIC(10,4), + vol NUMERIC(20,2), + amount NUMERIC(20,4), + updated_at TIMESTAMP DEFAULT NOW(), + + PRIMARY KEY (ts_code, trade_date) +); + +COMMENT ON TABLE index_daily IS '指数日线行情表 (Tushare: index_daily)'; + + +-- ============================================================ +-- 常用查询视图 +-- ============================================================ + +-- 视图1: 日线行情 + 估值指标联合查询 +CREATE OR REPLACE VIEW v_daily_with_valuation AS +SELECT + d.ts_code, + d.trade_date, + b.name, + b.industry, + b.area, + d.open, + d.high, + d.low, + d.close, + d.pre_close, + d.pct_chg, + d.vol, + d.amount, + d.turnover_rate, + db.pe, + db.pe_ttm, + db.pb, + db.ps, + db.ps_ttm, + db.total_mv, + db.circ_mv, + db.dv_ratio, + db.dv_ttm +FROM daily d +LEFT JOIN stock_basic b ON d.ts_code = b.ts_code +LEFT JOIN daily_basic db ON d.ts_code = db.ts_code AND d.trade_date = db.trade_date; + +COMMENT ON VIEW v_daily_with_valuation IS '日线行情+估值联合查询视图'; + + +-- 视图2: 最新财务指标 + 股票基本信息 +CREATE OR REPLACE VIEW v_latest_financials AS +SELECT DISTINCT ON (f.ts_code) + f.ts_code, + b.name, + b.industry, + b.area, + b.list_date, + f.end_date AS report_date, + f.eps, + f.roe, + f.roa, + f.roic, + f.current_ratio, + f.quick_ratio, + f.debt_to_assets, + f.gross_margin, + f.assets_turn, + f.cfps, + f.ocfps, + f.div_rec +FROM fina_indicator f +JOIN stock_basic b ON f.ts_code = b.ts_code +ORDER BY f.ts_code, f.end_date DESC; + +COMMENT ON VIEW v_latest_financials IS '最新财务指标+股票信息'; + +-- ============================================================ +-- 完成 +-- ============================================================ \ No newline at end of file diff --git a/quantitative_data/.ipynb_checkpoints/数据批量导入-checkpoint.ipynb b/quantitative_data/.ipynb_checkpoints/数据批量导入-checkpoint.ipynb new file mode 100644 index 0000000..9d2ff3b --- /dev/null +++ b/quantitative_data/.ipynb_checkpoints/数据批量导入-checkpoint.ipynb @@ -0,0 +1,647 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 量化投资数据批量导入\n", + "\n", + "## 目标\n", + "将 Tushare 的日线行情数据及公司基本面数据批量导入 Docker PostgreSQL (192.168.27.15:12345)\n", + "\n", + "## 数据库结构概览\n", + "\n", + "| 表名 | 说明 | Tushare 接口 |\n", + "|------|------|-------------|\n", + "| stock_basic | 股票基本信息 | stock_basic |\n", + "| trade_cal | 交易日历 | trade_cal |\n", + "| daily | 日线行情 | daily |\n", + "| daily_basic | 每日指标(估值/基本面) | daily_basic |\n", + "| adj_factor | 复权因子 | adj_factor |\n", + "| income | 利润表 | income |\n", + "| balancesheet | 资产负债表 | balancesheet |\n", + "| cashflow | 现金流量表 | cashflow |\n", + "| fina_indicator | 财务指标 | fina_indicator |\n", + "| moneyflow | 个股资金流向 | moneyflow |\n", + "| index_daily | 指数日线行情 | index_daily |\n", + "\n", + "## 使用步骤\n", + "1. 修改 `config.py` 中的数据库密码和 Tushare Token\n", + "2. 逐 Cell 运行本 Notebook" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## Step 0: 检查环境 & 连接测试" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "import os\n", + "os.chdir(r\"t:\\jupyter\\notebook\\quantitative_data\")\n", + "print(f\"工作目录: {os.getcwd()}\")\n", + "print(f\"Python 版本: {sys.version}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 检查依赖包\n", + "!pip list | findstr -i \"tushare pandas psycopg2 sqlalchemy\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 如果需要安装依赖，取消注释下面这行\n", + "# !pip install -r requirements.txt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 导入核心模块\n", + "from importer import (\n", + " get_pg_connection,\n", + " get_ts_pro,\n", + " init_database,\n", + " import_stock_basic,\n", + " import_trade_cal,\n", + " import_daily_batch,\n", + " import_daily_by_year,\n", + " import_daily_basic,\n", + " import_daily_basic_by_date,\n", + " import_adj_factor,\n", + " import_adj_factor_batch,\n", + " import_financial_statements,\n", + " import_index_daily,\n", + " get_all_stock_codes,\n", + " get_stock_codes_from_db,\n", + " full_import,\n", + " batch_insert,\n", + " logger,\n", + ")\n", + "from config import DB_CONFIG, TUSHARE_TOKEN, START_DATE, END_DATE\n", + "\n", + "print(\"模块导入成功!\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 测试数据库连接\n", + "try:\n", + " conn = get_pg_connection()\n", + " cursor = conn.cursor()\n", + " cursor.execute(\"SELECT version()\")\n", + " version = cursor.fetchone()[0]\n", + " print(f\"✓ PostgreSQL 连接成功!\")\n", + " print(f\" 服务器版本: {version}\")\n", + " print(f\" 连接信息: {DB_CONFIG['host']}:{DB_CONFIG['port']}/{DB_CONFIG['database']}\")\n", + " cursor.close()\n", + " conn.close()\n", + "except Exception as e:\n", + " print(f\"✗ 连接失败: {e}\")\n", + " print(\"请检查:\")\n", + " print(\" 1. Docker 容器是否已启动: docker ps | findstr postgres\")\n", + " print(\" 2. config.py 中的连接参数是否正确\")\n", + " print(\" 3. 防火墙是否开放 12345 端口\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 测试 Tushare API 连接\n", + "try:\n", + " pro = get_ts_pro()\n", + " # 简单测试: 获取一只股票信息\n", + " df = pro.stock_basic(ts_code=\"000001.SZ\", fields=\"ts_code,name,industry\")\n", + " print(f\"✓ Tushare API 连接成功!\")\n", + " print(f\" 测试查询: {df.iloc[0].to_dict()}\")\n", + "except Exception as e:\n", + " print(f\"✗ Tushare API 连接失败: {e}\")\n", + " print(\"请检查 config.py 中的 TUSHARE_TOKEN 是否正确\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## Step 1: 初始化数据库 Schema" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 执行 DDL，创建所有表结构\n", + "init_database()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## Step 2: 导入股票基本信息" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 导入全量 A 股股票基本信息（含上市和退市）\n", + "import_stock_basic()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 验证：查看导入结果\n", + "conn = get_pg_connection()\n", + "cursor = conn.cursor()\n", + "cursor.execute(\"SELECT COUNT(*) FROM stock_basic\")\n", + "print(f\"stock_basic 总记录数: {cursor.fetchone()[0]}\")\n", + "cursor.execute(\"SELECT list_status, COUNT(*) FROM stock_basic GROUP BY list_status\")\n", + "for row in cursor.fetchall():\n", + " print(f\" 状态 '{row[0]}': {row[1]} 只\")\n", + "cursor.close()\n", + "conn.close()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## Step 3: 导入交易日历" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 导入交易日历 (默认从 config.py 的 START_DATE ~ END_DATE)\n", + "import_trade_cal()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 或者指定日期范围\n", + "# import_trade_cal(start_date=\"2020-01-01\", end_date=\"2025-12-31\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 验证\n", + "conn = get_pg_connection()\n", + "cursor = conn.cursor()\n", + "cursor.execute(\"\"\"\n", + " SELECT exchange, MIN(cal_date) AS first_date, MAX(cal_date) AS last_date, COUNT(*) AS total\n", + " FROM trade_cal\n", + " GROUP BY exchange\n", + "\"\"\")\n", + "for row in cursor.fetchall():\n", + " print(f\" {row[0]}: {row[1]} ~ {row[2]}, 共 {row[3]} 条\")\n", + "cursor.close()\n", + "conn.close()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## Step 4: 导入日线行情 (核心表，最耗时)\n", + "\n", + "### 4.1 获取股票列表" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 获取所有需要导入的股票代码\n", + "stock_list = get_stock_codes_from_db()\n", + "print(f\"共 {len(stock_list)} 只股票需要导入日线行情\")\n", + "print(f\"前 10 只: {stock_list[:10]}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 4.2 按年批量导入 (推荐 - 断点续传友好)\n", + "\n", + "数据量估算: 约5000只股票 × 250交易日/年 × 16年 ≈ 2000万条记录" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 按年份逐批导入日线行情\n", + "# 如果中断，可以修改年份范围从断点继续\n", + "import_daily_by_year(\n", + " stock_list,\n", + " start_year=2010,\n", + " end_year=2025,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 4.3 单只股票导入 (补充/重试)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 导入单只股票日线行情 (用于补充导入或测试)\n", + "conn = get_pg_connection()\n", + "from importer import import_daily_for_stock\n", + "\n", + "n = import_daily_for_stock(\"000001.SZ\", \"2020-01-01\", \"2020-12-31\", conn)\n", + "print(f\"导入 000001.SZ 2020年数据: {n} 条\")\n", + "conn.close()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 验证日线数据\n", + "conn = get_pg_connection()\n", + "cursor = conn.cursor()\n", + "cursor.execute(\"\"\"\n", + " SELECT \n", + " COUNT(*) AS total_records,\n", + " COUNT(DISTINCT ts_code) AS stock_count,\n", + " MIN(trade_date) AS first_date,\n", + " MAX(trade_date) AS last_date\n", + " FROM daily\n", + "\"\"\")\n", + "for row in cursor.fetchall():\n", + " print(f\" 总记录数: {row[0]:,}\")\n", + " print(f\" 股票数量: {row[1]}\")\n", + " print(f\" 日期范围: {row[2]} ~ {row[3]}\")\n", + "cursor.close()\n", + "conn.close()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## Step 5: 导入每日指标 (估值数据)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 按交易日导入每日指标 (PE/PB/PS/总市值/流通市值等)\n", + "import_daily_basic_by_date(\n", + " start_date=\"2010-01-01\",\n", + " end_date=\"2025-12-31\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 验证\n", + "conn = get_pg_connection()\n", + "cursor = conn.cursor()\n", + "cursor.execute(\"SELECT COUNT(*), MIN(trade_date), MAX(trade_date) FROM daily_basic\")\n", + "row = cursor.fetchone()\n", + "print(f\" daily_basic: {row[0]:,} 条, {row[1]} ~ {row[2]}\")\n", + "cursor.close()\n", + "conn.close()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## Step 6: 导入复权因子" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 导入复权因子 (用于前复权/后复权价格计算)\n", + "import_adj_factor_batch(\n", + " stock_list,\n", + " start_date=\"2010-01-01\",\n", + " end_date=\"2025-12-31\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## Step 7: 导入财务数据 (三大报表 + 财务指标)\n", + "\n", + "⚠ 此步骤耗时较长，约需数小时（取决于股票数量）" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 导入利润表、资产负债表、现金流量表、财务指标\n", + "import_financial_statements(\n", + " stock_list,\n", + " start_date=\"2010-01-01\",\n", + " end_date=\"2025-12-31\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 验证财务数据\n", + "conn = get_pg_connection()\n", + "cursor = conn.cursor()\n", + "for table in [\"income\", \"balancesheet\", \"cashflow\", \"fina_indicator\"]:\n", + " cursor.execute(f\"SELECT COUNT(*) FROM {table}\")\n", + " count = cursor.fetchone()[0]\n", + " print(f\" {table}: {count:,} 条\")\n", + "cursor.close()\n", + "conn.close()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## Step 8: 导入指数日线行情" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 导入主要指数日线行情\n", + "import_index_daily(\n", + " index_codes=[\n", + " \"000001.SH\", # 上证指数\n", + " \"399001.SZ\", # 深证成指\n", + " \"000300.SH\", # 沪深300\n", + " \"000905.SH\", # 中证500\n", + " \"399006.SZ\", # 创业板指\n", + " \"000688.SH\", # 科创50\n", + " \"000016.SH\", # 上证50\n", + " \"399005.SZ\", # 中小100\n", + " \"000852.SH\", # 中证1000\n", + " ],\n", + " start_date=\"2010-01-01\",\n", + " end_date=\"2025-12-31\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## 一键全量导入 (可选)\n", + "\n", + "如果不想逐步执行，可以运行下面这个 Cell 一键完成所有导入" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 一键全量导入 (需数小时~数十小时，请谨慎)\n", + "# full_import(\n", + "# start_date=\"2010-01-01\",\n", + "# end_date=\"2025-12-31\",\n", + "# import_financials=True, # 设为 False 跳过财务数据加快速度\n", + "# )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## 数据验证与查询示例" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import psycopg2\n", + "\n", + "conn = get_pg_connection()\n", + "\n", + "# 各表统计\n", + "tables = [\"stock_basic\", \"daily\", \"daily_basic\", \"adj_factor\",\n", + " \"income\", \"balancesheet\", \"cashflow\", \"fina_indicator\",\n", + " \"trade_cal\", \"index_daily\"]\n", + "\n", + "print(f\"{'表名':<20} {'记录数':>12} {'最早日期':>12} {'最晚日期':>12}\")\n", + "print(\"-\" * 60)\n", + "for table in tables:\n", + " try:\n", + " count_sql = f\"SELECT COUNT(*) FROM {table}\"\n", + " count = pd.read_sql(count_sql, conn).iloc[0, 0]\n", + " \n", + " # 尝试获取日期范围\n", + " date_col = None\n", + " if table == \"daily\":\n", + " date_col = \"trade_date\"\n", + " elif table == \"daily_basic\":\n", + " date_col = \"trade_date\"\n", + " elif table in [\"income\", \"balancesheet\", \"cashflow\"]:\n", + " date_col = \"end_date\"\n", + " elif table == \"fina_indicator\":\n", + " date_col = \"end_date\"\n", + " elif table == \"trade_cal\":\n", + " date_col = \"cal_date\"\n", + " elif table == \"index_daily\":\n", + " date_col = \"trade_date\"\n", + " elif table == \"adj_factor\":\n", + " date_col = \"trade_date\"\n", + " \n", + " if date_col:\n", + " date_sql = f\"SELECT MIN({date_col}), MAX({date_col}) FROM {table}\"\n", + " min_d, max_d = pd.read_sql(date_sql, conn).iloc[0]\n", + " print(f\"{table:<20} {count:>12,} {str(min_d)[:10]:>12} {str(max_d)[:10]:>12}\")\n", + " else:\n", + " print(f\"{table:<20} {count:>12,}\")\n", + " except Exception as e:\n", + " print(f\"{table:<20} {'错误':>12}: {str(e)[:40]}\")\n", + "\n", + "conn.close()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 示例查询 1: 查询某股票最近10个交易日数据\n", + "query1 = \"\"\"\n", + "SELECT trade_date, open, high, low, close, vol, amount, pct_chg\n", + "FROM daily\n", + "WHERE ts_code = '000001.SZ'\n", + "ORDER BY trade_date DESC\n", + "LIMIT 10\n", + "\"\"\"\n", + "conn = get_pg_connection()\n", + "df1 = pd.read_sql(query1, conn)\n", + "print(\"平安银行(000001.SZ) 最近10个交易日:\")\n", + "display(df1)\n", + "conn.close()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 示例查询 2: 日线行情 + 估值指标联合查询 (使用视图)\n", + "query2 = \"\"\"\n", + "SELECT *\n", + "FROM v_daily_with_valuation\n", + "WHERE ts_code = '000001.SZ'\n", + " AND trade_date >= '2024-01-01'\n", + "ORDER BY trade_date DESC\n", + "LIMIT 10\n", + "\"\"\"\n", + "conn = get_pg_connection()\n", + "df2 = pd.read_sql(query2, conn)\n", + "print(\"平安银行 - 日线+估值:\")\n", + "display(df2[['trade_date', 'close', 'pct_chg', 'pe', 'pe_ttm', 'pb', 'total_mv']])\n", + "conn.close()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 示例查询 3: 最新财务指标 Top 20 (按 ROE 排序)\n", + "query3 = \"\"\"\n", + "SELECT *\n", + "FROM v_latest_financials\n", + "WHERE roe IS NOT NULL\n", + " AND roe > 0\n", + "ORDER BY roe DESC\n", + "LIMIT 20\n", + "\"\"\"\n", + "conn = get_pg_connection()\n", + "df3 = pd.read_sql(query3, conn)\n", + "print(\"ROE Top 20:\")\n", + "display(df3[['ts_code', 'name', 'industry', 'roe', 'roa', 'eps', 'debt_to_assets']])\n", + "conn.close()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/quantitative_data/config.py b/quantitative_data/config.py new file mode 100644 index 0000000..d1f9638 --- /dev/null +++ b/quantitative_data/config.py @@ -0,0 +1,19 @@ +""" +量化数据库配置文件 +""" +# PostgreSQL Docker 连接配置 +DB_CONFIG = { + "host": "192.168.27.15", + "port": 12345, + "database": "quant_db", + "user": "postgres", + "password": "postgres", # 请修改为实际密码 +} + +# Tushare API Token +TUSHARE_TOKEN = "你的token" # 请替换为你的tushare token + +# 批量导入参数 +BATCH_SIZE = 5000 # 每批次插入行数 +START_DATE = "2010-01-01" # 数据起始日期 +END_DATE = "2025-12-31" # 数据结束日期 \ No newline at end of file diff --git a/quantitative_data/importer.py b/quantitative_data/importer.py new file mode 100644 index 0000000..5d0c912 --- /dev/null +++ b/quantitative_data/importer.py @@ -0,0 +1,983 @@ +""" +量化数据导入核心模块 +连接 Docker PostgreSQL (192.168.27.15:12345) +从 Tushare 拉取数据并批量导入 +""" +import time +import logging +from datetime import datetime, timedelta +from typing import Optional, List, Dict + +import tushare as ts +import pandas as pd +import psycopg2 +from psycopg2 import sql +from psycopg2.extras import execute_values +from sqlalchemy import create_engine + +from config import DB_CONFIG, TUSHARE_TOKEN, BATCH_SIZE, START_DATE, END_DATE + +# ============================================================ +# 日志配置 +# ============================================================ +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + handlers=[ + logging.FileHandler("import_data.log", encoding="utf-8"), + logging.StreamHandler(), + ], +) +logger = logging.getLogger(__name__) + +# ============================================================ +# 初始化连接 +# ============================================================ + +# psycopg2 原生连接 (用于执行 DDL / 精细控制) +def get_pg_connection(): + """获取 psycopg2 原生连接""" + return psycopg2.connect( + host=DB_CONFIG["host"], + port=DB_CONFIG["port"], + database=DB_CONFIG["database"], + user=DB_CONFIG["user"], + password=DB_CONFIG["password"], + ) + + +# SQLAlchemy 引擎 (用于 DataFrame.to_sql) +_sqlalchemy_engine = None + + +def get_sqlalchemy_engine(): + """获取 SQLAlchemy 引擎 (单例)""" + global _sqlalchemy_engine + if _sqlalchemy_engine is None: + db_url = ( + f"postgresql://{DB_CONFIG['user']}:{DB_CONFIG['password']}" + f"@{DB_CONFIG['host']}:{DB_CONFIG['port']}/{DB_CONFIG['database']}" + ) + _sqlalchemy_engine = create_engine(db_url, pool_size=5, max_overflow=10) + return _sqlalchemy_engine + + +# Tushare Pro API (单例) +_ts_pro = None + + +def get_ts_pro(): + """获取 Tushare Pro API 实例 (单例)""" + global _ts_pro + if _ts_pro is None: + ts.set_token(TUSHARE_TOKEN) + _ts_pro = ts.pro_api() + logger.info("Tushare Pro API 初始化完成") + return _ts_pro + + +# ============================================================ +# 通用导入工具函数 +# ============================================================ + +def normalize_columns(df: pd.DataFrame) -> pd.DataFrame: + """ + 标准化列名：Tushare 返回的列名可能有大小写差异，统一转小写 + """ + df.columns = [c.lower() for c in df.columns] + return df + + +def safe_float(val): + """安全转换为浮点数，NaN -> None""" + try: + if pd.isna(val): + return None + return float(val) + except (ValueError, TypeError): + return None + + +def batch_insert(table_name: str, df: pd.DataFrame, conn, conflict_columns: List[str]): + """ + 使用 execute_values 批量 UPSERT (INSERT ... ON CONFLICT) + - table_name: 目标表名 + - df: 待导入 DataFrame + - conn: psycopg2 连接 + - conflict_columns: 冲突列 (唯一约束列)，冲突时更新其他列 + """ + if df.empty: + logger.warning(f" {table_name}: 空数据，跳过") + return 0 + + columns = list(df.columns) + rows = [tuple(row) for row in df.itertuples(index=False)] + + # 构建 ON CONFLICT 子句 + conflict_str = ", ".join(conflict_columns) + # 构建 UPDATE SET 子句 (排除冲突列) + update_cols = [c for c in columns if c not in conflict_columns] + if not update_cols: + # 没有需要更新的列，使用 DO NOTHING + upsert_sql = sql.SQL( + "INSERT INTO {table} ({cols}) VALUES %s " + "ON CONFLICT ({conflict}) DO NOTHING" + ).format( + table=sql.Identifier(table_name), + cols=sql.SQL(", ").join(map(sql.Identifier, columns)), + conflict=sql.SQL(conflict_str), + ) + else: + update_set = sql.SQL(", ").join( + sql.SQL("{col} = EXCLUDED.{col}").format(col=sql.Identifier(c)) + for c in update_cols + ) + upsert_sql = sql.SQL( + "INSERT INTO {table} ({cols}) VALUES %s " + "ON CONFLICT ({conflict}) DO UPDATE SET {update_set}" + ).format( + table=sql.Identifier(table_name), + cols=sql.SQL(", ").join(map(sql.Identifier, columns)), + conflict=sql.SQL(conflict_str), + update_set=update_set, + ) + + cursor = conn.cursor() + try: + execute_values(cursor, upsert_sql.as_string(cursor), rows, page_size=BATCH_SIZE) + conn.commit() + logger.info(f" {table_name}: 成功导入 {len(rows)} 条记录") + return len(rows) + except Exception as e: + conn.rollback() + logger.error(f" {table_name}: 批量导入失败 - {e}") + raise + finally: + cursor.close() + + +def fetch_with_retry(fetch_func, max_retries: int = 3, delay: float = 2.0): + """ + 带重试的数据获取装饰器 + - fetch_func: 数据获取函数 (返回 DataFrame) + - max_retries: 最大重试次数 + - delay: 重试间隔(秒) + """ + for attempt in range(max_retries): + try: + result = fetch_func() + if result is not None and not result.empty: + return result + logger.warning(f" 第 {attempt+1} 次获取返回空数据，重试...") + except Exception as e: + logger.warning(f" 第 {attempt+1} 次获取失败: {e}") + if attempt < max_retries - 1: + time.sleep(delay * (attempt + 1)) # 递增延迟 + return pd.DataFrame() + + +# ============================================================ +# 1. 导入股票基本信息 +# ============================================================ + +def import_stock_basic(): + """ + 导入股票基本信息 (stock_basic) + Tushare: stock_basic + """ + logger.info("=" * 60) + logger.info("[1/7] 导入股票基本信息 (stock_basic) ...") + + pro = get_ts_pro() + conn = get_pg_connection() + + try: + # 获取全量股票基本信息 + df = pro.stock_basic( + exchange="", + list_status="L", + fields="ts_code,symbol,name,area,industry,market,list_status,list_date,is_hs,act_name,act_ent_type", + ) + if df is None or df.empty: + logger.warning("未获取到股票基本信息") + return + + df = normalize_columns(df) + + # 转换日期格式 + if "list_date" in df.columns: + df["list_date"] = pd.to_datetime(df["list_date"], format="%Y%m%d", errors="coerce") + + logger.info(f" 获取到 {len(df)} 条股票基本信息") + + # 批量导入 + conflict_cols = ["ts_code"] + batch_insert("stock_basic", df, conn, conflict_cols) + + # 也尝试获取退市的股票 + try: + df_d = pro.stock_basic( + exchange="", + list_status="D", + fields="ts_code,symbol,name,area,industry,market,list_status,list_date,is_hs", + ) + if df_d is not None and not df_d.empty: + df_d = normalize_columns(df_d) + if "list_date" in df_d.columns: + df_d["list_date"] = pd.to_datetime( + df_d["list_date"], format="%Y%m%d", errors="coerce" + ) + batch_insert("stock_basic", df_d, conn, conflict_cols) + logger.info(f" 额外导入 {len(df_d)} 条退市股票信息") + except Exception as e: + logger.warning(f" 获取退市股票信息失败: {e}") + + except Exception as e: + logger.error(f" 导入股票基本信息失败: {e}") + raise + finally: + conn.close() + + +# ============================================================ +# 2. 导入交易日历 +# ============================================================ + +def import_trade_cal(start_date: str = None, end_date: str = None): + """ + 导入交易日历 (trade_cal) + Tushare: trade_cal + """ + if start_date is None: + start_date = START_DATE + if end_date is None: + end_date = END_DATE + + logger.info("=" * 60) + logger.info(f"[2/7] 导入交易日历 (trade_cal): {start_date} ~ {end_date}") + + pro = get_ts_pro() + conn = get_pg_connection() + + try: + # 上交所 + for exchange, ex_name in [("SSE", "上交所"), ("SZSE", "深交所")]: + df = pro.trade_cal( + exchange=exchange, + start_date=start_date.replace("-", ""), + end_date=end_date.replace("-", ""), + ) + if df is None or df.empty: + logger.warning(f" {ex_name} 交易日历为空") + continue + + df = normalize_columns(df) + + # 转换日期 + for col in ["cal_date", "pretrade_date"]: + if col in df.columns: + df[col] = pd.to_datetime(df[col], format="%Y%m%d", errors="coerce") + + # 重命名 is_open (Tushare 返回 0/1 整数) + if "is_open" in df.columns: + df["is_open"] = df["is_open"].astype(int) + + conflict_cols = ["exchange", "cal_date"] + batch_insert("trade_cal", df, conn, conflict_cols) + logger.info(f" {ex_name}: {len(df)} 条交易日历") + + except Exception as e: + logger.error(f" 导入交易日历失败: {e}") + raise + finally: + conn.close() + + time.sleep(0.5) # API 频率限制 + + +# ============================================================ +# 3. 导入日线行情 (核心表) +# ============================================================ + +def import_daily_for_stock(ts_code: str, start_date: str, end_date: str, conn) -> int: + """ + 导入单只股票的日线行情 + 返回导入的记录数 + """ + pro = get_ts_pro() + + def fetch(): + return pro.daily( + ts_code=ts_code, + start_date=start_date.replace("-", ""), + end_date=end_date.replace("-", ""), + ) + + df = fetch_with_retry(fetch, max_retries=2) + if df is None or df.empty: + return 0 + + df = normalize_columns(df) + + # 转换日期 + if "trade_date" in df.columns: + df["trade_date"] = pd.to_datetime(df["trade_date"], format="%Y%m%d", errors="coerce") + + # 数值列处理 NaN + numeric_cols = [ + "open", "high", "low", "close", "pre_close", "change", "pct_chg", + "vol", "amount", + ] + for col in numeric_cols: + if col in df.columns: + df[col] = pd.to_numeric(df[col], errors="coerce") + + # 额外列 (Tushare Pro 不同版本返回字段可能不同) + for col in ["turnover_rate", "volume_ratio", "ma5", "ma10", "ma20", "ma_v_5", "ma_v_10", "ma_v_20"]: + if col not in df.columns: + df[col] = None + + conflict_cols = ["ts_code", "trade_date"] + return batch_insert("daily", df, conn, conflict_cols) + + +def import_daily_batch( + stock_list: List[str], + start_date: str = None, + end_date: str = None, + sleep_interval: float = 0.3, +): + """ + 批量导入多只股票的日线行情 + - stock_list: 股票代码列表 + - sleep_interval: API 调用间隔 (避免频率限制) + """ + if start_date is None: + start_date = START_DATE + if end_date is None: + end_date = END_DATE + + total = len(stock_list) + logger.info("=" * 60) + logger.info( + f"[3/7] 导入日线行情 (daily): {start_date} ~ {end_date}, " + f"共 {total} 只股票" + ) + + conn = get_pg_connection() + success_count = 0 + fail_list = [] + + for i, ts_code in enumerate(stock_list, 1): + try: + n = import_daily_for_stock(ts_code, start_date, end_date, conn) + if n > 0: + success_count += 1 + if i % 50 == 0 or i == total: + logger.info(f" 进度: {i}/{total} 成功={success_count} 失败={len(fail_list)}") + except Exception as e: + logger.error(f" [{ts_code}] 导入失败: {e}") + fail_list.append(ts_code) + conn.rollback() + + time.sleep(sleep_interval) # API 频率控制 + + conn.close() + logger.info(f" 日线行情导入完成: 成功 {success_count}/{total}") + + if fail_list: + logger.warning(f" 失败列表({len(fail_list)}): {fail_list[:20]}...") + return fail_list + + +def import_daily_by_year( + stock_list: List[str], + start_year: int = 2010, + end_year: int = 2025, +): + """ + 按年份逐批导入日线行情 (断点续传友好) + 适合大数据量导入，每年每只股票可单独重试 + """ + logger.info("=" * 60) + logger.info( + f"[3/7] 按年导入日线行情: {start_year} ~ {end_year}, " + f"共 {len(stock_list)} 只股票" + ) + + total_imported = 0 + for year in range(start_year, end_year + 1): + year_start = f"{year}-01-01" + year_end = f"{year}-12-31" + logger.info(f"--- 导入 {year} 年日线行情 ---") + fail_list = import_daily_batch( + stock_list, + start_date=year_start, + end_date=year_end, + sleep_interval=0.2, + ) + total_imported += 1 + logger.info(f" {year} 年完成\n") + + logger.info(f" 所有年份日线行情导入完成!") + + +# ============================================================ +# 4. 导入每日指标 (daily_basic) - 估值/基本面 +# ============================================================ + +def import_daily_basic( + ts_code: str = None, + trade_date: str = None, + start_date: str = None, + end_date: str = None, + conn=None, +): + """ + 导入每日指标 (daily_basic) + Tushare: daily_basic + - 支持按单只股票导入 + - 支持按日期范围全市场导入 + """ + if start_date is None: + start_date = START_DATE + if end_date is None: + end_date = END_DATE + + pro = get_ts_pro() + own_conn = conn is None + if own_conn: + conn = get_pg_connection() + + try: + # 构建参数 + kwargs = { + "start_date": start_date.replace("-", ""), + "end_date": end_date.replace("-", ""), + } + if ts_code: + kwargs["ts_code"] = ts_code + if trade_date: + kwargs["trade_date"] = trade_date.replace("-", "") + + def fetch(): + return pro.daily_basic(**kwargs) + + df = fetch_with_retry(fetch, max_retries=2) + if df is None or df.empty: + return 0 + + df = normalize_columns(df) + + if "trade_date" in df.columns: + df["trade_date"] = pd.to_datetime(df["trade_date"], format="%Y%m%d", errors="coerce") + + numeric_cols = df.select_dtypes(include=["number"]).columns.tolist() + for col in numeric_cols: + df[col] = pd.to_numeric(df[col], errors="coerce") + + conflict_cols = ["ts_code", "trade_date"] + return batch_insert("daily_basic", df, conn, conflict_cols) + + finally: + if own_conn: + conn.close() + + +def import_daily_basic_by_date( + start_date: str = None, + end_date: str = None, +): + """ + 按日期批量导入每日指标 (全市场) + Tushare daily_basic 接口可按交易日获取全市场数据，比较高效 + """ + if start_date is None: + start_date = START_DATE + if end_date is None: + end_date = END_DATE + + logger.info("=" * 60) + logger.info(f"[4/7] 导入每日指标 (daily_basic): {start_date} ~ {end_date}") + + # 获取交易日列表 + conn = get_pg_connection() + try: + cursor = conn.cursor() + cursor.execute( + """ + SELECT DISTINCT cal_date FROM trade_cal + WHERE is_open = 1 + AND cal_date >= %s AND cal_date <= %s + ORDER BY cal_date + """, + (start_date, end_date), + ) + trade_dates = [row[0].strftime("%Y%m%d") for row in cursor.fetchall()] + cursor.close() + finally: + conn.close() + + total = len(trade_dates) + logger.info(f" 共 {total} 个交易日") + + conn = get_pg_connection() + for i, td in enumerate(trade_dates, 1): + try: + pro = get_ts_pro() + df = pro.daily_basic(trade_date=td) + if df is not None and not df.empty: + df = normalize_columns(df) + if "trade_date" in df.columns: + df["trade_date"] = pd.to_datetime(df["trade_date"], format="%Y%m%d", errors="coerce") + conflict_cols = ["ts_code", "trade_date"] + batch_insert("daily_basic", df, conn, conflict_cols) + except Exception as e: + logger.warning(f" [{td}] 导入失败: {e}") + conn.rollback() + + if i % 20 == 0 or i == total: + logger.info(f" 进度: {i}/{total}") + time.sleep(0.3) + + conn.close() + logger.info(" 每日指标导入完成") + + +# ============================================================ +# 5. 导入复权因子 +# ============================================================ + +def import_adj_factor( + ts_code: str = None, + start_date: str = None, + end_date: str = None, + conn=None, +): + """ + 导入复权因子 (adj_factor) + Tushare: adj_factor + """ + if start_date is None: + start_date = START_DATE + if end_date is None: + end_date = END_DATE + + pro = get_ts_pro() + own_conn = conn is None + if own_conn: + conn = get_pg_connection() + + try: + kwargs = { + "start_date": start_date.replace("-", ""), + "end_date": end_date.replace("-", ""), + } + if ts_code: + kwargs["ts_code"] = ts_code + + def fetch(): + return pro.adj_factor(**kwargs) + + df = fetch_with_retry(fetch, max_retries=2) + if df is None or df.empty: + return 0 + + df = normalize_columns(df) + + if "trade_date" in df.columns: + df["trade_date"] = pd.to_datetime(df["trade_date"], format="%Y%m%d", errors="coerce") + + conflict_cols = ["ts_code", "trade_date"] + return batch_insert("adj_factor", df, conn, conflict_cols) + + finally: + if own_conn: + conn.close() + + +def import_adj_factor_batch( + stock_list: List[str], + start_date: str = None, + end_date: str = None, +): + """批量导入复权因子""" + if start_date is None: + start_date = START_DATE + if end_date is None: + end_date = END_DATE + + total = len(stock_list) + logger.info("=" * 60) + logger.info(f"[5/7] 导入复权因子 (adj_factor): {total} 只股票") + + conn = get_pg_connection() + for i, ts_code in enumerate(stock_list, 1): + try: + import_adj_factor(ts_code=ts_code, start_date=start_date, end_date=end_date, conn=conn) + except Exception as e: + logger.warning(f" [{ts_code}] 复权因子导入失败: {e}") + conn.rollback() + + if i % 100 == 0 or i == total: + logger.info(f" 进度: {i}/{total}") + time.sleep(0.25) + + conn.close() + logger.info(" 复权因子导入完成") + + +# ============================================================ +# 6. 导入财务数据 (利润表、资产负债表、现金流量表、财务指标) +# ============================================================ + +def import_financial_statements( + stock_list: List[str], + start_date: str = None, + end_date: str = None, +): + """ + 按股票批量导入三大报表 + 财务指标 + - income: 利润表 + - balancesheet: 资产负债表 + - cashflow: 现金流量表 + - fina_indicator: 财务指标 + """ + if start_date is None: + start_date = START_DATE + if end_date is None: + end_date = END_DATE + + total = len(stock_list) + logger.info("=" * 60) + logger.info( + f"[6/7] 导入财务数据: {start_date} ~ {end_date}, " + f"共 {total} 只股票" + ) + + period_start = start_date.replace("-", "") + period_end = end_date.replace("-", "") + pro = get_ts_pro() + conn = get_pg_connection() + + for i, ts_code in enumerate(stock_list, 1): + for table_name, fetch_method in [ + ("income", pro.income), + ("balancesheet", pro.balancesheet), + ("cashflow", pro.cashflow), + ("fina_indicator", pro.fina_indicator), + ]: + try: + if table_name == "fina_indicator": + # fina_indicator 参数略有不同 + df = fetch_method( + ts_code=ts_code, + start_date=period_start, + end_date=period_end, + ) + else: + df = fetch_method( + ts_code=ts_code, + start_date=period_start, + end_date=period_end, + ) + + if df is None or df.empty: + continue + + df = normalize_columns(df) + + # 转换日期列 + for col in ["ann_date", "f_ann_date", "end_date"]: + if col in df.columns: + df[col] = pd.to_datetime(df[col], format="%Y%m%d", errors="coerce") + + if table_name == "fina_indicator": + conflict_cols = ["ts_code", "end_date"] + else: + conflict_cols = ["ts_code", "end_date", "report_type"] + + batch_insert(table_name, df, conn, conflict_cols) + + except Exception as e: + logger.debug(f" [{ts_code}] {table_name}: {e}") + conn.rollback() + + if i % 50 == 0 or i == total: + logger.info(f" 财务数据进度: {i}/{total}") + time.sleep(0.3) + + conn.close() + logger.info(" 财务数据导入完成") + + +# ============================================================ +# 7. 导入指数日线行情 +# ============================================================ + +def import_index_daily( + index_codes: List[str] = None, + start_date: str = None, + end_date: str = None, +): + """ + 导入指数日线行情 (index_daily) + 默认导入主要指数：上证指数、深证成指、沪深300、中证500、创业板指、科创50 + """ + if index_codes is None: + index_codes = [ + "000001.SH", # 上证指数 + "399001.SZ", # 深证成指 + "000300.SH", # 沪深300 + "000905.SH", # 中证500 + "399006.SZ", # 创业板指 + "000688.SH", # 科创50 + ] + if start_date is None: + start_date = START_DATE + if end_date is None: + end_date = END_DATE + + logger.info("=" * 60) + logger.info(f"[7/7] 导入指数日线行情: {len(index_codes)} 个指数") + + pro = get_ts_pro() + conn = get_pg_connection() + + for idx_code in index_codes: + try: + def fetch(): + return pro.index_daily( + ts_code=idx_code, + start_date=start_date.replace("-", ""), + end_date=end_date.replace("-", ""), + ) + + df = fetch_with_retry(fetch, max_retries=2) + if df is not None and not df.empty: + df = normalize_columns(df) + if "trade_date" in df.columns: + df["trade_date"] = pd.to_datetime( + df["trade_date"], format="%Y%m%d", errors="coerce" + ) + conflict_cols = ["ts_code", "trade_date"] + n = batch_insert("index_daily", df, conn, conflict_cols) + logger.info(f" {idx_code}: {n} 条") + else: + logger.warning(f" {idx_code}: 无数据") + except Exception as e: + logger.error(f" {idx_code}: {e}") + conn.rollback() + time.sleep(0.3) + + conn.close() + logger.info(" 指数日线行情导入完成") + + +# ============================================================ +# 8. 初始化数据库 Schema +# ============================================================ + +def init_database(): + """ + 执行 DDL，创建所有表结构 + """ + logger.info("=" * 60) + logger.info("初始化数据库 Schema ...") + + # 尝试创建数据库 (如果不存在) + try: + admin_conn = psycopg2.connect( + host=DB_CONFIG["host"], + port=DB_CONFIG["port"], + database="postgres", # 连接默认 postgres 库来创建新库 + user=DB_CONFIG["user"], + password=DB_CONFIG["password"], + ) + admin_conn.autocommit = True + cursor = admin_conn.cursor() + + # 检查数据库是否存在 + cursor.execute( + "SELECT 1 FROM pg_database WHERE datname = %s", + (DB_CONFIG["database"],), + ) + if cursor.fetchone() is None: + cursor.execute( + sql.SQL("CREATE DATABASE {} ENCODING 'UTF8'").format( + sql.Identifier(DB_CONFIG["database"]) + ) + ) + logger.info(f" 数据库 {DB_CONFIG['database']} 创建成功") + else: + logger.info(f" 数据库 {DB_CONFIG['database']} 已存在") + + cursor.close() + admin_conn.close() + except Exception as e: + logger.warning(f" 创建数据库步骤跳过 (可能无权限): {e}") + + # 执行 DDL + conn = get_pg_connection() + try: + with open("schema.sql", "r", encoding="utf-8") as f: + ddl_sql = f.read() + + # 按分号分割，逐条执行 (忽略被注释掉的分区表DDL) + statements = [s.strip() for s in ddl_sql.split(";") if s.strip() + and not s.strip().startswith("--")] + cursor = conn.cursor() + for stmt in statements: + if stmt and not stmt.startswith("--"): + try: + cursor.execute(stmt) + except Exception as e: + logger.debug(f" SQL 跳过: {str(e)[:100]}") + + conn.commit() + cursor.close() + logger.info(" Schema 初始化完成") + except Exception as e: + conn.rollback() + logger.error(f" Schema 初始化失败: {e}") + raise + finally: + conn.close() + + +# ============================================================ +# 9. 获取所有股票列表 (辅助) +# ============================================================ + +def get_all_stock_codes(include_delisted: bool = False) -> List[str]: + """ + 从 Tushare 获取所有 A 股股票代码列表 + """ + pro = get_ts_pro() + codes = [] + + for status, label in [("L", "上市"), ("D", "退市"), ("P", "暂停")]: + if status != "L" and not include_delisted: + continue + try: + df = pro.stock_basic( + exchange="", + list_status=status, + fields="ts_code", + ) + if df is not None and not df.empty: + codes.extend(df["ts_code"].tolist()) + except Exception as e: + logger.warning(f" 获取 {label} 股票列表失败: {e}") + + logger.info(f" 获取到 {len(codes)} 只股票代码") + return codes + + +def get_stock_codes_from_db(conn=None) -> List[str]: + """ + 从已导入的 stock_basic 表获取股票代码列表 + """ + own_conn = conn is None + if own_conn: + conn = get_pg_connection() + + try: + cursor = conn.cursor() + cursor.execute("SELECT ts_code FROM stock_basic WHERE list_status = 'L' ORDER BY ts_code") + codes = [row[0] for row in cursor.fetchall()] + cursor.close() + return codes + finally: + if own_conn: + conn.close() + + +# ============================================================ +# 10. 一键全量导入 +# ============================================================ + +def full_import( + start_date: str = None, + end_date: str = None, + import_financials: bool = True, + stock_codes: List[str] = None, +): + """ + 一键全量导入： + 1. 初始化 Schema + 2. 股票基本信息 + 3. 交易日历 + 4. 日线行情 + 5. 每日指标(估值) + 6. 复权因子 + 7. 财务数据 (可选) + 8. 指数日线行情 + + 参数: + - start_date, end_date: 数据范围 + - import_financials: 是否导入财务数据 (耗时较长) + - stock_codes: 指定股票列表，不传则全量导入 + """ + if start_date is None: + start_date = START_DATE + if end_date is None: + end_date = END_DATE + + start_time = datetime.now() + logger.info("=" * 70) + logger.info(f" 开始全量数据导入: {start_date} ~ {end_date}") + logger.info(f" PostgreSQL: {DB_CONFIG['host']}:{DB_CONFIG['port']}/{DB_CONFIG['database']}") + logger.info("=" * 70) + + # Step 0: 初始化 Schema + init_database() + + # Step 1: 股票基本信息 + import_stock_basic() + + # Step 2: 交易日历 + import_trade_cal(start_date, end_date) + + # 获取股票列表 + if stock_codes is None: + stock_codes = get_stock_codes_from_db() + if not stock_codes: + logger.error("无法获取股票列表，请先导入 stock_basic") + return + + # Step 3: 日线行情 (按年导入) + import_daily_by_year( + stock_codes, + start_year=int(start_date[:4]), + end_year=int(end_date[:4]), + ) + + # Step 4: 每日指标 (按日期导入) + import_daily_basic_by_date(start_date, end_date) + + # Step 5: 复权因子 + import_adj_factor_batch(stock_codes, start_date, end_date) + + # Step 6: 财务数据 + if import_financials: + import_financial_statements(stock_codes, start_date, end_date) + + # Step 7: 指数日线行情 + import_index_daily(start_date=start_date, end_date=end_date) + + elapsed = datetime.now() - start_time + logger.info("=" * 70) + logger.info(f" 全量数据导入完成! 总耗时: {elapsed}") + logger.info("=" * 70) + + +if __name__ == "__main__": + # 测试连接 + try: + conn = get_pg_connection() + logger.info(f"成功连接到 PostgreSQL: {DB_CONFIG['host']}:{DB_CONFIG['port']}") + conn.close() + except Exception as e: + logger.error(f"无法连接到 PostgreSQL: {e}") + logger.error("请确认 Docker 容器已启动，且 config.py 中的连接参数正确") \ No newline at end of file diff --git a/quantitative_data/requirements.txt b/quantitative_data/requirements.txt new file mode 100644 index 0000000..d8b1d64 --- /dev/null +++ b/quantitative_data/requirements.txt @@ -0,0 +1,4 @@ +tushare>=1.4.0 +pandas>=1.5.0 +psycopg2-binary>=2.9.0 +sqlalchemy>=2.0.0 \ No newline at end of file diff --git a/quantitative_data/schema.sql b/quantitative_data/schema.sql new file mode 100644 index 0000000..e19c387 --- /dev/null +++ b/quantitative_data/schema.sql @@ -0,0 +1,576 @@ +-- ============================================================ +-- 量化投资数据库 Schema +-- PostgreSQL (Docker: 192.168.27.15:12345) +-- 数据来源：Tushare +-- ============================================================ + +-- 先创建数据库（需要超级用户权限，在 psql 中手动执行）： +-- CREATE DATABASE quant_db ENCODING 'UTF8'; + +-- ============================================================ +-- 1. 股票基本信息表 (stock_basic) +-- tushare接口: stock_basic +-- ============================================================ +CREATE TABLE IF NOT EXISTS stock_basic ( + ts_code VARCHAR(20) PRIMARY KEY, -- 股票代码 (如 000001.SZ) + symbol VARCHAR(10) NOT NULL, -- 股票代码 (如 000001) + name VARCHAR(50) NOT NULL, -- 股票名称 + area VARCHAR(30), -- 地区 + industry VARCHAR(50), -- 行业 + market VARCHAR(10), -- 市场类型 (主板/中小板/创业板/科创板) + list_date DATE, -- 上市日期 + list_status CHAR(1) DEFAULT 'L', -- 上市状态 L上市 D退市 P暂停上市 + is_hs VARCHAR(5), -- 是否沪深港通标的 + act_name VARCHAR(100), -- 实控人名称 + act_ent_type VARCHAR(50), -- 实控人企业性质 + updated_at TIMESTAMP DEFAULT NOW() -- 记录更新时间 +); + +COMMENT ON TABLE stock_basic IS '股票基本信息表 (Tushare: stock_basic)'; + +-- 索引 +CREATE INDEX IF NOT EXISTS idx_stock_basic_industry ON stock_basic(industry); +CREATE INDEX IF NOT EXISTS idx_stock_basic_area ON stock_basic(area); +CREATE INDEX IF NOT EXISTS idx_stock_basic_list_status ON stock_basic(list_status); + + +-- ============================================================ +-- 2. 日线行情表 (daily) - 按年份分区 +-- tushare接口: daily / pro_bar +-- ============================================================ +CREATE TABLE IF NOT EXISTS daily ( + id BIGSERIAL, -- 自增主键 + ts_code VARCHAR(20) NOT NULL, -- 股票代码 + trade_date DATE NOT NULL, -- 交易日期 + open NUMERIC(12,3), -- 开盘价 + high NUMERIC(12,3), -- 最高价 + low NUMERIC(12,3), -- 最低价 + close NUMERIC(12,3), -- 收盘价 + pre_close NUMERIC(12,3), -- 昨收价 + change NUMERIC(12,3), -- 涨跌额 + pct_chg NUMERIC(10,4), -- 涨跌幅 (%) + vol NUMERIC(20,2), -- 成交量 (手) + amount NUMERIC(20,4), -- 成交额 (千元) + turnover_rate NUMERIC(10,4), -- 换手率 (%) + volume_ratio NUMERIC(10,4), -- 量比 + ma5 NUMERIC(12,3), -- 5日均价 + ma10 NUMERIC(12,3), -- 10日均价 + ma20 NUMERIC(12,3), -- 20日均价 + ma_v_5 NUMERIC(20,2), -- 5日均量 + ma_v_10 NUMERIC(20,2), -- 10日均量 + ma_v_20 NUMERIC(20,2), -- 20日均量 + updated_at TIMESTAMP DEFAULT NOW(), + + PRIMARY KEY (ts_code, trade_date) +); + +COMMENT ON TABLE daily IS '日线行情表 (Tushare: daily)'; + +-- 核心索引 +CREATE INDEX IF NOT EXISTS idx_daily_trade_date ON daily(trade_date); +CREATE INDEX IF NOT EXISTS idx_daily_ts_code ON daily(ts_code); +CREATE INDEX IF NOT EXISTS idx_daily_ts_code_date ON daily(ts_code, trade_date); + + +-- ============================================================ +-- 3. 日线行情表 - 按年份分区子表（优化查询性能） +-- PostgreSQL 原生分区表 (PG 10+) +-- 如果版本不支持，可以跳过，直接使用 daily 表 +-- ============================================================ + +-- 仅 PG 10+ 支持以下语法，如果报错请删除分区表相关语句 +-- CREATE TABLE daily_partitioned ( +-- id BIGSERIAL, +-- ts_code VARCHAR(20) NOT NULL, +-- trade_date DATE NOT NULL, +-- open NUMERIC(12,3), +-- high NUMERIC(12,3), +-- low NUMERIC(12,3), +-- close NUMERIC(12,3), +-- pre_close NUMERIC(12,3), +-- change NUMERIC(12,3), +-- pct_chg NUMERIC(10,4), +-- vol NUMERIC(20,2), +-- amount NUMERIC(20,4), +-- turnover_rate NUMERIC(10,4), +-- volume_ratio NUMERIC(10,4), +-- ma5 NUMERIC(12,3), +-- ma10 NUMERIC(12,3), +-- ma20 NUMERIC(12,3), +-- ma_v_5 NUMERIC(20,2), +-- ma_v_10 NUMERIC(20,2), +-- ma_v_20 NUMERIC(20,2), +-- updated_at TIMESTAMP DEFAULT NOW(), +-- PRIMARY KEY (ts_code, trade_date, trade_date) -- 分区键必须包含在主键中 +-- ) PARTITION BY RANGE (trade_date); + + +-- ============================================================ +-- 4. 复权因子表 (adj_factor) +-- tushare接口: adj_factor +-- 用于计算前复权/后复权价格 +-- ============================================================ +CREATE TABLE IF NOT EXISTS adj_factor ( + ts_code VARCHAR(20) NOT NULL, + trade_date DATE NOT NULL, + adj_factor NUMERIC(16,8), -- 复权因子 + updated_at TIMESTAMP DEFAULT NOW(), + + PRIMARY KEY (ts_code, trade_date) +); + +COMMENT ON TABLE adj_factor IS '复权因子表 (Tushare: adj_factor)'; + + +-- ============================================================ +-- 5. 个股资金流向表 (moneyflow) +-- tushare接口: moneyflow +-- ============================================================ +CREATE TABLE IF NOT EXISTS moneyflow ( + ts_code VARCHAR(20) NOT NULL, + trade_date DATE NOT NULL, + buy_sm_vol NUMERIC(20,2), -- 小单买入量(手) + buy_sm_amount NUMERIC(20,4), -- 小单买入金额(万元) + sell_sm_vol NUMERIC(20,2), -- 小单卖出量(手) + sell_sm_amount NUMERIC(20,4), -- 小单卖出金额(万元) + buy_md_vol NUMERIC(20,2), -- 中单买入量(手) + buy_md_amount NUMERIC(20,4), -- 中单买入金额(万元) + sell_md_vol NUMERIC(20,2), -- 中单卖出量(手) + sell_md_amount NUMERIC(20,4), -- 中单卖出金额(万元) + buy_lg_vol NUMERIC(20,2), -- 大单买入量(手) + buy_lg_amount NUMERIC(20,4), -- 大单买入金额(万元) + sell_lg_vol NUMERIC(20,2), -- 大单卖出量(手) + sell_lg_amount NUMERIC(20,4), -- 大单卖出金额(万元) + buy_elg_vol NUMERIC(20,2), -- 特大单买入量(手) + buy_elg_amount NUMERIC(20,4), -- 特大单买入金额(万元) + sell_elg_vol NUMERIC(20,2), -- 特大单卖出量(手) + sell_elg_amount NUMERIC(20,4), -- 特大单卖出金额(万元) + net_mf_vol NUMERIC(20,2), -- 净流入量(手) + net_mf_amount NUMERIC(20,4), -- 净流入额(万元) + updated_at TIMESTAMP DEFAULT NOW(), + + PRIMARY KEY (ts_code, trade_date) +); + +COMMENT ON TABLE moneyflow IS '个股资金流向表 (Tushare: moneyflow)'; + + +-- ============================================================ +-- 6. 每日指标表 (daily_basic) - 包含基本面/估值数据 +-- tushare接口: daily_basic +-- ============================================================ +CREATE TABLE IF NOT EXISTS daily_basic ( + ts_code VARCHAR(20) NOT NULL, + trade_date DATE NOT NULL, + close NUMERIC(12,3), -- 收盘价 + turnover_rate NUMERIC(10,4), -- 换手率(%) + turnover_rate_f NUMERIC(10,4), -- 自由流通股换手率(%) + volume_ratio NUMERIC(10,4), -- 量比 + pe NUMERIC(16,4), -- 市盈率(PE) + pe_ttm NUMERIC(16,4), -- 市盈率(TTM) + pb NUMERIC(16,4), -- 市净率(PB) + ps NUMERIC(16,4), -- 市销率(PS) + ps_ttm NUMERIC(16,4), -- 市销率(TTM) + dv_ratio NUMERIC(10,4), -- 股息率(%) + dv_ttm NUMERIC(10,4), -- 股息率(TTM)(%) + total_share NUMERIC(20,4), -- 总股本(万股) + float_share NUMERIC(20,4), -- 流通股本(万股) + free_share NUMERIC(20,4), -- 自由流通股本(万股) + total_mv NUMERIC(20,4), -- 总市值(万元) + circ_mv NUMERIC(20,4), -- 流通市值(万元) + updated_at TIMESTAMP DEFAULT NOW(), + + PRIMARY KEY (ts_code, trade_date) +); + +COMMENT ON TABLE daily_basic IS '每日指标表/基本面估值表 (Tushare: daily_basic)'; + + +-- ============================================================ +-- 7. 利润表 (income) +-- tushare接口: income +-- ============================================================ +CREATE TABLE IF NOT EXISTS income ( + ts_code VARCHAR(20) NOT NULL, + ann_date DATE, -- 公告日期 + f_ann_date DATE, -- 实际公告日期 + end_date DATE NOT NULL, -- 报告期 + report_type VARCHAR(10), -- 报告类型 + comp_type CHAR(1), -- 公司类型 + end_type VARCHAR(10), -- 报告期类型 + basic_eps NUMERIC(16,4), -- 基本每股收益 + diluted_eps NUMERIC(16,4), -- 稀释每股收益 + total_revenue NUMERIC(20,4), -- 营业总收入(元) + revenue NUMERIC(20,4), -- 营业收入(元) + int_income NUMERIC(20,4), -- 利息收入 + prem_earned NUMERIC(20,4), -- 已赚保费 + comm_income NUMERIC(20,4), -- 手续费及佣金收入 + n_commis_income NUMERIC(20,4), -- 手续费及佣金净收入 + n_oth_income NUMERIC(20,4), -- 其他经营净收益 + n_oth_b_income NUMERIC(20,4), -- 加:其他业务净收益 + prem_income NUMERIC(20,4), -- 保险业务收入 + out_prem NUMERIC(20,4), -- 减:分出保费 + une_prem_reser NUMERIC(20,4), -- 提取未到期责任准备金 + reins_income NUMERIC(20,4), -- 其中:分保费收入 + n_sec_tb_income NUMERIC(20,4), -- 代理买卖证券业务净收入 + n_sec_uw_income NUMERIC(20,4), -- 证券承销业务净收入 + n_asset_mg_income NUMERIC(20,4), -- 受托客户资产管理业务净收入 + oth_b_income NUMERIC(20,4), -- 其他业务收入 + fv_value_chg NUMERIC(20,4), -- 加:公允价值变动净收益 + invest_income NUMERIC(20,4), -- 加:投资净收益 + ass_invest_income NUMERIC(20,4), -- 其中:对联营企业和合营企业的投资收益 + forex_gain NUMERIC(20,4), -- 加:汇兑净收益 + total_cogs NUMERIC(20,4), -- 营业总成本 + oper_cost NUMERIC(20,4), -- 减:营业成本 + int_exp NUMERIC(20,4), -- 减:利息支出 + comm_exp NUMERIC(20,4), -- 减:手续费及佣金支出 + biz_tax_surchg NUMERIC(20,4), -- 减:营业税金及附加 + sell_exp NUMERIC(20,4), -- 减:销售费用 + admin_exp NUMERIC(20,4), -- 减:管理费用 + fin_exp NUMERIC(20,4), -- 减:财务费用 + assets_impair_loss NUMERIC(20,4), -- 减:资产减值损失 + prem_refund NUMERIC(20,4), -- 退保金 + compens_payout NUMERIC(20,4), -- 赔付总支出 + reser_insur_liab NUMERIC(20,4), -- 提取保险责任准备金 + div_payt NUMERIC(20,4), -- 保户红利支出 + reins_exp NUMERIC(20,4), -- 分保费用 + oper_exp NUMERIC(20,4), -- 营业支出 + compens_payout_refu NUMERIC(20,4), -- 减:摊回赔付支出 + insur_reser_refu NUMERIC(20,4), -- 减:摊回保险责任准备金 + reins_cost_refund NUMERIC(20,4), -- 减:摊回分保费用 + other_bus_cost NUMERIC(20,4), -- 其他业务成本 + operate_profit NUMERIC(20,4), -- 营业利润 + non_oper_income NUMERIC(20,4), -- 加:营业外收入 + non_oper_exp NUMERIC(20,4), -- 减:营业外支出 + nca_disploss NUMERIC(20,4), -- 其中:减:非流动资产处置净损失 + total_profit NUMERIC(20,4), -- 利润总额 + income_tax NUMERIC(20,4), -- 所得税费用 + n_income NUMERIC(20,4), -- 净利润(含少数股东损益) + n_income_attr_p NUMERIC(20,4), -- 净利润(不含少数股东损益) + minority_gain NUMERIC(20,4), -- 少数股东损益 + oth_compre_income NUMERIC(20,4), -- 其他综合收益 + total_compre_income NUMERIC(20,4), -- 综合收益总额 + comp_pre_inc_attr_p NUMERIC(20,4), -- 归属于母公司(或股东)的综合收益总额 + comp_inc_attr_m_s NUMERIC(20,4), -- 归属于少数股东的综合收益总额 + ebit NUMERIC(20,4), -- 息税前利润 + ebitda NUMERIC(20,4), -- 息税折旧摊销前利润 + insurance_exp NUMERIC(20,4), -- 保险业务支出 + undist_profit NUMERIC(20,4), -- 年初未分配利润 + distable_profit NUMERIC(20,4), -- 可分配利润 + updated_at TIMESTAMP DEFAULT NOW(), + + PRIMARY KEY (ts_code, end_date, report_type) +); + +COMMENT ON TABLE income IS '利润表 (Tushare: income)'; + + +-- ============================================================ +-- 8. 资产负债表 (balancesheet) +-- tushare接口: balancesheet +-- ============================================================ +CREATE TABLE IF NOT EXISTS balancesheet ( + ts_code VARCHAR(20) NOT NULL, + ann_date DATE, + f_ann_date DATE, + end_date DATE NOT NULL, -- 报告期 + report_type VARCHAR(10), + comp_type CHAR(1), + end_type VARCHAR(10), + + -- 资产 + total_hldr_eqy_exc_min_int NUMERIC(20,4), -- 股东权益合计(不含少数股东权益) + total_hldr_eqy_inc_min_int NUMERIC(20,4), -- 股东权益合计(含少数股东权益) + total_assets NUMERIC(20,4), -- 资产总计 + total_cur_assets NUMERIC(20,4), -- 流动资产合计 + total_nca NUMERIC(20,4), -- 非流动资产合计 + money_cap NUMERIC(20,4), -- 货币资金 + trad_asset NUMERIC(20,4), -- 交易性金融资产 + notes_receiv NUMERIC(20,4), -- 应收票据 + account_receiv NUMERIC(20,4), -- 应收账款 + oth_receiv NUMERIC(20,4), -- 其他应收款 + prepayment NUMERIC(20,4), -- 预付款项 + div_receiv NUMERIC(20,4), -- 应收股利 + int_receiv NUMERIC(20,4), -- 应收利息 + inventories NUMERIC(20,4), -- 存货 + amor_exp NUMERIC(20,4), -- 待摊费用 + nca_within_1y NUMERIC(20,4), -- 一年内到期的非流动资产 + sett_rsrv NUMERIC(20,4), -- 结算备付金 + loanto_oth_bank_fi NUMERIC(20,4), -- 拆出资金 + premium_receiv NUMERIC(20,4), -- 应收保费 + reinsur_receiv NUMERIC(20,4), -- 应收分保账款 + reinsur_res_receiv NUMERIC(20,4), -- 应收分保合同准备金 + pur_resale_fa NUMERIC(20,4), -- 买入返售金融资产 + oth_cur_assets NUMERIC(20,4), -- 其他流动资产 + + -- 非流动资产 + fa_avail_for_sale NUMERIC(20,4), -- 可供出售金融资产 + htm_invest NUMERIC(20,4), -- 持有至到期投资 + lt_eqt_invest NUMERIC(20,4), -- 长期股权投资 + invest_real_estate NUMERIC(20,4), -- 投资性房地产 + time_deposits NUMERIC(20,4), -- 定期存款 + oth_assets NUMERIC(20,4), -- 其他资产 + lt_rec NUMERIC(20,4), -- 长期应收款 + fix_assets NUMERIC(20,4), -- 固定资产 + cip NUMERIC(20,4), -- 在建工程 + const_materials NUMERIC(20,4), -- 工程物资 + fixed_assets_disp NUMERIC(20,4), -- 固定资产清理 + produc_bio_assets NUMERIC(20,4), -- 生产性生物资产 + oil_and_gas_assets NUMERIC(20,4), -- 油气资产 + intan_assets NUMERIC(20,4), -- 无形资产 + r_and_d NUMERIC(20,4), -- 研发支出 + goodwill NUMERIC(20,4), -- 商誉 + lt_amor_exp NUMERIC(20,4), -- 长期待摊费用 + defer_tax_assets NUMERIC(20,4), -- 递延所得税资产 + decr_in_disbur NUMERIC(20,4), -- 待处理流动资产损益 + oth_nca NUMERIC(20,4), -- 其他非流动资产 + + -- 负债 + total_liab NUMERIC(20,4), -- 负债合计 + total_cur_liab NUMERIC(20,4), -- 流动负债合计 + total_ncl NUMERIC(20,4), -- 非流动负债合计 + short_borr NUMERIC(20,4), -- 短期借款 + lt_borr NUMERIC(20,4), -- 长期借款 + st_notes_payable NUMERIC(20,4), -- 应付票据 + acct_payable NUMERIC(20,4), -- 应付账款 + adv_receipts NUMERIC(20,4), -- 预收款项 + bonds_payable NUMERIC(20,4), -- 应付债券 + oth_payable NUMERIC(20,4), -- 其他应付款 + div_payable NUMERIC(20,4), -- 应付股利 + int_payable NUMERIC(20,4), -- 应付利息 + lt_payable NUMERIC(20,4), -- 长期应付款 + special_payable NUMERIC(20,4), -- 专项应付款 + updated_at TIMESTAMP DEFAULT NOW(), + + PRIMARY KEY (ts_code, end_date, report_type) +); + +COMMENT ON TABLE balancesheet IS '资产负债表 (Tushare: balancesheet)'; + + +-- ============================================================ +-- 9. 现金流量表 (cashflow) +-- tushare接口: cashflow +-- ============================================================ +CREATE TABLE IF NOT EXISTS cashflow ( + ts_code VARCHAR(20) NOT NULL, + ann_date DATE, + f_ann_date DATE, + end_date DATE NOT NULL, -- 报告期 + report_type VARCHAR(10), + comp_type CHAR(1), + end_type VARCHAR(10), + + -- 经营活动 + net_profit NUMERIC(20,4), -- 净利润 + fin_exp NUMERIC(20,4), -- 财务费用 + c_fr_sale_sg NUMERIC(20,4), -- 销售商品、提供劳务收到的现金 + recp_tax_rends NUMERIC(20,4), -- 收到的税费返还 + n_depos_incr_fi NUMERIC(20,4), -- 客户存款和同业存放款项净增加额 + n_incr_loans_cb NUMERIC(20,4), -- 向中央银行借款净增加额 + n_incr_borr_oth_fi NUMERIC(20,4), -- 向其他金融机构拆入资金净增加额 + prem_fr_orig_contr NUMERIC(20,4), -- 收到原保险合同保费取得的现金 + n_incr_insured_dep NUMERIC(20,4), -- 保户储金净增加额 + n_reinsur_prem NUMERIC(20,4), -- 收到再保业务现金净额 + n_incr_disp_tfa NUMERIC(20,4), -- 处置交易性金融资产净增加额 + ifc_cash_incr NUMERIC(20,4), -- 收取利息、手续费及佣金的现金 + n_incr_disp_faas NUMERIC(20,4), -- 处置可供出售金融资产净增加额 + n_incr_loans_oth_bank NUMERIC(20,4), -- 拆入资金净增加额 + dcr_cash_paym NUMERIC(20,4), -- 回购业务资金净增加额 + oth_cash_recp_ral_oper_act NUMERIC(20,4), -- 收到其他与经营活动有关的现金 + st_cash_out_act NUMERIC(20,4), -- 经营活动现金流出小计 + n_cashflow_act NUMERIC(20,4), -- 经营活动产生的现金流量净额 + + -- 投资活动 + c_inf_fr_invest_act NUMERIC(20,4), -- 投资活动现金流入小计 + c_out_invest_act NUMERIC(20,4), -- 投资活动现金流出小计 + n_cashflow_inv_act NUMERIC(20,4), -- 投资活动产生的现金流量净额 + + -- 筹资活动 + c_inf_fr_fin_act NUMERIC(20,4), -- 筹资活动现金流入小计 + st_cash_out_fin_act NUMERIC(20,4), -- 筹资活动现金流出小计 + n_cash_flows_fin_act NUMERIC(20,4), -- 筹资活动产生的现金流量净额 + eff_fx_flu_cash NUMERIC(20,4), -- 汇率变动对现金的影响 + n_incr_cash_cash_equ NUMERIC(20,4), -- 现金及现金等价物净增加额 + c_cash_equ_beg_period NUMERIC(20,4), -- 期初现金及现金等价物余额 + c_cash_equ_end_period NUMERIC(20,4), -- 期末现金及现金等价物余额 + updated_at TIMESTAMP DEFAULT NOW(), + + PRIMARY KEY (ts_code, end_date, report_type) +); + +COMMENT ON TABLE cashflow IS '现金流量表 (Tushare: cashflow)'; + + +-- ============================================================ +-- 10. 财务指标表 (fina_indicator) - 核心基本面指标 +-- tushare接口: fina_indicator +-- ============================================================ +CREATE TABLE IF NOT EXISTS fina_indicator ( + ts_code VARCHAR(20) NOT NULL, + ann_date DATE, + end_date DATE NOT NULL, -- 报告期 + -- 每股指标 + eps NUMERIC(16,4), -- 每股收益 + dt_eps NUMERIC(16,4), -- 稀释每股收益 + total_revenue_ps NUMERIC(16,4), -- 每股营业总收入 + revenue_ps NUMERIC(16,4), -- 每股营业收入 + capital_rese_ps NUMERIC(16,4), -- 每股资本公积 + surplus_rese_ps NUMERIC(16,4), -- 每股盈余公积 + undist_profit_ps NUMERIC(16,4), -- 每股未分配利润 + extra_item NUMERIC(16,4), -- 非经常性损益 + profit_dedt NUMERIC(16,4), -- 扣除非经常性损益后的净利润(扣非净利润) + -- 成长能力 + gross_margin NUMERIC(16,4), -- 毛利 + current_ratio NUMERIC(16,4), -- 流动比率 + quick_ratio NUMERIC(16,4), -- 速动比率 + cash_ratio NUMERIC(16,4), -- 保守速动比率 + invturn_days NUMERIC(16,4), -- 存货周转天数 + arturn_days NUMERIC(16,4), -- 应收账款周转天数 + inv_turn NUMERIC(16,4), -- 存货周转率 + ar_turn NUMERIC(16,4), -- 应收账款周转率 + assets_turn NUMERIC(16,4), -- 总资产周转率 + -- 盈利能力 + roe NUMERIC(16,4), -- 净资产收益率 + roe_dt NUMERIC(16,4), -- 净资产收益率(扣除非经常损益) + roa NUMERIC(16,4), -- 总资产净利率 + roa2 NUMERIC(16,4), -- 总资产报酬率 + roic NUMERIC(16,4), -- 投入资本回报率 + -- 杠杆/偿债 + longdeb_to_debt NUMERIC(16,4), -- 长期借款与资产总计之比 + debt_to_assets NUMERIC(16,4), -- 资产负债率 + ncfm NUMERIC(16,4), -- 销售商品提供劳务收到的现金/营业收入 + -- 收益质量 + profit_to_op NUMERIC(16,4), -- 经营活动净收益/利润总额 + cfo_to_or NUMERIC(16,4), -- 经营活动现金流净额/营业收入 + ocf_to_or NUMERIC(16,4), -- 经营现金净流量/营业总收入 + -- 杜邦分析 + dupont_roe NUMERIC(16,4), -- 净资产收益率(杜邦) + dupont_assets_turn NUMERIC(16,4), -- 资产周转率(杜邦) + dupont_equity_multi NUMERIC(16,4), -- 权益乘数(杜邦) + dupont_net_profit_margin NUMERIC(16,4), -- 销售净利率(杜邦) + -- 分红 + div_rec NUMERIC(16,4), -- 每股股利(税前) + -- 其他 + ocfps NUMERIC(16,4), -- 每股经营活动产生的现金流量净额 + cfps NUMERIC(16,4), -- 每股现金流量净额 + ebit_ps NUMERIC(16,4), -- 每股息税前利润 + fcff_ps NUMERIC(16,4), -- 每股企业自由现金流量 + fcfe_ps NUMERIC(16,4), -- 每股股东自由现金流量 + updated_at TIMESTAMP DEFAULT NOW(), + + PRIMARY KEY (ts_code, end_date) +); + +COMMENT ON TABLE fina_indicator IS '财务指标表 (Tushare: fina_indicator)'; + + +-- ============================================================ +-- 11. 交易日历表 (trade_cal) +-- tushare接口: trade_cal +-- ============================================================ +CREATE TABLE IF NOT EXISTS trade_cal ( + exchange VARCHAR(10) NOT NULL, -- 交易所 SSE上交所 SZSE深交所 + cal_date DATE NOT NULL, -- 日历日期 + is_open INTEGER NOT NULL, -- 是否交易 0休市 1交易 + pretrade_date DATE, -- 上一个交易日 + updated_at TIMESTAMP DEFAULT NOW(), + + PRIMARY KEY (exchange, cal_date) +); + +COMMENT ON TABLE trade_cal IS '交易日历表 (Tushare: trade_cal)'; + + +-- ============================================================ +-- 12. 指数日线行情表 (index_daily) +-- tushare接口: index_daily +-- 用于记录主要指数(沪深300、中证500等)的日线行情 +-- ============================================================ +CREATE TABLE IF NOT EXISTS index_daily ( + ts_code VARCHAR(20) NOT NULL, -- 指数代码 (如 000300.SH) + trade_date DATE NOT NULL, + close NUMERIC(12,4), + open NUMERIC(12,4), + high NUMERIC(12,4), + low NUMERIC(12,4), + pre_close NUMERIC(12,4), + change NUMERIC(12,4), + pct_chg NUMERIC(10,4), + vol NUMERIC(20,2), + amount NUMERIC(20,4), + updated_at TIMESTAMP DEFAULT NOW(), + + PRIMARY KEY (ts_code, trade_date) +); + +COMMENT ON TABLE index_daily IS '指数日线行情表 (Tushare: index_daily)'; + + +-- ============================================================ +-- 常用查询视图 +-- ============================================================ + +-- 视图1: 日线行情 + 估值指标联合查询 +CREATE OR REPLACE VIEW v_daily_with_valuation AS +SELECT + d.ts_code, + d.trade_date, + b.name, + b.industry, + b.area, + d.open, + d.high, + d.low, + d.close, + d.pre_close, + d.pct_chg, + d.vol, + d.amount, + d.turnover_rate, + db.pe, + db.pe_ttm, + db.pb, + db.ps, + db.ps_ttm, + db.total_mv, + db.circ_mv, + db.dv_ratio, + db.dv_ttm +FROM daily d +LEFT JOIN stock_basic b ON d.ts_code = b.ts_code +LEFT JOIN daily_basic db ON d.ts_code = db.ts_code AND d.trade_date = db.trade_date; + +COMMENT ON VIEW v_daily_with_valuation IS '日线行情+估值联合查询视图'; + + +-- 视图2: 最新财务指标 + 股票基本信息 +CREATE OR REPLACE VIEW v_latest_financials AS +SELECT DISTINCT ON (f.ts_code) + f.ts_code, + b.name, + b.industry, + b.area, + b.list_date, + f.end_date AS report_date, + f.eps, + f.roe, + f.roa, + f.roic, + f.current_ratio, + f.quick_ratio, + f.debt_to_assets, + f.gross_margin, + f.assets_turn, + f.cfps, + f.ocfps, + f.div_rec +FROM fina_indicator f +JOIN stock_basic b ON f.ts_code = b.ts_code +ORDER BY f.ts_code, f.end_date DESC; + +COMMENT ON VIEW v_latest_financials IS '最新财务指标+股票信息'; + +-- ============================================================ +-- 完成 +-- ============================================================ \ No newline at end of file diff --git a/quantitative_data/数据批量导入.ipynb b/quantitative_data/数据批量导入.ipynb new file mode 100644 index 0000000..5476d3f --- /dev/null +++ b/quantitative_data/数据批量导入.ipynb @@ -0,0 +1,647 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 量化投资数据批量导入\n", + "\n", + "## 目标\n", + "将 Tushare 的日线行情数据及公司基本面数据批量导入 Docker PostgreSQL (192.168.27.15:12345)\n", + "\n", + "## 数据库结构概览\n", + "\n", + "| 表名 | 说明 | Tushare 接口 |\n", + "|------|------|-------------|\n", + "| stock_basic | 股票基本信息 | stock_basic |\n", + "| trade_cal | 交易日历 | trade_cal |\n", + "| daily | 日线行情 | daily |\n", + "| daily_basic | 每日指标(估值/基本面) | daily_basic |\n", + "| adj_factor | 复权因子 | adj_factor |\n", + "| income | 利润表 | income |\n", + "| balancesheet | 资产负债表 | balancesheet |\n", + "| cashflow | 现金流量表 | cashflow |\n", + "| fina_indicator | 财务指标 | fina_indicator |\n", + "| moneyflow | 个股资金流向 | moneyflow |\n", + "| index_daily | 指数日线行情 | index_daily |\n", + "\n", + "## 使用步骤\n", + "1. 修改 `config.py` 中的数据库密码和 Tushare Token\n", + "2. 逐 Cell 运行本 Notebook" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## Step 0: 检查环境 & 连接测试" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "import os\n", + "os.chdir(r\"t:\\jupyter\\notebook\\quantitative_data\")\n", + "print(f\"工作目录: {os.getcwd()}\")\n", + "print(f\"Python 版本: {sys.version}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 检查依赖包\n", + "!pip list | findstr -i \"tushare pandas psycopg2 sqlalchemy\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 如果需要安装依赖，取消注释下面这行\n", + "# !pip install -r requirements.txt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 导入核心模块\n", + "from importer import (\n", + " get_pg_connection,\n", + " get_ts_pro,\n", + " init_database,\n", + " import_stock_basic,\n", + " import_trade_cal,\n", + " import_daily_batch,\n", + " import_daily_by_year,\n", + " import_daily_basic,\n", + " import_daily_basic_by_date,\n", + " import_adj_factor,\n", + " import_adj_factor_batch,\n", + " import_financial_statements,\n", + " import_index_daily,\n", + " get_all_stock_codes,\n", + " get_stock_codes_from_db,\n", + " full_import,\n", + " batch_insert,\n", + " logger,\n", + ")\n", + "from config import DB_CONFIG, TUSHARE_TOKEN, START_DATE, END_DATE\n", + "\n", + "print(\"模块导入成功!\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 测试数据库连接\n", + "try:\n", + " conn = get_pg_connection()\n", + " cursor = conn.cursor()\n", + " cursor.execute(\"SELECT version()\")\n", + " version = cursor.fetchone()[0]\n", + " print(f\"✓ PostgreSQL 连接成功!\")\n", + " print(f\" 服务器版本: {version}\")\n", + " print(f\" 连接信息: {DB_CONFIG['host']}:{DB_CONFIG['port']}/{DB_CONFIG['database']}\")\n", + " cursor.close()\n", + " conn.close()\n", + "except Exception as e:\n", + " print(f\"✗ 连接失败: {e}\")\n", + " print(\"请检查:\")\n", + " print(\" 1. Docker 容器是否已启动: docker ps | findstr postgres\")\n", + " print(\" 2. config.py 中的连接参数是否正确\")\n", + " print(\" 3. 防火墙是否开放 12345 端口\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 测试 Tushare API 连接\n", + "try:\n", + " pro = get_ts_pro()\n", + " # 简单测试: 获取一只股票信息\n", + " df = pro.stock_basic(ts_code=\"000001.SZ\", fields=\"ts_code,name,industry\")\n", + " print(f\"✓ Tushare API 连接成功!\")\n", + " print(f\" 测试查询: {df.iloc[0].to_dict()}\")\n", + "except Exception as e:\n", + " print(f\"✗ Tushare API 连接失败: {e}\")\n", + " print(\"请检查 config.py 中的 TUSHARE_TOKEN 是否正确\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## Step 1: 初始化数据库 Schema" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 执行 DDL，创建所有表结构\n", + "init_database()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## Step 2: 导入股票基本信息" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 导入全量 A 股股票基本信息（含上市和退市）\n", + "import_stock_basic()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 验证：查看导入结果\n", + "conn = get_pg_connection()\n", + "cursor = conn.cursor()\n", + "cursor.execute(\"SELECT COUNT(*) FROM stock_basic\")\n", + "print(f\"stock_basic 总记录数: {cursor.fetchone()[0]}\")\n", + "cursor.execute(\"SELECT list_status, COUNT(*) FROM stock_basic GROUP BY list_status\")\n", + "for row in cursor.fetchall():\n", + " print(f\" 状态 '{row[0]}': {row[1]} 只\")\n", + "cursor.close()\n", + "conn.close()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## Step 3: 导入交易日历" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 导入交易日历 (默认从 config.py 的 START_DATE ~ END_DATE)\n", + "import_trade_cal()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 或者指定日期范围\n", + "# import_trade_cal(start_date=\"2020-01-01\", end_date=\"2025-12-31\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 验证\n", + "conn = get_pg_connection()\n", + "cursor = conn.cursor()\n", + "cursor.execute(\"\"\"\n", + " SELECT exchange, MIN(cal_date) AS first_date, MAX(cal_date) AS last_date, COUNT(*) AS total\n", + " FROM trade_cal\n", + " GROUP BY exchange\n", + "\"\"\")\n", + "for row in cursor.fetchall():\n", + " print(f\" {row[0]}: {row[1]} ~ {row[2]}, 共 {row[3]} 条\")\n", + "cursor.close()\n", + "conn.close()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## Step 4: 导入日线行情 (核心表，最耗时)\n", + "\n", + "### 4.1 获取股票列表" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 获取所有需要导入的股票代码\n", + "stock_list = get_stock_codes_from_db()\n", + "print(f\"共 {len(stock_list)} 只股票需要导入日线行情\")\n", + "print(f\"前 10 只: {stock_list[:10]}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 4.2 按年批量导入 (推荐 - 断点续传友好)\n", + "\n", + "数据量估算: 约5000只股票 × 250交易日/年 × 16年 ≈ 2000万条记录" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 按年份逐批导入日线行情\n", + "# 如果中断，可以修改年份范围从断点继续\n", + "import_daily_by_year(\n", + " stock_list,\n", + " start_year=2010,\n", + " end_year=2025,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 4.3 单只股票导入 (补充/重试)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 导入单只股票日线行情 (用于补充导入或测试)\n", + "conn = get_pg_connection()\n", + "from importer import import_daily_for_stock\n", + "\n", + "n = import_daily_for_stock(\"000001.SZ\", \"2020-01-01\", \"2020-12-31\", conn)\n", + "print(f\"导入 000001.SZ 2020年数据: {n} 条\")\n", + "conn.close()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 验证日线数据\n", + "conn = get_pg_connection()\n", + "cursor = conn.cursor()\n", + "cursor.execute(\"\"\"\n", + " SELECT \n", + " COUNT(*) AS total_records,\n", + " COUNT(DISTINCT ts_code) AS stock_count,\n", + " MIN(trade_date) AS first_date,\n", + " MAX(trade_date) AS last_date\n", + " FROM daily\n", + "\"\"\")\n", + "for row in cursor.fetchall():\n", + " print(f\" 总记录数: {row[0]:,}\")\n", + " print(f\" 股票数量: {row[1]}\")\n", + " print(f\" 日期范围: {row[2]} ~ {row[3]}\")\n", + "cursor.close()\n", + "conn.close()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## Step 5: 导入每日指标 (估值数据)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 按交易日导入每日指标 (PE/PB/PS/总市值/流通市值等)\n", + "import_daily_basic_by_date(\n", + " start_date=\"2010-01-01\",\n", + " end_date=\"2025-12-31\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 验证\n", + "conn = get_pg_connection()\n", + "cursor = conn.cursor()\n", + "cursor.execute(\"SELECT COUNT(*), MIN(trade_date), MAX(trade_date) FROM daily_basic\")\n", + "row = cursor.fetchone()\n", + "print(f\" daily_basic: {row[0]:,} 条, {row[1]} ~ {row[2]}\")\n", + "cursor.close()\n", + "conn.close()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## Step 6: 导入复权因子" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 导入复权因子 (用于前复权/后复权价格计算)\n", + "import_adj_factor_batch(\n", + " stock_list,\n", + " start_date=\"2010-01-01\",\n", + " end_date=\"2025-12-31\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## Step 7: 导入财务数据 (三大报表 + 财务指标)\n", + "\n", + "⚠ 此步骤耗时较长，约需数小时（取决于股票数量）" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 导入利润表、资产负债表、现金流量表、财务指标\n", + "import_financial_statements(\n", + " stock_list,\n", + " start_date=\"2010-01-01\",\n", + " end_date=\"2025-12-31\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 验证财务数据\n", + "conn = get_pg_connection()\n", + "cursor = conn.cursor()\n", + "for table in [\"income\", \"balancesheet\", \"cashflow\", \"fina_indicator\"]:\n", + " cursor.execute(f\"SELECT COUNT(*) FROM {table}\")\n", + " count = cursor.fetchone()[0]\n", + " print(f\" {table}: {count:,} 条\")\n", + "cursor.close()\n", + "conn.close()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## Step 8: 导入指数日线行情" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 导入主要指数日线行情\n", + "import_index_daily(\n", + " index_codes=[\n", + " \"000001.SH\", # 上证指数\n", + " \"399001.SZ\", # 深证成指\n", + " \"000300.SH\", # 沪深300\n", + " \"000905.SH\", # 中证500\n", + " \"399006.SZ\", # 创业板指\n", + " \"000688.SH\", # 科创50\n", + " \"000016.SH\", # 上证50\n", + " \"399005.SZ\", # 中小100\n", + " \"000852.SH\", # 中证1000\n", + " ],\n", + " start_date=\"2010-01-01\",\n", + " end_date=\"2025-12-31\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## 一键全量导入 (可选)\n", + "\n", + "如果不想逐步执行，可以运行下面这个 Cell 一键完成所有导入" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 一键全量导入 (需数小时~数十小时，请谨慎)\n", + "# full_import(\n", + "# start_date=\"2010-01-01\",\n", + "# end_date=\"2025-12-31\",\n", + "# import_financials=True, # 设为 False 跳过财务数据加快速度\n", + "# )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## 数据验证与查询示例" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import psycopg2\n", + "\n", + "conn = get_pg_connection()\n", + "\n", + "# 各表统计\n", + "tables = [\"stock_basic\", \"daily\", \"daily_basic\", \"adj_factor\",\n", + " \"income\", \"balancesheet\", \"cashflow\", \"fina_indicator\",\n", + " \"trade_cal\", \"index_daily\"]\n", + "\n", + "print(f\"{'表名':<20} {'记录数':>12} {'最早日期':>12} {'最晚日期':>12}\")\n", + "print(\"-\" * 60)\n", + "for table in tables:\n", + " try:\n", + " count_sql = f\"SELECT COUNT(*) FROM {table}\"\n", + " count = pd.read_sql(count_sql, conn).iloc[0, 0]\n", + " \n", + " # 尝试获取日期范围\n", + " date_col = None\n", + " if table == \"daily\":\n", + " date_col = \"trade_date\"\n", + " elif table == \"daily_basic\":\n", + " date_col = \"trade_date\"\n", + " elif table in [\"income\", \"balancesheet\", \"cashflow\"]:\n", + " date_col = \"end_date\"\n", + " elif table == \"fina_indicator\":\n", + " date_col = \"end_date\"\n", + " elif table == \"trade_cal\":\n", + " date_col = \"cal_date\"\n", + " elif table == \"index_daily\":\n", + " date_col = \"trade_date\"\n", + " elif table == \"adj_factor\":\n", + " date_col = \"trade_date\"\n", + " \n", + " if date_col:\n", + " date_sql = f\"SELECT MIN({date_col}), MAX({date_col}) FROM {table}\"\n", + " min_d, max_d = pd.read_sql(date_sql, conn).iloc[0]\n", + " print(f\"{table:<20} {count:>12,} {str(min_d)[:10]:>12} {str(max_d)[:10]:>12}\")\n", + " else:\n", + " print(f\"{table:<20} {count:>12,}\")\n", + " except Exception as e:\n", + " print(f\"{table:<20} {'错误':>12}: {str(e)[:40]}\")\n", + "\n", + "conn.close()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 示例查询 1: 查询某股票最近10个交易日数据\n", + "query1 = \"\"\"\n", + "SELECT trade_date, open, high, low, close, vol, amount, pct_chg\n", + "FROM daily\n", + "WHERE ts_code = '000001.SZ'\n", + "ORDER BY trade_date DESC\n", + "LIMIT 10\n", + "\"\"\"\n", + "conn = get_pg_connection()\n", + "df1 = pd.read_sql(query1, conn)\n", + "print(\"平安银行(000001.SZ) 最近10个交易日:\")\n", + "display(df1)\n", + "conn.close()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 示例查询 2: 日线行情 + 估值指标联合查询 (使用视图)\n", + "query2 = \"\"\"\n", + "SELECT *\n", + "FROM v_daily_with_valuation\n", + "WHERE ts_code = '000001.SZ'\n", + " AND trade_date >= '2024-01-01'\n", + "ORDER BY trade_date DESC\n", + "LIMIT 10\n", + "\"\"\"\n", + "conn = get_pg_connection()\n", + "df2 = pd.read_sql(query2, conn)\n", + "print(\"平安银行 - 日线+估值:\")\n", + "display(df2[['trade_date', 'close', 'pct_chg', 'pe', 'pe_ttm', 'pb', 'total_mv']])\n", + "conn.close()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 示例查询 3: 最新财务指标 Top 20 (按 ROE 排序)\n", + "query3 = \"\"\"\n", + "SELECT *\n", + "FROM v_latest_financials\n", + "WHERE roe IS NOT NULL\n", + " AND roe > 0\n", + "ORDER BY roe DESC\n", + "LIMIT 20\n", + "\"\"\"\n", + "conn = get_pg_connection()\n", + "df3 = pd.read_sql(query3, conn)\n", + "print(\"ROE Top 20:\")\n", + "display(df3[['ts_code', 'name', 'industry', 'roe', 'roa', 'eps', 'debt_to_assets']])\n", + "conn.close()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}