feat: 修复daily历史覆盖度缺口+增加覆盖度校验

- 新增 repair_daily_backfill.py:按daily_basic基准检测并补拉覆盖不足交易日
  (解决2012-2013沪市+创业板整年缺失问题,断点续传日期级检测无法发现)
- importer.py: 新增 check_daily_coverage() 覆盖度审计函数
- incremental_import.py: 增量导入后自动跑覆盖度校验并告警
- 数据批量导入.ipynb: 增加数据完整性检验单元格
This commit is contained in:
xiaoqiang
2026-08-26 10:47:32 +00:00
parent e07e9426c7
commit 84912ef99c
4 changed files with 326 additions and 0 deletions
+64
View File
@@ -1566,6 +1566,70 @@ def get_missing_daily_dates(
conn.close()
def check_daily_coverage(
start_date: Optional[str] = None,
end_date: Optional[str] = None,
conn=None,
tolerance: float = 0.05,
) -> List[Dict]:
"""
检查 daily 表每日覆盖度(以 daily_basic 为基准,单条 SQL 聚合)。
背景:resume_daily_by_date 只做「日期级」缺失检测(NOT EXISTS trade_date),
若某交易日只有部分股票入库(如 2012-2013 沪市+创业板缺失),会误判为"已完整",
导致覆盖度缺口永不补拉。本函数做「覆盖度级」校验:
daily 当日股票数 < daily_basic 当日股票数 × (1 - tolerance) → 判定覆盖不足
返回: [{"trade_date", "daily_cnt", "daily_basic_cnt", "coverage_pct"}, ...]
"""
if start_date is None:
start_date = START_DATE
if end_date is None:
end_date = END_DATE
own_conn = conn is None
if own_conn:
conn = get_pg_connection()
try:
cursor = conn.cursor()
cursor.execute(
"""
SELECT db.trade_date,
COALESCE(d.cnt, 0) AS daily_cnt,
db.cnt AS db_cnt,
ROUND(COALESCE(d.cnt, 0)::numeric / db.cnt * 100, 1) AS coverage_pct
FROM (SELECT trade_date, COUNT(DISTINCT ts_code) AS cnt
FROM daily_basic
WHERE trade_date BETWEEN %s AND %s
GROUP BY trade_date) db
LEFT JOIN (SELECT trade_date, COUNT(DISTINCT ts_code) AS cnt
FROM daily
WHERE trade_date BETWEEN %s AND %s
GROUP BY trade_date) d
ON d.trade_date = db.trade_date
WHERE COALESCE(d.cnt, 0) < db.cnt * (1 - %s)
ORDER BY db.trade_date
""",
(start_date, end_date, start_date, end_date, tolerance),
)
rows = [
{
"trade_date": r[0].strftime("%Y-%m-%d") if hasattr(r[0], "strftime") else str(r[0])[:10],
"daily_cnt": r[1],
"daily_basic_cnt": r[2],
"coverage_pct": float(r[3]),
}
for r in cursor.fetchall()
]
cursor.close()
return rows
finally:
if own_conn:
conn.close()
def resume_daily_by_date(
start_date: Optional[str] = None,
end_date: Optional[str] = None,