feat: 修复daily历史覆盖度缺口+增加覆盖度校验
- 新增 repair_daily_backfill.py:按daily_basic基准检测并补拉覆盖不足交易日 (解决2012-2013沪市+创业板整年缺失问题,断点续传日期级检测无法发现) - importer.py: 新增 check_daily_coverage() 覆盖度审计函数 - incremental_import.py: 增量导入后自动跑覆盖度校验并告警 - 数据批量导入.ipynb: 增加数据完整性检验单元格
This commit is contained in:
@@ -35,6 +35,7 @@ from importer import (
|
||||
import_adj_factor_batch,
|
||||
import_index_daily,
|
||||
import_financial_statements,
|
||||
check_daily_coverage,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("incremental")
|
||||
@@ -140,6 +141,27 @@ def run_daily(end_date, dry_run, limit):
|
||||
except Exception as e:
|
||||
logger.error(f" {table} 增量导入失败: {e}")
|
||||
|
||||
# 5. 覆盖度校验(以 daily_basic 为基准,检查 daily 是否缺部分股票)
|
||||
# 防止"日期存在但覆盖不全"的缺口(如 2012-2013 沪市+创业板缺失)被增量逻辑跳过
|
||||
try:
|
||||
coverage_start = "2010-01-01" # 全历史检查(单条 SQL 聚合,开销小)
|
||||
logger.info(f"覆盖度校验: daily vs daily_basic ({coverage_start} ~ {end_date})")
|
||||
if dry_run:
|
||||
logger.info("[dry-run] 跳过覆盖度校验")
|
||||
else:
|
||||
partial = check_daily_coverage(coverage_start, end_date, conn=conn, tolerance=0.05)
|
||||
if partial:
|
||||
logger.warning(f"⚠ 发现 {len(partial)} 个交易日覆盖不足 (daily < daily_basic×95%):")
|
||||
for p in partial[:10]:
|
||||
logger.warning(f" {p['trade_date']}: daily={p['daily_cnt']} vs daily_basic={p['daily_basic_cnt']} ({p['coverage_pct']}%)")
|
||||
if len(partial) > 10:
|
||||
logger.warning(f" ... 其余 {len(partial)-10} 个交易日略")
|
||||
logger.warning(" 请运行 repair_daily_backfill.py 修复历史缺口,或检查近期导入是否被中断")
|
||||
else:
|
||||
logger.info(" ✓ 覆盖度正常,无缺失交易日")
|
||||
except Exception as e:
|
||||
logger.error(f" 覆盖度校验失败: {e}")
|
||||
|
||||
conn.close()
|
||||
logger.info("每日增量导入完成")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user