feat: 修复daily历史覆盖度缺口+增加覆盖度校验
- 新增 repair_daily_backfill.py:按daily_basic基准检测并补拉覆盖不足交易日 (解决2012-2013沪市+创业板整年缺失问题,断点续传日期级检测无法发现) - importer.py: 新增 check_daily_coverage() 覆盖度审计函数 - incremental_import.py: 增量导入后自动跑覆盖度校验并告警 - 数据批量导入.ipynb: 增加数据完整性检验单元格
This commit is contained in:
@@ -1566,6 +1566,70 @@ def get_missing_daily_dates(
|
||||
conn.close()
|
||||
|
||||
|
||||
def check_daily_coverage(
|
||||
start_date: Optional[str] = None,
|
||||
end_date: Optional[str] = None,
|
||||
conn=None,
|
||||
tolerance: float = 0.05,
|
||||
) -> List[Dict]:
|
||||
"""
|
||||
检查 daily 表每日覆盖度(以 daily_basic 为基准,单条 SQL 聚合)。
|
||||
|
||||
背景:resume_daily_by_date 只做「日期级」缺失检测(NOT EXISTS trade_date),
|
||||
若某交易日只有部分股票入库(如 2012-2013 沪市+创业板缺失),会误判为"已完整",
|
||||
导致覆盖度缺口永不补拉。本函数做「覆盖度级」校验:
|
||||
|
||||
daily 当日股票数 < daily_basic 当日股票数 × (1 - tolerance) → 判定覆盖不足
|
||||
|
||||
返回: [{"trade_date", "daily_cnt", "daily_basic_cnt", "coverage_pct"}, ...]
|
||||
"""
|
||||
if start_date is None:
|
||||
start_date = START_DATE
|
||||
if end_date is None:
|
||||
end_date = END_DATE
|
||||
|
||||
own_conn = conn is None
|
||||
if own_conn:
|
||||
conn = get_pg_connection()
|
||||
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT db.trade_date,
|
||||
COALESCE(d.cnt, 0) AS daily_cnt,
|
||||
db.cnt AS db_cnt,
|
||||
ROUND(COALESCE(d.cnt, 0)::numeric / db.cnt * 100, 1) AS coverage_pct
|
||||
FROM (SELECT trade_date, COUNT(DISTINCT ts_code) AS cnt
|
||||
FROM daily_basic
|
||||
WHERE trade_date BETWEEN %s AND %s
|
||||
GROUP BY trade_date) db
|
||||
LEFT JOIN (SELECT trade_date, COUNT(DISTINCT ts_code) AS cnt
|
||||
FROM daily
|
||||
WHERE trade_date BETWEEN %s AND %s
|
||||
GROUP BY trade_date) d
|
||||
ON d.trade_date = db.trade_date
|
||||
WHERE COALESCE(d.cnt, 0) < db.cnt * (1 - %s)
|
||||
ORDER BY db.trade_date
|
||||
""",
|
||||
(start_date, end_date, start_date, end_date, tolerance),
|
||||
)
|
||||
rows = [
|
||||
{
|
||||
"trade_date": r[0].strftime("%Y-%m-%d") if hasattr(r[0], "strftime") else str(r[0])[:10],
|
||||
"daily_cnt": r[1],
|
||||
"daily_basic_cnt": r[2],
|
||||
"coverage_pct": float(r[3]),
|
||||
}
|
||||
for r in cursor.fetchall()
|
||||
]
|
||||
cursor.close()
|
||||
return rows
|
||||
finally:
|
||||
if own_conn:
|
||||
conn.close()
|
||||
|
||||
|
||||
def resume_daily_by_date(
|
||||
start_date: Optional[str] = None,
|
||||
end_date: Optional[str] = None,
|
||||
|
||||
Reference in New Issue
Block a user