Compare commits
5
Commits
b868e63e40
...
f45d130a58
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f45d130a58 | ||
|
|
2629b696c2 | ||
|
|
4dd658c0a7 | ||
|
|
7612bae51e | ||
|
|
94c8305d9c |
@@ -4,3 +4,4 @@ scipy>=1.10.0
|
||||
matplotlib>=3.7.0
|
||||
tushare>=1.3.0
|
||||
psycopg2-binary>=2.9.0
|
||||
python-dotenv>=1.0.0
|
||||
|
||||
@@ -4,8 +4,8 @@
|
||||
# ========================================
|
||||
|
||||
# PostgreSQL 数据库连接
|
||||
QUANT_DB_HOST=192.168.27.15
|
||||
QUANT_DB_PORT=12345
|
||||
QUANT_DB_HOST=192.168.27.11
|
||||
QUANT_DB_PORT=5438
|
||||
QUANT_DB_NAME=quant_db
|
||||
QUANT_DB_USER=postgres
|
||||
QUANT_DB_PASSWORD=your_password_here
|
||||
|
||||
@@ -11,16 +11,45 @@
|
||||
也可创建 .env 文件(参考 .env.example)
|
||||
"""
|
||||
import os
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
# 加载 .env 文件中的环境变量
|
||||
_LOADED = False
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
# 使用 resolve() 获取绝对路径,避免 os.chdir() 改变工作目录后 __file__ 变为相对路径
|
||||
env_path = Path(__file__).resolve().parent / ".env"
|
||||
# 如果 resolve 后的路径不存在,尝试从当前文件所在目录查找
|
||||
if not env_path.exists():
|
||||
# 备用方案:也检查 Notebook 的当前工作目录
|
||||
cwd_path = Path.cwd() / ".env"
|
||||
if cwd_path.exists():
|
||||
env_path = cwd_path
|
||||
|
||||
if env_path.exists():
|
||||
load_dotenv(dotenv_path=env_path, override=True)
|
||||
print(f"✓ 已加载环境变量文件: {env_path}")
|
||||
_LOADED = True
|
||||
else:
|
||||
print(f"⚠ 未找到 .env 文件。检查路径: {Path(__file__).resolve().parent / '.env'}, 当前目录: {Path.cwd() / '.env'}")
|
||||
except ImportError:
|
||||
print("⚠ python-dotenv 未安装,无法自动加载 .env 文件。请手动设置环境变量或执行: pip install python-dotenv")
|
||||
|
||||
# PostgreSQL Docker 连接配置
|
||||
_PASSWORD = os.environ.get("QUANT_DB_PASSWORD", "")
|
||||
|
||||
DB_CONFIG = {
|
||||
"host": os.environ.get("QUANT_DB_HOST", "192.168.27.15"),
|
||||
"port": int(os.environ.get("QUANT_DB_PORT", "12345")),
|
||||
"host": os.environ.get("QUANT_DB_HOST", "192.168.27.11"),
|
||||
"port": int(os.environ.get("QUANT_DB_PORT", "5438")),
|
||||
"database": os.environ.get("QUANT_DB_NAME", "quant_db"),
|
||||
"user": os.environ.get("QUANT_DB_USER", "postgres"),
|
||||
"password": os.environ.get("QUANT_DB_PASSWORD", ""),
|
||||
"password": _PASSWORD,
|
||||
}
|
||||
|
||||
# 对密码进行 URL 编码(用于 SQLAlchemy 连接串),避免特殊字符导致解析错误
|
||||
PASSWORD_ENCODED = quote_plus(_PASSWORD) if _PASSWORD else ""
|
||||
|
||||
# Tushare API Token
|
||||
TUSHARE_TOKEN = os.environ.get("TUSHARE_TOKEN", "")
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""
|
||||
量化数据导入核心模块
|
||||
连接 Docker PostgreSQL (192.168.27.15:12345)
|
||||
连接 Docker PostgreSQL (192.168.27.11:5438)
|
||||
从 Tushare 拉取数据并批量导入
|
||||
"""
|
||||
import time
|
||||
@@ -15,7 +15,7 @@ from psycopg2 import sql
|
||||
from psycopg2.extras import execute_values
|
||||
from sqlalchemy import create_engine
|
||||
|
||||
from config import DB_CONFIG, TUSHARE_TOKEN, BATCH_SIZE, START_DATE, END_DATE
|
||||
from config import DB_CONFIG, TUSHARE_TOKEN, BATCH_SIZE, START_DATE, END_DATE, PASSWORD_ENCODED
|
||||
|
||||
# ============================================================
|
||||
# 日志配置
|
||||
@@ -55,7 +55,7 @@ def get_sqlalchemy_engine():
|
||||
global _sqlalchemy_engine
|
||||
if _sqlalchemy_engine is None:
|
||||
db_url = (
|
||||
f"postgresql://{DB_CONFIG['user']}:{DB_CONFIG['password']}"
|
||||
f"postgresql://{DB_CONFIG['user']}:{PASSWORD_ENCODED}"
|
||||
f"@{DB_CONFIG['host']}:{DB_CONFIG['port']}/{DB_CONFIG['database']}"
|
||||
)
|
||||
_sqlalchemy_engine = create_engine(db_url, pool_size=5, max_overflow=10)
|
||||
@@ -185,6 +185,9 @@ def import_stock_basic():
|
||||
导入股票基本信息 (stock_basic)
|
||||
Tushare: stock_basic
|
||||
"""
|
||||
# 确保数据库表结构已初始化
|
||||
init_database()
|
||||
|
||||
logger.info("=" * 60)
|
||||
logger.info("[1/7] 导入股票基本信息 (stock_basic) ...")
|
||||
|
||||
@@ -820,7 +823,9 @@ def init_database():
|
||||
# 执行 DDL
|
||||
conn = get_pg_connection()
|
||||
try:
|
||||
with open("schema.sql", "r", encoding="utf-8") as f:
|
||||
import os as _os
|
||||
schema_path = _os.path.join(_os.path.dirname(_os.path.abspath(__file__)), "schema.sql")
|
||||
with open(schema_path, "r", encoding="utf-8") as f:
|
||||
ddl_sql = f.read()
|
||||
|
||||
# 按分号分割,逐条执行 (忽略被注释掉的分区表DDL)
|
||||
|
||||
@@ -2,3 +2,4 @@ tushare>=1.4.0
|
||||
pandas>=1.5.0
|
||||
psycopg2-binary>=2.9.0
|
||||
sqlalchemy>=2.0.0
|
||||
python-dotenv>=1.0.0
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
-- ============================================================
|
||||
-- 量化投资数据库 Schema
|
||||
-- PostgreSQL (Docker: 192.168.27.15:12345)
|
||||
-- PostgreSQL (Docker: 192.168.27.11:5438)
|
||||
-- 数据来源:Tushare
|
||||
-- ============================================================
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
"# 量化投资数据批量导入\n",
|
||||
"\n",
|
||||
"## 目标\n",
|
||||
"将 Tushare 的日线行情数据及公司基本面数据批量导入 Docker PostgreSQL (192.168.27.15:12345)\n",
|
||||
"将 Tushare 的日线行情数据及公司基本面数据批量导入 Docker PostgreSQL (192.168.27.11:5438)\n",
|
||||
"\n",
|
||||
"## 数据库结构概览\n",
|
||||
"\n",
|
||||
@@ -40,9 +40,18 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"工作目录: t:\\jupyter\\notebook\\quantitative_data\n",
|
||||
"Python 版本: 3.10.2 (heads/master:d9999f5, Dec 16 2022, 16:20:32) [MSC v.1929 64 bit (AMD64)]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import sys\n",
|
||||
"import os\n",
|
||||
@@ -53,12 +62,33 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"pandas 1.5.0\n",
|
||||
"python-dotenv 1.2.2\n",
|
||||
"SQLAlchemy 1.3.24\n",
|
||||
"tushare 1.2.89\n",
|
||||
"vnpy-tushare 1.2.85.1\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"[notice] A new release of pip available: 22.2.2 -> 26.2\n",
|
||||
"[notice] To update, run: python.exe -m pip install --upgrade pip\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# 检查依赖包\n",
|
||||
"!pip list | findstr -i \"tushare pandas psycopg2 sqlalchemy\""
|
||||
"!pip list | findstr -i \"tushare pandas psycopg2-binary sqlalchemy python-dotenv\""
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -68,14 +98,22 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# 如果需要安装依赖,取消注释下面这行\n",
|
||||
"# !pip install -r requirements.txt"
|
||||
"#!pip install -r requirements.txt"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"模块导入成功!\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# 导入核心模块\n",
|
||||
"from importer import (\n",
|
||||
@@ -105,9 +143,22 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"✗ 连接失败: connection to server at \"192.168.27.11\", port 5438 failed: fe_sendauth: no password supplied\n",
|
||||
"\n",
|
||||
"请检查:\n",
|
||||
" 1. Docker 容器是否已启动: docker ps | findstr postgres\n",
|
||||
" 2. 环境变量 (.env) 中的连接参数是否正确\n",
|
||||
" 3. 防火墙是否开放 5438 端口\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# 测试数据库连接\n",
|
||||
"try:\n",
|
||||
@@ -125,7 +176,7 @@
|
||||
" print(\"请检查:\")\n",
|
||||
" print(\" 1. Docker 容器是否已启动: docker ps | findstr postgres\")\n",
|
||||
" print(\" 2. 环境变量 (.env) 中的连接参数是否正确\")\n",
|
||||
" print(\" 3. 防火墙是否开放 12345 端口\")"
|
||||
" print(\" 3. 防火墙是否开放 5438 端口\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -625,7 +676,7 @@
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
@@ -639,7 +690,7 @@
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.6"
|
||||
"version": "3.10.2"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
|
||||
Reference in New Issue
Block a user