Compare commits

...
5 Commits
7 changed files with 114 additions and 27 deletions
+1
View File
@@ -4,3 +4,4 @@ scipy>=1.10.0
matplotlib>=3.7.0 matplotlib>=3.7.0
tushare>=1.3.0 tushare>=1.3.0
psycopg2-binary>=2.9.0 psycopg2-binary>=2.9.0
python-dotenv>=1.0.0
+2 -2
View File
@@ -4,8 +4,8 @@
# ======================================== # ========================================
# PostgreSQL 数据库连接 # PostgreSQL 数据库连接
QUANT_DB_HOST=192.168.27.15 QUANT_DB_HOST=192.168.27.11
QUANT_DB_PORT=12345 QUANT_DB_PORT=5438
QUANT_DB_NAME=quant_db QUANT_DB_NAME=quant_db
QUANT_DB_USER=postgres QUANT_DB_USER=postgres
QUANT_DB_PASSWORD=your_password_here QUANT_DB_PASSWORD=your_password_here
+32 -3
View File
@@ -11,16 +11,45 @@
也可创建 .env 文件(参考 .env.example 也可创建 .env 文件(参考 .env.example
""" """
import os import os
from pathlib import Path
from urllib.parse import quote_plus
# 加载 .env 文件中的环境变量
_LOADED = False
try:
from dotenv import load_dotenv
# 使用 resolve() 获取绝对路径,避免 os.chdir() 改变工作目录后 __file__ 变为相对路径
env_path = Path(__file__).resolve().parent / ".env"
# 如果 resolve 后的路径不存在,尝试从当前文件所在目录查找
if not env_path.exists():
# 备用方案:也检查 Notebook 的当前工作目录
cwd_path = Path.cwd() / ".env"
if cwd_path.exists():
env_path = cwd_path
if env_path.exists():
load_dotenv(dotenv_path=env_path, override=True)
print(f"✓ 已加载环境变量文件: {env_path}")
_LOADED = True
else:
print(f"⚠ 未找到 .env 文件。检查路径: {Path(__file__).resolve().parent / '.env'}, 当前目录: {Path.cwd() / '.env'}")
except ImportError:
print("⚠ python-dotenv 未安装,无法自动加载 .env 文件。请手动设置环境变量或执行: pip install python-dotenv")
# PostgreSQL Docker 连接配置 # PostgreSQL Docker 连接配置
_PASSWORD = os.environ.get("QUANT_DB_PASSWORD", "")
DB_CONFIG = { DB_CONFIG = {
"host": os.environ.get("QUANT_DB_HOST", "192.168.27.15"), "host": os.environ.get("QUANT_DB_HOST", "192.168.27.11"),
"port": int(os.environ.get("QUANT_DB_PORT", "12345")), "port": int(os.environ.get("QUANT_DB_PORT", "5438")),
"database": os.environ.get("QUANT_DB_NAME", "quant_db"), "database": os.environ.get("QUANT_DB_NAME", "quant_db"),
"user": os.environ.get("QUANT_DB_USER", "postgres"), "user": os.environ.get("QUANT_DB_USER", "postgres"),
"password": os.environ.get("QUANT_DB_PASSWORD", ""), "password": _PASSWORD,
} }
# 对密码进行 URL 编码(用于 SQLAlchemy 连接串),避免特殊字符导致解析错误
PASSWORD_ENCODED = quote_plus(_PASSWORD) if _PASSWORD else ""
# Tushare API Token # Tushare API Token
TUSHARE_TOKEN = os.environ.get("TUSHARE_TOKEN", "") TUSHARE_TOKEN = os.environ.get("TUSHARE_TOKEN", "")
+9 -4
View File
@@ -1,6 +1,6 @@
""" """
量化数据导入核心模块 量化数据导入核心模块
连接 Docker PostgreSQL (192.168.27.15:12345) 连接 Docker PostgreSQL (192.168.27.11:5438)
从 Tushare 拉取数据并批量导入 从 Tushare 拉取数据并批量导入
""" """
import time import time
@@ -15,7 +15,7 @@ from psycopg2 import sql
from psycopg2.extras import execute_values from psycopg2.extras import execute_values
from sqlalchemy import create_engine from sqlalchemy import create_engine
from config import DB_CONFIG, TUSHARE_TOKEN, BATCH_SIZE, START_DATE, END_DATE from config import DB_CONFIG, TUSHARE_TOKEN, BATCH_SIZE, START_DATE, END_DATE, PASSWORD_ENCODED
# ============================================================ # ============================================================
# 日志配置 # 日志配置
@@ -55,7 +55,7 @@ def get_sqlalchemy_engine():
global _sqlalchemy_engine global _sqlalchemy_engine
if _sqlalchemy_engine is None: if _sqlalchemy_engine is None:
db_url = ( db_url = (
f"postgresql://{DB_CONFIG['user']}:{DB_CONFIG['password']}" f"postgresql://{DB_CONFIG['user']}:{PASSWORD_ENCODED}"
f"@{DB_CONFIG['host']}:{DB_CONFIG['port']}/{DB_CONFIG['database']}" f"@{DB_CONFIG['host']}:{DB_CONFIG['port']}/{DB_CONFIG['database']}"
) )
_sqlalchemy_engine = create_engine(db_url, pool_size=5, max_overflow=10) _sqlalchemy_engine = create_engine(db_url, pool_size=5, max_overflow=10)
@@ -185,6 +185,9 @@ def import_stock_basic():
导入股票基本信息 (stock_basic) 导入股票基本信息 (stock_basic)
Tushare: stock_basic Tushare: stock_basic
""" """
# 确保数据库表结构已初始化
init_database()
logger.info("=" * 60) logger.info("=" * 60)
logger.info("[1/7] 导入股票基本信息 (stock_basic) ...") logger.info("[1/7] 导入股票基本信息 (stock_basic) ...")
@@ -820,7 +823,9 @@ def init_database():
# 执行 DDL # 执行 DDL
conn = get_pg_connection() conn = get_pg_connection()
try: try:
with open("schema.sql", "r", encoding="utf-8") as f: import os as _os
schema_path = _os.path.join(_os.path.dirname(_os.path.abspath(__file__)), "schema.sql")
with open(schema_path, "r", encoding="utf-8") as f:
ddl_sql = f.read() ddl_sql = f.read()
# 按分号分割,逐条执行 (忽略被注释掉的分区表DDL) # 按分号分割,逐条执行 (忽略被注释掉的分区表DDL)
+1
View File
@@ -2,3 +2,4 @@ tushare>=1.4.0
pandas>=1.5.0 pandas>=1.5.0
psycopg2-binary>=2.9.0 psycopg2-binary>=2.9.0
sqlalchemy>=2.0.0 sqlalchemy>=2.0.0
python-dotenv>=1.0.0
+1 -1
View File
@@ -1,6 +1,6 @@
-- ============================================================ -- ============================================================
-- 量化投资数据库 Schema -- 量化投资数据库 Schema
-- PostgreSQL (Docker: 192.168.27.15:12345) -- PostgreSQL (Docker: 192.168.27.11:5438)
-- 数据来源:Tushare -- 数据来源:Tushare
-- ============================================================ -- ============================================================
+64 -13
View File
@@ -7,7 +7,7 @@
"# 量化投资数据批量导入\n", "# 量化投资数据批量导入\n",
"\n", "\n",
"## 目标\n", "## 目标\n",
"将 Tushare 的日线行情数据及公司基本面数据批量导入 Docker PostgreSQL (192.168.27.15:12345)\n", "将 Tushare 的日线行情数据及公司基本面数据批量导入 Docker PostgreSQL (192.168.27.11:5438)\n",
"\n", "\n",
"## 数据库结构概览\n", "## 数据库结构概览\n",
"\n", "\n",
@@ -40,9 +40,18 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 8,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"工作目录: t:\\jupyter\\notebook\\quantitative_data\n",
"Python 版本: 3.10.2 (heads/master:d9999f5, Dec 16 2022, 16:20:32) [MSC v.1929 64 bit (AMD64)]\n"
]
}
],
"source": [ "source": [
"import sys\n", "import sys\n",
"import os\n", "import os\n",
@@ -53,12 +62,33 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 9,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"pandas 1.5.0\n",
"python-dotenv 1.2.2\n",
"SQLAlchemy 1.3.24\n",
"tushare 1.2.89\n",
"vnpy-tushare 1.2.85.1\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"[notice] A new release of pip available: 22.2.2 -> 26.2\n",
"[notice] To update, run: python.exe -m pip install --upgrade pip\n"
]
}
],
"source": [ "source": [
"# 检查依赖包\n", "# 检查依赖包\n",
"!pip list | findstr -i \"tushare pandas psycopg2 sqlalchemy\"" "!pip list | findstr -i \"tushare pandas psycopg2-binary sqlalchemy python-dotenv\""
] ]
}, },
{ {
@@ -73,9 +103,17 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 10,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"模块导入成功!\n"
]
}
],
"source": [ "source": [
"# 导入核心模块\n", "# 导入核心模块\n",
"from importer import (\n", "from importer import (\n",
@@ -105,9 +143,22 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 11,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"✗ 连接失败: connection to server at \"192.168.27.11\", port 5438 failed: fe_sendauth: no password supplied\n",
"\n",
"请检查:\n",
" 1. Docker 容器是否已启动: docker ps | findstr postgres\n",
" 2. 环境变量 (.env) 中的连接参数是否正确\n",
" 3. 防火墙是否开放 5438 端口\n"
]
}
],
"source": [ "source": [
"# 测试数据库连接\n", "# 测试数据库连接\n",
"try:\n", "try:\n",
@@ -125,7 +176,7 @@
" print(\"请检查:\")\n", " print(\"请检查:\")\n",
" print(\" 1. Docker 容器是否已启动: docker ps | findstr postgres\")\n", " print(\" 1. Docker 容器是否已启动: docker ps | findstr postgres\")\n",
" print(\" 2. 环境变量 (.env) 中的连接参数是否正确\")\n", " print(\" 2. 环境变量 (.env) 中的连接参数是否正确\")\n",
" print(\" 3. 防火墙是否开放 12345 端口\")" " print(\" 3. 防火墙是否开放 5438 端口\")"
] ]
}, },
{ {
@@ -625,7 +676,7 @@
], ],
"metadata": { "metadata": {
"kernelspec": { "kernelspec": {
"display_name": "Python 3 (ipykernel)", "display_name": "Python 3",
"language": "python", "language": "python",
"name": "python3" "name": "python3"
}, },
@@ -639,7 +690,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.11.6" "version": "3.10.2"
} }
}, },
"nbformat": 4, "nbformat": 4,