Compare commits

...
5 Commits
7 changed files with 114 additions and 27 deletions
+1
View File
@@ -4,3 +4,4 @@ scipy>=1.10.0
matplotlib>=3.7.0
tushare>=1.3.0
psycopg2-binary>=2.9.0
python-dotenv>=1.0.0
+2 -2
View File
@@ -4,8 +4,8 @@
# ========================================
# PostgreSQL 数据库连接
QUANT_DB_HOST=192.168.27.15
QUANT_DB_PORT=12345
QUANT_DB_HOST=192.168.27.11
QUANT_DB_PORT=5438
QUANT_DB_NAME=quant_db
QUANT_DB_USER=postgres
QUANT_DB_PASSWORD=your_password_here
+33 -4
View File
@@ -8,19 +8,48 @@
设置方式:
Windows: set QUANT_DB_PASSWORD=your_password && set TUSHARE_TOKEN=your_token
Linux: export QUANT_DB_PASSWORD=your_password && export TUSHARE_TOKEN=your_token
也可创建 .env 文件(参考 .env.example
也可创建 .env 文件(参考 .env.example
"""
import os
from pathlib import Path
from urllib.parse import quote_plus
# 加载 .env 文件中的环境变量
_LOADED = False
try:
from dotenv import load_dotenv
# 使用 resolve() 获取绝对路径,避免 os.chdir() 改变工作目录后 __file__ 变为相对路径
env_path = Path(__file__).resolve().parent / ".env"
# 如果 resolve 后的路径不存在,尝试从当前文件所在目录查找
if not env_path.exists():
# 备用方案:也检查 Notebook 的当前工作目录
cwd_path = Path.cwd() / ".env"
if cwd_path.exists():
env_path = cwd_path
if env_path.exists():
load_dotenv(dotenv_path=env_path, override=True)
print(f"✓ 已加载环境变量文件: {env_path}")
_LOADED = True
else:
print(f"⚠ 未找到 .env 文件。检查路径: {Path(__file__).resolve().parent / '.env'}, 当前目录: {Path.cwd() / '.env'}")
except ImportError:
print("⚠ python-dotenv 未安装,无法自动加载 .env 文件。请手动设置环境变量或执行: pip install python-dotenv")
# PostgreSQL Docker 连接配置
_PASSWORD = os.environ.get("QUANT_DB_PASSWORD", "")
DB_CONFIG = {
"host": os.environ.get("QUANT_DB_HOST", "192.168.27.15"),
"port": int(os.environ.get("QUANT_DB_PORT", "12345")),
"host": os.environ.get("QUANT_DB_HOST", "192.168.27.11"),
"port": int(os.environ.get("QUANT_DB_PORT", "5438")),
"database": os.environ.get("QUANT_DB_NAME", "quant_db"),
"user": os.environ.get("QUANT_DB_USER", "postgres"),
"password": os.environ.get("QUANT_DB_PASSWORD", ""),
"password": _PASSWORD,
}
# 对密码进行 URL 编码(用于 SQLAlchemy 连接串),避免特殊字符导致解析错误
PASSWORD_ENCODED = quote_plus(_PASSWORD) if _PASSWORD else ""
# Tushare API Token
TUSHARE_TOKEN = os.environ.get("TUSHARE_TOKEN", "")
+9 -4
View File
@@ -1,6 +1,6 @@
"""
量化数据导入核心模块
连接 Docker PostgreSQL (192.168.27.15:12345)
连接 Docker PostgreSQL (192.168.27.11:5438)
从 Tushare 拉取数据并批量导入
"""
import time
@@ -15,7 +15,7 @@ from psycopg2 import sql
from psycopg2.extras import execute_values
from sqlalchemy import create_engine
from config import DB_CONFIG, TUSHARE_TOKEN, BATCH_SIZE, START_DATE, END_DATE
from config import DB_CONFIG, TUSHARE_TOKEN, BATCH_SIZE, START_DATE, END_DATE, PASSWORD_ENCODED
# ============================================================
# 日志配置
@@ -55,7 +55,7 @@ def get_sqlalchemy_engine():
global _sqlalchemy_engine
if _sqlalchemy_engine is None:
db_url = (
f"postgresql://{DB_CONFIG['user']}:{DB_CONFIG['password']}"
f"postgresql://{DB_CONFIG['user']}:{PASSWORD_ENCODED}"
f"@{DB_CONFIG['host']}:{DB_CONFIG['port']}/{DB_CONFIG['database']}"
)
_sqlalchemy_engine = create_engine(db_url, pool_size=5, max_overflow=10)
@@ -185,6 +185,9 @@ def import_stock_basic():
导入股票基本信息 (stock_basic)
Tushare: stock_basic
"""
# 确保数据库表结构已初始化
init_database()
logger.info("=" * 60)
logger.info("[1/7] 导入股票基本信息 (stock_basic) ...")
@@ -820,7 +823,9 @@ def init_database():
# 执行 DDL
conn = get_pg_connection()
try:
with open("schema.sql", "r", encoding="utf-8") as f:
import os as _os
schema_path = _os.path.join(_os.path.dirname(_os.path.abspath(__file__)), "schema.sql")
with open(schema_path, "r", encoding="utf-8") as f:
ddl_sql = f.read()
# 按分号分割,逐条执行 (忽略被注释掉的分区表DDL)
+1
View File
@@ -2,3 +2,4 @@ tushare>=1.4.0
pandas>=1.5.0
psycopg2-binary>=2.9.0
sqlalchemy>=2.0.0
python-dotenv>=1.0.0
+1 -1
View File
@@ -1,6 +1,6 @@
-- ============================================================
-- 量化投资数据库 Schema
-- PostgreSQL (Docker: 192.168.27.15:12345)
-- PostgreSQL (Docker: 192.168.27.11:5438)
-- 数据来源:Tushare
-- ============================================================
+65 -14
View File
@@ -7,7 +7,7 @@
"# 量化投资数据批量导入\n",
"\n",
"## 目标\n",
"将 Tushare 的日线行情数据及公司基本面数据批量导入 Docker PostgreSQL (192.168.27.15:12345)\n",
"将 Tushare 的日线行情数据及公司基本面数据批量导入 Docker PostgreSQL (192.168.27.11:5438)\n",
"\n",
"## 数据库结构概览\n",
"\n",
@@ -40,9 +40,18 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 8,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"工作目录: t:\\jupyter\\notebook\\quantitative_data\n",
"Python 版本: 3.10.2 (heads/master:d9999f5, Dec 16 2022, 16:20:32) [MSC v.1929 64 bit (AMD64)]\n"
]
}
],
"source": [
"import sys\n",
"import os\n",
@@ -53,12 +62,33 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 9,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"pandas 1.5.0\n",
"python-dotenv 1.2.2\n",
"SQLAlchemy 1.3.24\n",
"tushare 1.2.89\n",
"vnpy-tushare 1.2.85.1\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"[notice] A new release of pip available: 22.2.2 -> 26.2\n",
"[notice] To update, run: python.exe -m pip install --upgrade pip\n"
]
}
],
"source": [
"# 检查依赖包\n",
"!pip list | findstr -i \"tushare pandas psycopg2 sqlalchemy\""
"!pip list | findstr -i \"tushare pandas psycopg2-binary sqlalchemy python-dotenv\""
]
},
{
@@ -68,14 +98,22 @@
"outputs": [],
"source": [
"# 如果需要安装依赖,取消注释下面这行\n",
"# !pip install -r requirements.txt"
"#!pip install -r requirements.txt"
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 10,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"模块导入成功!\n"
]
}
],
"source": [
"# 导入核心模块\n",
"from importer import (\n",
@@ -105,9 +143,22 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 11,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"✗ 连接失败: connection to server at \"192.168.27.11\", port 5438 failed: fe_sendauth: no password supplied\n",
"\n",
"请检查:\n",
" 1. Docker 容器是否已启动: docker ps | findstr postgres\n",
" 2. 环境变量 (.env) 中的连接参数是否正确\n",
" 3. 防火墙是否开放 5438 端口\n"
]
}
],
"source": [
"# 测试数据库连接\n",
"try:\n",
@@ -125,7 +176,7 @@
" print(\"请检查:\")\n",
" print(\" 1. Docker 容器是否已启动: docker ps | findstr postgres\")\n",
" print(\" 2. 环境变量 (.env) 中的连接参数是否正确\")\n",
" print(\" 3. 防火墙是否开放 12345 端口\")"
" print(\" 3. 防火墙是否开放 5438 端口\")"
]
},
{
@@ -625,7 +676,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
@@ -639,7 +690,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.6"
"version": "3.10.2"
}
},
"nbformat": 4,