189 lines
6.3 KiB
Python
189 lines
6.3 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
|
|
import os
|
|
import sys
|
|
import paramiko
|
|
from datetime import datetime
|
|
import time
|
|
import warnings
|
|
|
|
# 忽略paramiko的警告
|
|
warnings.filterwarnings("ignore", category=DeprecationWarning, module="paramiko")
|
|
|
|
|
|
def sftp_transfer():
|
|
# 获取当前日期
|
|
today_date = datetime.now().strftime('%Y%m%d')
|
|
|
|
# sftp服务器
|
|
host = '220.203.233.97'
|
|
port = 16888
|
|
username = 'sysop'
|
|
password = 'swjg_Boyizc1212'
|
|
|
|
# 远程路径 - 更新为v4
|
|
remote_base_path = f'/home/sysop/kr_md_save_centos7_9_v4/data/stockdata/{today_date}'
|
|
|
|
# 本地路径
|
|
local_base_path = f'G:\\data_new\\data\\stockdata\\{today_date}'
|
|
|
|
# 如果本地目录不存在，则创建
|
|
if not os.path.exists(local_base_path):
|
|
os.makedirs(local_base_path, exist_ok=True)
|
|
|
|
# 连接SFTP服务器
|
|
try:
|
|
transport = paramiko.Transport((host, port))
|
|
transport.connect(username=username, password=password)
|
|
sftp = paramiko.SFTPClient.from_transport(transport)
|
|
|
|
print(f"成功连接到SFTP服务器: {host}:{port}")
|
|
print(f"远程目录: {remote_base_path}")
|
|
print(f"本地目录: {local_base_path}")
|
|
|
|
# 列出远程目录中的所有文件
|
|
try:
|
|
remote_files = sftp.listdir(remote_base_path)
|
|
except FileNotFoundError:
|
|
print(f"错误: 远程目录不存在: {remote_base_path}")
|
|
print("可能原因: 1) 日期格式错误 2) 远程服务器上该日期的数据还未生成")
|
|
return False, 0, 0, []
|
|
|
|
if not remote_files:
|
|
print("警告: 远程目录为空，没有文件需要传输")
|
|
return True, 0, 0, []
|
|
|
|
print(f"找到 {len(remote_files)} 个文件需要传输")
|
|
|
|
# 传输文件
|
|
transferred_count = 0
|
|
transferred_files = []
|
|
failed_files = []
|
|
|
|
for filename in remote_files:
|
|
# 修复路径拼接问题，确保使用正确的路径分隔符
|
|
remote_file_path = f"{remote_base_path}/{filename}"
|
|
local_file_path = os.path.join(local_base_path, filename)
|
|
|
|
# 调试信息
|
|
print(f"远程文件路径: {remote_file_path}")
|
|
print(f"本地文件路径: {local_file_path}")
|
|
|
|
try:
|
|
# 检查文件是否存在且可读
|
|
try:
|
|
file_stat = sftp.stat(remote_file_path)
|
|
print(f"文件大小: {file_stat.st_size} 字节")
|
|
except Exception as e:
|
|
print(f"警告: 无法获取文件状态: {str(e)}")
|
|
|
|
print(f"正在传输: {filename} ...")
|
|
sftp.get(remote_file_path, local_file_path)
|
|
|
|
# 验证本地文件是否成功创建
|
|
if os.path.exists(local_file_path):
|
|
file_size = os.path.getsize(local_file_path)
|
|
transferred_count += 1
|
|
transferred_files.append(filename)
|
|
print(f" 完成 - 文件大小: {file_size} 字节")
|
|
else:
|
|
failed_files.append(filename)
|
|
print(f" 失败: 本地文件未创建")
|
|
|
|
except Exception as e:
|
|
failed_files.append(filename)
|
|
print(f" 传输失败: {str(e)}")
|
|
|
|
print(f"传输完成: 成功传输 {transferred_count}/{len(remote_files)} 个文件")
|
|
|
|
# 输出详细结果
|
|
if transferred_files:
|
|
print(f"成功传输的文件 ({len(transferred_files)}):")
|
|
for f in transferred_files:
|
|
print(f" ✓ {f}")
|
|
|
|
if failed_files:
|
|
print(f"传输失败的文件 ({len(failed_files)}):")
|
|
for f in failed_files:
|
|
print(f" ✗ {f}")
|
|
|
|
# 关闭连接
|
|
sftp.close()
|
|
transport.close()
|
|
|
|
return transferred_count > 0, transferred_count, len(remote_files), transferred_files
|
|
|
|
except Exception as e:
|
|
print(f"连接或传输过程中发生错误: {str(e)}")
|
|
return False, 0, 0, []
|
|
|
|
|
|
def create_success_marker(date_str, success=True, transferred=0, total=0, elapsed_time=0):
|
|
"""创建成功标记文件"""
|
|
try:
|
|
# 成功标记目录
|
|
success_dir = 'G:\\data_new\\success'
|
|
|
|
# 如果目录不存在则创建
|
|
if not os.path.exists(success_dir):
|
|
os.makedirs(success_dir, exist_ok=True)
|
|
|
|
# 创建成功标记文件（无后缀）
|
|
marker_file = os.path.join(success_dir, date_str)
|
|
|
|
# 如果传输成功才创建标记文件
|
|
if success and transferred > 0:
|
|
with open(marker_file, 'w', encoding='utf-8') as f:
|
|
pass # 创建空文件，不写入内容
|
|
print(f"成功标记文件已创建: {marker_file}")
|
|
return True
|
|
else:
|
|
# 如果传输失败，删除可能存在的旧标记文件
|
|
if os.path.exists(marker_file):
|
|
os.remove(marker_file)
|
|
print(f"删除失败的标记文件: {marker_file}")
|
|
return False
|
|
except Exception as e:
|
|
print(f"创建成功标记文件失败: {str(e)}")
|
|
return False
|
|
|
|
|
|
def main():
|
|
print("=" * 60)
|
|
print(f"开始执行SFTP数据同步任务")
|
|
print(f"时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
|
print("=" * 60)
|
|
|
|
# 获取当前日期
|
|
today_date = datetime.now().strftime('%Y%m%d')
|
|
|
|
# 记录开始时间
|
|
start_time = time.time()
|
|
|
|
# 执行传输任务
|
|
success, transferred_count, total_count, transferred_files = sftp_transfer()
|
|
|
|
# 记录结束时间
|
|
end_time = time.time()
|
|
elapsed_time = end_time - start_time
|
|
|
|
# 创建成功标记文件
|
|
if success and transferred_count > 0:
|
|
create_success_marker(today_date, success, transferred_count, total_count, elapsed_time)
|
|
|
|
print("=" * 60)
|
|
if success:
|
|
if total_count > 0:
|
|
print(f"任务执行成功! 传输 {transferred_count}/{total_count} 个文件，耗时: {elapsed_time:.2f}秒")
|
|
else:
|
|
print(f"任务执行成功! 远程目录为空，耗时: {elapsed_time:.2f}秒")
|
|
else:
|
|
print(f"任务执行失败! 耗时: {elapsed_time:.2f}秒")
|
|
print("=" * 60)
|
|
|
|
return 0 if success and transferred_count > 0 else 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main()) |