228 lines
7.7 KiB
Python
228 lines
7.7 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
|
|
import os
|
|
import sys
|
|
import paramiko
|
|
from datetime import datetime
|
|
import time
|
|
import warnings
|
|
|
|
# 忽略paramiko的警告
|
|
warnings.filterwarnings("ignore", category=DeprecationWarning, module="paramiko")
|
|
|
|
|
|
def sftp_transfer():
|
|
# 获取当前日期
|
|
today_date = datetime.now().strftime('%Y%m%d')
|
|
|
|
# sftp服务器
|
|
host = '220.203.233.97'
|
|
port = 16888
|
|
username = 'sysop'
|
|
password = 'swjg_Boyizc1212'
|
|
|
|
# 远程路径 - 更新为v4
|
|
remote_base_path = f'/home/sysop/kr_md_save_centos7_9_v4/data/stockdata/{today_date}'
|
|
|
|
# 本地路径
|
|
local_base_path = f'G:\\data_new\\data\\stockdata\\{today_date}'
|
|
|
|
# 如果本地目录不存在，则创建
|
|
if not os.path.exists(local_base_path):
|
|
os.makedirs(local_base_path, exist_ok=True)
|
|
|
|
# 连接SFTP服务器
|
|
try:
|
|
transport = paramiko.Transport((host, port))
|
|
transport.connect(username=username, password=password)
|
|
sftp = paramiko.SFTPClient.from_transport(transport)
|
|
|
|
print(f"成功连接到SFTP服务器: {host}:{port}")
|
|
print(f"远程目录: {remote_base_path}")
|
|
print(f"本地目录: {local_base_path}")
|
|
|
|
# 列出远程目录中的所有文件
|
|
try:
|
|
remote_files = sftp.listdir(remote_base_path)
|
|
except FileNotFoundError:
|
|
print(f"错误: 远程目录不存在: {remote_base_path}")
|
|
print("可能原因: 1) 日期格式错误 2) 远程服务器上该日期的数据还未生成")
|
|
return False, 0, 0, []
|
|
|
|
if not remote_files:
|
|
print("警告: 远程目录为空，没有文件需要传输")
|
|
return True, 0, 0, []
|
|
|
|
print(f"找到 {len(remote_files)} 个文件需要传输")
|
|
|
|
# 传输文件
|
|
transferred_count = 0
|
|
transferred_files = []
|
|
failed_files = []
|
|
|
|
for filename in remote_files:
|
|
# 修复路径拼接问题，确保使用正确的路径分隔符
|
|
remote_file_path = f"{remote_base_path}/{filename}"
|
|
local_file_path = os.path.join(local_base_path, filename)
|
|
|
|
# 调试信息
|
|
print(f"远程文件路径: {remote_file_path}")
|
|
print(f"本地文件路径: {local_file_path}")
|
|
|
|
try:
|
|
# 检查文件是否存在且可读
|
|
try:
|
|
file_stat = sftp.stat(remote_file_path)
|
|
print(f"文件大小: {file_stat.st_size} 字节")
|
|
except Exception as e:
|
|
print(f"警告: 无法获取文件状态: {str(e)}")
|
|
|
|
print(f"正在传输: {filename} ...")
|
|
sftp.get(remote_file_path, local_file_path)
|
|
|
|
# 验证本地文件是否成功创建
|
|
if os.path.exists(local_file_path):
|
|
file_size = os.path.getsize(local_file_path)
|
|
transferred_count += 1
|
|
transferred_files.append(filename)
|
|
print(f" 完成 - 文件大小: {file_size} 字节")
|
|
else:
|
|
failed_files.append(filename)
|
|
print(f" 失败: 本地文件未创建")
|
|
|
|
except Exception as e:
|
|
failed_files.append(filename)
|
|
print(f" 传输失败: {str(e)}")
|
|
|
|
print(f"传输完成: 成功传输 {transferred_count}/{len(remote_files)} 个文件")
|
|
|
|
# 输出详细结果
|
|
if transferred_files:
|
|
print(f"成功传输的文件 ({len(transferred_files)}):")
|
|
for f in transferred_files:
|
|
print(f" ✓ {f}")
|
|
|
|
if failed_files:
|
|
print(f"传输失败的文件 ({len(failed_files)}):")
|
|
for f in failed_files:
|
|
print(f" ✗ {f}")
|
|
|
|
# 关闭连接
|
|
sftp.close()
|
|
transport.close()
|
|
|
|
return transferred_count > 0, transferred_count, len(remote_files), transferred_files
|
|
|
|
except Exception as e:
|
|
print(f"连接或传输过程中发生错误: {str(e)}")
|
|
return False, 0, 0, []
|
|
|
|
|
|
def create_success_marker(date_str, success=True, transferred=0, total=0, elapsed_time=0, transferred_files=None):
|
|
"""
|
|
在本地创建成功标记文件
|
|
只有在传输成功时才创建文件
|
|
|
|
参数:
|
|
date_str: 日期字符串
|
|
success: 是否成功
|
|
transferred: 成功传输的文件数
|
|
total: 总文件数
|
|
elapsed_time: 传输耗时
|
|
transferred_files: 成功传输的文件列表
|
|
"""
|
|
# 成功标记目录
|
|
success_dir = r"G:\data_new\success"
|
|
marker_file = os.path.join(success_dir, date_str)
|
|
|
|
try:
|
|
# 确保目录存在
|
|
os.makedirs(success_dir, exist_ok=True)
|
|
|
|
# 只有在传输成功且传输文件数大于0时才创建文件
|
|
if success and transferred > 0:
|
|
# 创建文件并写入传输信息
|
|
with open(marker_file, 'w', encoding='utf-8') as f:
|
|
timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
|
content = f"传输成功 - {timestamp}\n"
|
|
content += f"传输日期: {date_str}\n"
|
|
content += f"传输文件数: {transferred}/{total}\n"
|
|
content += f"传输耗时: {elapsed_time:.2f}秒\n"
|
|
content += "=" * 40 + "\n"
|
|
|
|
# 添加传输的文件列表（如果有）
|
|
if transferred_files and len(transferred_files) > 0:
|
|
content += f"成功传输的文件列表 ({len(transferred_files)}):\n"
|
|
for i, file_name in enumerate(transferred_files, 1):
|
|
content += f" {i:3d}. {file_name}\n"
|
|
else:
|
|
content += "文件列表: 无\n"
|
|
|
|
f.write(content)
|
|
|
|
print(f"成功标记文件创建成功: {marker_file}")
|
|
print(f"传输状态: 成功，传输 {transferred}/{total} 个文件")
|
|
return True
|
|
else:
|
|
# 传输失败时，如果文件已存在，删除它
|
|
if os.path.exists(marker_file):
|
|
os.remove(marker_file)
|
|
print(f"删除失败的标记文件: {marker_file}")
|
|
|
|
# 根据失败原因给出不同的提示
|
|
if not success:
|
|
print("传输失败: 连接或传输过程中出现错误")
|
|
elif transferred == 0:
|
|
print("传输失败: 没有成功传输任何文件")
|
|
else:
|
|
print("传输失败: 未知原因")
|
|
|
|
return False
|
|
except Exception as e:
|
|
print(f"操作成功标记文件失败: {e}")
|
|
return False
|
|
|
|
|
|
def main():
|
|
print("=" * 60)
|
|
print(f"开始执行SFTP数据同步任务")
|
|
print(f"时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
|
print("=" * 60)
|
|
|
|
# 获取当前日期
|
|
today_date = datetime.now().strftime('%Y%m%d')
|
|
|
|
# 记录开始时间
|
|
start_time = time.time()
|
|
|
|
# 执行传输任务
|
|
success, transferred_count, total_count, transferred_files = sftp_transfer()
|
|
|
|
# 记录结束时间
|
|
end_time = time.time()
|
|
elapsed_time = end_time - start_time
|
|
|
|
# 创建成功标记文件
|
|
marker_created = create_success_marker(
|
|
today_date,
|
|
success,
|
|
transferred_count,
|
|
total_count,
|
|
elapsed_time,
|
|
transferred_files
|
|
)
|
|
|
|
print("=" * 60)
|
|
if success and transferred_count > 0:
|
|
print(f"任务执行成功! 传输 {transferred_count}/{total_count} 个文件，耗时: {elapsed_time:.2f}秒")
|
|
else:
|
|
print(
|
|
f"任务执行失败或没有文件传输! 成功: {success}, 传输文件数: {transferred_count}, 耗时: {elapsed_time:.2f}秒")
|
|
print("=" * 60)
|
|
|
|
return 0 if success and transferred_count > 0 else 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main()) |