wei/sync/sftp_sync_boyigo.py

#############
#2025-11-17 
#weiyongfu  
#sync L2    
#单线程效率慢
#############
import paramiko
import os
import stat
from datetime import datetime
import sys

def sync_remote_to_local():
    # 服务器连接信息
    hostname = "59.36.23.197"
    port = 16888
    username = "boyigo"
    
    # 私钥文件路径
    private_key_path = r"C:\Users\admin\Desktop\boyigo"
    
    # 获取当天日期
    today = datetime.now()
    year = today.strftime("%Y")
    month = today.strftime("%m")
    day = today.strftime("%d")
    date_ymd = today.strftime("%Y%m%d")  # 格式: 20251114
    date_md = today.strftime("%m%d")     # 格式: 1114
    
    # 根据日期动态生成同步路径
    sync_paths = [
        {
            "remote": f"/list/10.99.9.75_sftp/L2/{date_ymd}SH",
            "local": fr"E:\data\SHL2\{year}\{date_ymd}"
        },
        {
            "remote": f"/list/10.99.9.75_sftp/L2/{date_ymd}SZ", 
            "local": fr"E:\data\SZL2\{year}\{date_md}"
        }
    ]
    
    # 创建SSH客户端
    ssh_client = paramiko.SSHClient()
    ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    
    try:
        print(f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S')} - 开始连接服务器...")
        print(f"主机: {hostname}:{port}, 用户: {username}")
        
        # 尝试加载私钥文件
        try:
            # 首先尝试作为RSA密钥加载
            private_key = paramiko.RSAKey.from_private_key_file(private_key_path)
            print(f"使用RSA私钥文件: {private_key_path}")
        except paramiko.ssh_exception.PasswordRequiredException:
            # 如果私钥有密码保护，需要输入密码
            key_password = input("请输入私钥密码: ")
            private_key = paramiko.RSAKey.from_private_key_file(private_key_path, password=key_password)
        except Exception as e1:
            print(f"RSA密钥加载失败: {e1}")
            try:
                # 尝试作为ECDSA密钥加载
                private_key = paramiko.ECDSAKey.from_private_key_file(private_key_path)
                print(f"使用ECDSA私钥文件: {private_key_path}")
            except Exception as e2:
                print(f"ECDSA密钥加载失败: {e2}")
                try:
                    # 尝试作为Ed25519密钥加载
                    private_key = paramiko.Ed25519Key.from_private_key_file(private_key_path)
                    print(f"使用Ed25519私钥文件: {private_key_path}")
                except Exception as e3:
                    print(f"所有密钥格式尝试失败: {e3}")
                    print("请检查私钥文件路径和格式")
                    return False
        
        # 使用密钥建立连接
        ssh_client.connect(hostname, port, username, pkey=private_key, timeout=30)
        sftp = ssh_client.open_sftp()
        
        print("连接成功，开始同步文件...")
        print(f"同步日期: {date_ymd}")
        
        # 遍历所有需要同步的路径
        total_files = 0
        for path_info in sync_paths:
            remote_path = path_info["remote"]
            local_path = path_info["local"]
            
            print(f"\n正在同步: {remote_path} -> {local_path}")
            
            # 确保本地目录存在
            os.makedirs(local_path, exist_ok=True)
            
            # 同步目录并统计文件数量
            file_count = sync_directory(sftp, remote_path, local_path)
            total_files += file_count
        
        print(f"\n{datetime.now().strftime('%Y-%m-%d %H:%M:%S')} - 同步完成!")
        print(f"总共处理了 {total_files} 个文件")
        
    except Exception as e:
        print(f"同步过程中出现错误: {e}")
        return False
    finally:
        # 关闭连接
        try:
            sftp.close()
        except:
            pass
        ssh_client.close()
    
    return True

def sync_directory(sftp, remote_path, local_path):
    """递归同步目录，返回处理的文件数量"""
    file_count = 0
    try:
        # 获取远程目录列表
        remote_items = sftp.listdir_attr(remote_path)
    except Exception as e:
        print(f"  无法访问远程目录 {remote_path}: {e}")
        return file_count
    
    for item in remote_items:
        remote_item_path = os.path.join(remote_path, item.filename).replace('\\', '/')
        local_item_path = os.path.join(local_path, item.filename)
        
        # 判断是文件还是目录
        if stat.S_ISDIR(item.st_mode):
            # 如果是目录，递归同步
            os.makedirs(local_item_path, exist_ok=True)
            sub_count = sync_directory(sftp, remote_item_path, local_item_path)
            file_count += sub_count
        else:
            # 如果是文件，检查是否需要同步
            if sync_file(sftp, remote_item_path, local_item_path, item):
                file_count += 1
    
    if file_count > 0:
        print(f"  {remote_path}: 处理了 {file_count} 个文件")
    
    return file_count

def sync_file(sftp, remote_file_path, local_file_path, remote_attr):
    """同步单个文件，返回True如果文件被下载或更新"""
    try:
        # 检查本地文件是否存在
        if os.path.exists(local_file_path):
            local_stat = os.stat(local_file_path)
            local_size = local_stat.st_size
            local_mtime = local_stat.st_mtime
            
            # 获取远程文件修改时间
            remote_mtime = remote_attr.st_mtime
            
            # 如果文件大小和修改时间都相同，则跳过
            if local_size == remote_attr.st_size and abs(local_mtime - remote_mtime) < 2:
                return False
            else:
                print(f"  更新: {os.path.basename(remote_file_path)} (大小: {local_size} -> {remote_attr.st_size})")
        else:
            print(f"  下载: {os.path.basename(remote_file_path)} (大小: {remote_attr.st_size} bytes)")
        
        # 下载文件
        sftp.get(remote_file_path, local_file_path)
        
        # 保持文件修改时间
        os.utime(local_file_path, (remote_attr.st_atime, remote_attr.st_mtime))
        
        return True
        
    except Exception as e:
        print(f"  下载失败 {os.path.basename(remote_file_path)}: {e}")
        return False

def main():
    """主函数"""
    print("=" * 60)
    print("远程文件同步工具 (SFTP + 密钥认证)")
    print("=" * 60)
    
    success = sync_remote_to_local()
    
    if success:
        print("\n同步成功!")
        return 0
    else:
        print("\n同步失败!")
        return 1

if __name__ == "__main__":
    exit_code = main()
    input("\n按Enter键退出...")
    sys.exit(exit_code)