다중 대용량 파일 병렬 로딩 & 멀티프로세싱

from concurrent.futures import ProcessPoolExecutor, as_completed
import pandas as pd
import glob, os, time

def process_single_file(file_path):
    df = pd.read_csv(file_path, usecols=['branch_id', 'amount'])
    return df['amount'].sum(), len(df)

def parallel_aggregate(file_list, max_workers=8):
    total_sales, total_rows = 0, 0
    start = time.time()
    
    with ProcessPoolExecutor(max_workers=max_workers) as executor:
        futures = {executor.submit(process_single_file, f): f for f in file_list}
        for future in as_completed(futures):
            s, r = future.result()
            total_sales += s
            total_rows += r
            
    print(f"🔥 [병렬 집계 완료] {len(file_list)}개 파일 | 총 {total_rows:,}행 | 소요: {time.time()-start:.2f}초")
    return total_sales