import os
import hashlib
TEMP_DIR = "temp_chunks"
OUTPUT_FILE = "merged_clean.txt"
NUM_BUCKETS = 100 # ممكن تزود لو الداتا ضخمة جًدا
# إنشاء فولدر مؤقت
[Link](TEMP_DIR, exist_ok=True)
# =============================
# توزيع البيانات:1 المرحلة
# =============================
print("🚀 توزيع البيانات علىbuckets...")
bucket_files = [open(f"{TEMP_DIR}/bucket_{i}.txt", "w", encoding="utf-8") for i in
range(NUM_BUCKETS)]
for filename in [Link]():
if [Link](".txt") and filename != OUTPUT_FILE:
with open(filename, "r", encoding="utf-8", errors="ignore") as f:
for line in f:
line = [Link]()
if not line:
continue
# hash لتحديدbucket
h = int(hashlib.md5([Link]()).hexdigest(), 16)
bucket_index = h % NUM_BUCKETS
bucket_files[bucket_index].write(line + "\n")
# اقفل الملفات
for f in bucket_files:
[Link]()
# =============================
# إزالة التكرار داخل كل:2 المرحلةbucket
# =============================
print("🚀 إزالة التكرار...")
with open(OUTPUT_FILE, "w", encoding="utf-8") as outfile:
for i in range(NUM_BUCKETS):
path = f"{TEMP_DIR}/bucket_{i}.txt"
if not [Link](path):
continue
unique_lines = set()
with open(path, "r", encoding="utf-8") as f:
for line in f:
unique_lines.add([Link]())
for line in unique_lines:
[Link](line + "\n")
print(f"✅ Bucket {i} processed ({len(unique_lines)} lines)")
)"} {OUTPUT_FILEتم الدمج +حذف التكرار في 🔥print(f"\n