conda create -n adm python=3.10 -y
conda activate adm

pip install torch torchvision torchaudio
pip install tqdm wandb blobfile timm thop omegaconf
conda install -c conda-forge mpi4py mpich

# make folders to save experimental results
cd guided-diffusion
mkdir -p experiments/cache; mkdir -p experiments/sample; mkdir -p experiments/kd

• cifar10_32: Download ADM_cifar10_baseline.pt in DDPM-IP and move it to model/32x32_diffusion_cifar10.pt.
• celeba_64: Download DDPM_IP_celeba64.pt in DDPM-IP and move it to model/DDPM_IP_celeba64.pt.
• imagenet_32: Download DDPM_IP_imageNet32.pt in DDPM-IP and move it to model/DDPM_IP_imageNet32.pt.

path/to/config could be found in Here

① Preparation: Prepare dynamic batch for DKDM

CUDA_VISIBLE_DEVICES=0 mpiexec -n 1 python scripts/dynamic_prepare.py --config path/to/config

Note: You can download the dynamic batches generated by the authors from OneDrive, including CIFAR10-Dynamic-0.4.cache, CelebA64-Dynamic-0.4.cache and ImageNet32-Dynamic-0.4.cache. Then put them in experiments/cache.

② Training:

# cifar10_32, imagenet_32
CUDA_VISIBLE_DEVICES=0 mpiexec -n 1 python scripts/image_kd.py --config path/to/config
# celeba_64
CUDA_VISIBLE_DEVICES=0,1,2,3 mpiexec -n 4 python scripts/image_kd.py --config path/to/config

① Preparation: Prepare the synthetic dataset for baselines:

# cifar10_32
CUDA_VISIBLE_DEVICES=0,1,2,3 mpiexec -n 4 python scripts/image_sample.py --config yaml/cifar10_32/teacher.yaml --model_path model/32x32_diffusion_cifar10.pt --batch_size 2500 --num_samples 50000 --timestep_respacing 1000 --use_ddim False
# celeba_64
CUDA_VISIBLE_DEVICES=0,1,2,3 mpiexec -n 4 python scripts/image_sample.py --config yaml/celeba_64/teacher.yaml --model_path model/DDPM_IP_celeba64.pt --batch_size 300 --num_samples 202599 --timestep_respacing 100 --use_ddim False
# imagenet_32
CUDA_VISIBLE_DEVICES=0,1,2,3 mpiexec -n 4 python scripts/image_sample.py --config yaml/imagenet_32/teacher.yaml --model_path model/DDPM_IP_imageNet32.pt --batch_size 3072 --num_samples 1281167 --timestep_respacing 100 --use_ddim False

② Data-Free Training:

# cifar10_32, imagenet_32
CUDA_VISIBLE_DEVICES=0 mpiexec -n 1 python scripts/image_train.py --data_dir path/to/synthetic_data --config path/to/config
# celeba_64
CUDA_VISIBLE_DEVICES=0,1,2,3 mpiexec -n 4 python scripts/image_train.py --data_dir path/to/synthetic_data --config path/to/config

1. Refer to datasets/mix.py to generate the mixed dataset: 5% ~ 20% (Data-Limited Training), 100% (Data-Based Training).
2. Run the same command as above while specifying the path in data-free training to the mixed dataset.

Note: Our setting are based on guided-diffusion and DDPM-IP.

① Prepare environment

conda create -n eval python=3.10
conda activate eval
python3 -m pip install tensorflow[and-cuda]
pip install scipy requests tqdm

② Prepare npz files

Note: Here the setting is the same as DDPM-IP.

• Download Dataset: CIFAR-10 (refer to datasets/download_cifar10.py), CelebA (from GoogleDrive), ImageNet-32 (from ImageNet official website)
• Generate npz files: Refer to evaluations/cifar_npz.py, evaluations/celeba64.npz and evaluations/imagenet32_npz.py, run these scripts to get cifar10_train.npz, celeba64_train.npz and ImageNet32_train_all.npz.

③ Sample images from the trained models

CUDA_VISIBLE_DEVICES=0 mpiexec -n 1 python scripts/image_sample.py --config /path/to/config --model_path /path/to/model --batch_size 2500 --num_samples 50000 --timestep_respacing 50 --use_ddim False

④ Evaluation

python evaluations/evaluator.py path/to/cifar10_train.npz path/to/sample_batch.npz

You could download the models trained by our Dynamic Iterative Distillation from OneDrive.

Main Experiment

• DKDM: yaml/cifar10_32/exp/CIFAR10-14M-Dynamic-0.4.yaml
• Baseline: yaml/cifar10_32/exp/CIFAR10-14M-Baseline.yaml

Different Architectures

• 57M CNN Architecture
  • DKDM: yaml/cifar10_32/exp/CIFAR10-57M-Dynamic-0.4.yaml
  • Baseline: yaml/cifar10_32/exp/CIFAR10-57M-Baseline.yaml
• 44M CNN Architecture (residual blocks: 3 -> 2)
  • DKDM: yaml/cifar10_32/exp/CIFAR10-44M-Dynamic-0.4.yaml
  • Baseline: yaml/cifar10_32/exp/CIFAR10-44M-Baseline.yaml
• 32M CNN Architecture (channels: 128 -> 96)
  • DKDM: yaml/cifar10_32/exp/CIFAR10-32M-Dynamic-0.4.yaml
  • Baseline: yaml/cifar10_32/exp/CIFAR10-32M-Baseline.yaml
• 24M CNN Architecture (residual blocks: 3 -> 2, channels: 128 -> 96)
  • DKDM: yaml/cifar10_32/exp/CIFAR10-24M-Dynamic-0.4.yaml
  • Baseline: yaml/cifar10_32/exp/CIFAR10-24M-Baseline.yaml
• 14M CNN Architecture (channels: 128 -> 64)
  • DKDM: yaml/cifar10_32/exp/CIFAR10-14M-Dynamic-0.4.yaml
  • Baseline: yaml/cifar10_32/exp/CIFAR10-14M-Baseline.yaml

Ablation Study

rho means $\rho$ for dynamic iterative distillation:

• Iterative Distillation: yaml/cifar10_32/exp/ablation/iterative-distillation/CIFAR10-57M-Iterative.yaml
• Shuffled Iterative Distillation: yaml/cifar10_32/exp/ablation/shuffled-iterative-distillation/CIFAR10-57M-Shuffled.yaml
• Dynamic Iterative Distillation
  • rho=0.01: yaml/cifar10_32/exp/ablation/dynamic-iterative-distillation/CIFAR10-57M-Dynamic-0.01.yaml
  • rho=0.02: yaml/cifar10_32/exp/ablation/dynamic-iterative-distillation/CIFAR10-57M-Dynamic-0.02.yaml
  • rho=0.03: yaml/cifar10_32/exp/ablation/dynamic-iterative-distillation/CIFAR10-57M-Dynamic-0.03.yaml
  • rho=0.05: yaml/cifar10_32/exp/ablation/dynamic-iterative-distillation/CIFAR10-57M-Dynamic-0.05.yaml
  • rho=0.1: yaml/cifar10_32/exp/ablation/dynamic-iterative-distillation/CIFAR10-57M-Dynamic-0.1.yaml
  • rho=0.2: yaml/cifar10_32/exp/ablation/dynamic-iterative-distillation/CIFAR10-57M-Dynamic-0.2.yaml
  • rho=0.3: yaml/cifar10_32/exp/ablation/dynamic-iterative-distillation/CIFAR10-57M-Dynamic-0.3.yaml
  • rho=0.4: yaml/cifar10_32/exp/ablation/dynamic-iterative-distillation/CIFAR10-57M-Dynamic-0.4.yaml

• DKDM: yaml/celeba_64/exp/CelebA64-57M-Dynamic-0.4.yaml
• Baseline: yaml/celeba_64/exp/CelebA64-57M-Baseline.yaml

• DKDM: yaml/imagenet_32/exp/ImageNet32-14M-Dynamic-0.4.yaml
• Baseline: yaml/imagenet_32/exp/ImageNet32-14M-Baseline.yaml