Skip to content

Commit 41b4865

Browse files
stduhpfstduhpf
committed
add uncond slg variant
1 parent bc75149 commit 41b4865

File tree

3 files changed

+52
-23
lines changed

3 files changed

+52
-23
lines changed

‎examples/cli/main.cpp

+12-3
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,7 @@ struct SDParams {
136136
float slg_scale = 0.0f;
137137
float skip_layer_start = 0.01f;
138138
float skip_layer_end = 0.2f;
139+
bool slg_uncond = false;
139140

140141
float apg_eta = 1.0f;
141142
float apg_momentum = 0.0f;
@@ -240,11 +241,14 @@ void print_usage(int argc, const char* argv[]) {
240241
printf(" (replaces saturation with a smooth approximation)\n");
241242
printf(" --slg-scale SCALE skip layer guidance (SLG) scale, only for DiT models: (default: 0)\n");
242243
printf(" 0 means disabled, a value of 2.5 is nice for sd3.5 medium\n");
243-
printf(" --eta SCALE eta in DDIM, only for DDIM and TCD: (default: 0)\n");
244+
printf(" --slg-uncond Use CFG's forward pass for SLG instead of a separate pass, only for DiT models\n");
245+
printf(" To use this, it's recommended to keep slg-scale to 0, both for performance and quality reasons\n");
246+
printf(" This should be slightly faster than normal cfg when cfg_scale != 1.\n");
244247
printf(" --skip-layers LAYERS Layers to skip for SLG steps: (default: [7,8,9])\n");
245248
printf(" --skip-layer-start START SLG enabling point: (default: 0.01)\n");
246249
printf(" --skip-layer-end END SLG disabling point: (default: 0.2)\n");
247250
printf(" SLG will be enabled at step int([STEPS]*[START]) and disabled at int([STEPS]*[END])\n");
251+
printf(" --eta SCALE eta in DDIM, only for DDIM and TCD: (default: 0)\n");
248252
printf(" --strength STRENGTH strength for noising/unnoising (default: 0.75)\n");
249253
printf(" --style-ratio STYLE-RATIO strength for keeping input identity (default: 20%%)\n");
250254
printf(" --control-strength STRENGTH strength to apply Control Net (default: 0.9)\n");
@@ -611,6 +615,8 @@ void parse_args(int argc, const char** argv, SDParams& params) {
611615
break;
612616
}
613617
params.slg_scale = std::stof(argv[i]);
618+
} else if (arg == "--slg-uncond") {
619+
params.slg_uncond = true;
614620
} else if (arg == "--skip-layers") {
615621
if (++i >= argc) {
616622
invalid_arg = true;
@@ -816,6 +822,7 @@ std::string get_image_params(SDParams params, int64_t seed) {
816822
}
817823
}
818824
if (params.slg_scale != 0 && params.skip_layers.size() != 0) {
825+
parameter_string += "Unconditional SLG: " + std::string(params.slg_uncond ? "True" : "False") + ", ";
819826
parameter_string += "SLG scale: " + std::to_string(params.cfg_scale) + ", ";
820827
parameter_string += "Skip layers: [";
821828
for (const auto& layer : params.skip_layers) {
@@ -1072,7 +1079,8 @@ int main(int argc, const char* argv[]) {
10721079
params.skip_layers.size(),
10731080
params.slg_scale,
10741081
params.skip_layer_start,
1075-
params.skip_layer_end},
1082+
params.skip_layer_end,
1083+
params.slg_uncond},
10761084
sd_apg_params_t{params.apg_eta,
10771085
params.apg_momentum,
10781086
params.apg_norm_threshold,
@@ -1145,7 +1153,8 @@ int main(int argc, const char* argv[]) {
11451153
params.skip_layers.size(),
11461154
params.slg_scale,
11471155
params.skip_layer_start,
1148-
params.skip_layer_end},
1156+
params.skip_layer_end,
1157+
params.slg_uncond},
11491158
sd_apg_params_t{params.apg_eta,
11501159
params.apg_momentum,
11511160
params.apg_norm_threshold,

‎stable-diffusion.cpp

+39-20
Original file line numberDiff line numberDiff line change
@@ -922,7 +922,7 @@ class StableDiffusionGGML {
922922
const std::vector<float>& sigmas,
923923
int start_merge_step,
924924
SDCondition id_cond,
925-
sd_slg_params_t slg_params = {NULL, 0, 0, 0, 0},
925+
sd_slg_params_t slg_params = {NULL, 0, 0, 0, false},
926926
sd_apg_params_t apg_params = {1, 0, 0, 0},
927927
ggml_tensor* noise_mask = nullptr) {
928928
std::vector<int> skip_layers(slg_params.skip_layers, slg_params.skip_layers + slg_params.skip_layers_count);
@@ -949,7 +949,7 @@ class StableDiffusionGGML {
949949
struct ggml_tensor* noised_input = ggml_dup_tensor(work_ctx, noise);
950950

951951
bool has_unconditioned = cfg_scale != 1.0 && uncond.c_crossattn != NULL;
952-
bool has_skiplayer = slg_params.scale != 0.0 && skip_layers.size() > 0;
952+
bool has_skiplayer = (slg_params.scale != 0.0 || slg_params.slg_uncond) && skip_layers.size() > 0;
953953

954954
// denoise wrapper
955955
struct ggml_tensor* out_cond = ggml_dup_tensor(work_ctx, x);
@@ -961,7 +961,9 @@ class StableDiffusionGGML {
961961
}
962962
if (has_skiplayer) {
963963
if (sd_version_is_dit(version)) {
964-
out_skip = ggml_dup_tensor(work_ctx, x);
964+
if (slg_params.scale != 0.0) {
965+
out_skip = ggml_dup_tensor(work_ctx, x);
966+
}
965967
} else {
966968
has_skiplayer = false;
967969
LOG_WARN("SLG is incompatible with %s models", model_version_to_str[version]);
@@ -970,7 +972,7 @@ class StableDiffusionGGML {
970972
struct ggml_tensor* denoised = ggml_dup_tensor(work_ctx, x);
971973

972974
struct ggml_tensor* preview_tensor = NULL;
973-
auto sd_preview_mode = sd_get_preview_mode();
975+
auto sd_preview_mode = sd_get_preview_mode();
974976
if (sd_preview_mode != SD_PREVIEW_NONE && sd_preview_mode != SD_PREVIEW_PROJ) {
975977
preview_tensor = ggml_new_tensor_4d(work_ctx, GGML_TYPE_F32,
976978
(denoised->ne[0] * 8),
@@ -1040,6 +1042,8 @@ class StableDiffusionGGML {
10401042
control_strength,
10411043
&out_cond);
10421044
}
1045+
int step_count = sigmas.size();
1046+
bool is_skiplayer_step = has_skiplayer && step > (int)(slg_params.skip_layer_start * step_count) && step < (int)(slg_params.skip_layer_end * step_count);
10431047

10441048
float* negative_data = NULL;
10451049
if (has_unconditioned) {
@@ -1048,24 +1052,39 @@ class StableDiffusionGGML {
10481052
control_net->compute(n_threads, noised_input, control_hint, timesteps, uncond.c_crossattn, uncond.c_vector);
10491053
controls = control_net->controls;
10501054
}
1051-
diffusion_model->compute(n_threads,
1052-
noised_input,
1053-
timesteps,
1054-
uncond.c_crossattn,
1055-
uncond.c_concat,
1056-
uncond.c_vector,
1057-
guidance_tensor,
1058-
-1,
1059-
controls,
1060-
control_strength,
1061-
&out_uncond);
1055+
if (is_skiplayer_step && slg_params.slg_uncond) {
1056+
LOG_DEBUG("Skipping layers at uncond step %d\n", step);
1057+
diffusion_model->compute(n_threads,
1058+
noised_input,
1059+
timesteps,
1060+
uncond.c_crossattn,
1061+
uncond.c_concat,
1062+
uncond.c_vector,
1063+
guidance_tensor,
1064+
-1,
1065+
controls,
1066+
control_strength,
1067+
&out_uncond,
1068+
NULL,
1069+
skip_layers);
1070+
} else {
1071+
diffusion_model->compute(n_threads,
1072+
noised_input,
1073+
timesteps,
1074+
uncond.c_crossattn,
1075+
uncond.c_concat,
1076+
uncond.c_vector,
1077+
guidance_tensor,
1078+
-1,
1079+
controls,
1080+
control_strength,
1081+
&out_uncond);
1082+
}
10621083
negative_data = (float*)out_uncond->data;
10631084
}
10641085

1065-
int step_count = sigmas.size();
1066-
bool is_skiplayer_step = has_skiplayer && step > (int)(slg_params.skip_layer_start * step_count) && step < (int)(slg_params.skip_layer_end * step_count);
10671086
float* skip_layer_data = NULL;
1068-
if (is_skiplayer_step) {
1087+
if (is_skiplayer_step && slg_params.scale != 0.0) {
10691088
LOG_DEBUG("Skipping layers at step %d\n", step);
10701089
// skip layer (same as conditionned)
10711090
diffusion_model->compute(n_threads,
@@ -1153,7 +1172,7 @@ class StableDiffusionGGML {
11531172
latent_result = positive_data[i] + (cfg_scale - 1) * delta;
11541173
}
11551174
}
1156-
if (is_skiplayer_step) {
1175+
if (is_skiplayer_step && slg_params.scale != 0.0) {
11571176
latent_result = latent_result + (positive_data[i] - skip_layer_data[i]) * slg_params.scale;
11581177
}
11591178
// v = latent_result, eps = latent_result
@@ -1177,7 +1196,7 @@ class StableDiffusionGGML {
11771196
pretty_progress(step, (int)steps, (t1 - t0) / 1000000.f);
11781197
// LOG_INFO("step %d sampling completed taking %.2fs", step, (t1 - t0) * 1.0f / 1000000);
11791198
}
1180-
auto sd_preview_cb = sd_get_preview_callback();
1199+
auto sd_preview_cb = sd_get_preview_callback();
11811200
auto sd_preview_mode = sd_get_preview_mode();
11821201
if (sd_preview_cb != NULL) {
11831202
if (step % sd_get_preview_interval() == 0) {

‎stable-diffusion.h

+1
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,7 @@ typedef struct {
140140
float scale;
141141
float skip_layer_start;
142142
float skip_layer_end;
143+
bool slg_uncond;
143144
} sd_slg_params_t;
144145

145146
typedef void (*sd_log_cb_t)(enum sd_log_level_t level, const char* text, void* data);

0 commit comments

Comments
 (0)