Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

AYS, Align Your Steps, Scheduler Support #241

Merged
merged 5 commits into from
Apr 29, 2024
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
format code and avoid some warnings
leejet committed Apr 29, 2024
commit 37036d9acc7fc1f324721694350a2d9ce9a1d033
84 changes: 40 additions & 44 deletions denoiser.hpp
Original file line number Diff line number Diff line change
@@ -80,31 +80,30 @@ struct DiscreteSchedule : SigmaSchedule {
https://research.nvidia.com/labs/toronto-ai/AlignYourSteps/howto.html
*/
struct AYSSchedule : SigmaSchedule {
/* interp and linearInterp adapted from dpilger26's NumCpp library:
/* interp and linear_interp adapted from dpilger26's NumCpp library:
* https://github.com/dpilger26/NumCpp/tree/5e40aab74d14e257d65d3dc385c9ff9e2120c60e */
constexpr double interp(double left, double right, double perc) noexcept {
return (left * (1. - perc)) + (right * perc);
}

/* This will make the assumption that the reference x and y values are
* already sorted in ascending order because they are being generated as
/* This will make the assumption that the reference x and y values are
* already sorted in ascending order because they are being generated as
* such in the calling function */
std::vector<double> linearInterp(std::vector<float> new_x,
const std::vector<float> ref_x, const std::vector<float> ref_y)
{
std::vector<double> linear_interp(std::vector<float> new_x,
const std::vector<float> ref_x,
const std::vector<float> ref_y) {
const size_t len_x = new_x.size();
size_t i = 0;
size_t j = 0;
size_t i = 0;
size_t j = 0;
std::vector<double> new_y(len_x);

if (ref_x.size() != ref_y.size()) {
if (ref_x.size() != ref_y.size()) {
LOG_ERROR("Linear Interoplation Failed: length mismatch");
return new_y;
return new_y;
}

/* serves as the bounds checking for the below while loop */
if ((new_x[0] < ref_x[0])
|| (new_x[new_x.size() - 1] > ref_x[ref_x.size() - 1])) {
if ((new_x[0] < ref_x[0]) || (new_x[new_x.size() - 1] > ref_x[ref_x.size() - 1])) {
LOG_ERROR("Linear Interpolation Failed: bad bounds");
return new_y;
}
@@ -115,8 +114,7 @@ struct AYSSchedule : SigmaSchedule {
continue;
}

const double perc = static_cast<double>(new_x[i] - ref_x[j])
/ static_cast<double>(ref_x[j + 1] - ref_x[j]);
const double perc = static_cast<double>(new_x[i] - ref_x[j]) / static_cast<double>(ref_x[j + 1] - ref_x[j]);

new_y[i] = interp(ref_y[j], ref_y[j + 1], perc);
i++;
@@ -125,8 +123,7 @@ struct AYSSchedule : SigmaSchedule {
return new_y;
}

std::vector<float> linearSpace(const float start, const float end,
const size_t num_points) {
std::vector<float> linear_space(const float start, const float end, const size_t num_points) {
std::vector<float> result(num_points);
const float inc = (end - start) / (static_cast<float>(num_points - 1));

@@ -141,10 +138,10 @@ struct AYSSchedule : SigmaSchedule {
return result;
}

std::vector<float> logLinearInterpolation(std::vector<float> sigma_in,
const size_t new_len) {
std::vector<float> log_linear_interpolation(std::vector<float> sigma_in,
const size_t new_len) {
const size_t s_len = sigma_in.size();
std::vector<float> x_vals = linearSpace(0.f, 1.f, s_len);
std::vector<float> x_vals = linear_space(0.f, 1.f, s_len);
std::vector<float> y_vals(s_len);

/* Reverses the input array to be ascending instead of descending,
@@ -153,8 +150,8 @@ struct AYSSchedule : SigmaSchedule {
y_vals[i] = std::log(sigma_in[s_len - i - 1]);
}

std::vector<float> new_x_vals = linearSpace(0.f, 1.f, new_len);
std::vector<double> new_y_vals = linearInterp(new_x_vals, x_vals, y_vals);
std::vector<float> new_x_vals = linear_space(0.f, 1.f, new_len);
std::vector<double> new_y_vals = linear_interp(new_x_vals, x_vals, y_vals);
std::vector<float> results(new_len);

for (size_t i = 0; i < new_len; i++) {
@@ -164,56 +161,55 @@ struct AYSSchedule : SigmaSchedule {
return results;
}


std::vector<float> get_sigmas(uint32_t len) {
const std::vector<float> noise_levels[] = {
/* SD1.5 */
{14.6146412293, 6.4745760956, 3.8636745985, 2.6946151520,
1.8841921177, 1.3943805092, 0.9642583904, 0.6523686016,
0.3977456272, 0.1515232662, 0.0291671582},
{14.6146412293f, 6.4745760956f, 3.8636745985f, 2.6946151520f,
1.8841921177f, 1.3943805092f, 0.9642583904f, 0.6523686016f,
0.3977456272f, 0.1515232662f, 0.0291671582f},
/* SDXL */
{14.6146412293, 6.3184485287, 3.7681790315, 2.1811480769,
1.3405244945, 0.8620721141, 0.5550693289, 0.3798540708,
0.2332364134, 0.1114188177, 0.0291671582},
{14.6146412293f, 6.3184485287f, 3.7681790315f, 2.1811480769f,
1.3405244945f, 0.8620721141f, 0.5550693289f, 0.3798540708f,
0.2332364134f, 0.1114188177f, 0.0291671582f},
/* SVD */
{700.00, 54.5, 15.886, 7.977, 4.248, 1.789, 0.981, 0.403,
0.173, 0.034, 0.002},
{700.00f, 54.5f, 15.886f, 7.977f, 4.248f, 1.789f, 0.981f, 0.403f,
0.173f, 0.034f, 0.002f},
};

std::vector<float> inputs;
std::vector<float> results(len + 1);

switch (version) {
case VERSION_2_x: /* fallthrough */
LOG_WARN("AYS not designed for SD2.X models");
case VERSION_1_x:
LOG_INFO("AYS using SD1.5 noise levels");
LOG_WARN("AYS not designed for SD2.X models");
case VERSION_1_x:
LOG_INFO("AYS using SD1.5 noise levels");
inputs = noise_levels[0];
break;
case VERSION_XL:
LOG_INFO("AYS using SDXL noise levels");
LOG_INFO("AYS using SDXL noise levels");
inputs = noise_levels[1];
break;
case VERSION_SVD:
LOG_INFO("AYS using SVD noise levels");
LOG_INFO("AYS using SVD noise levels");
inputs = noise_levels[2];
break;
default:
LOG_ERROR("Version not compatable with AYS scheduler");
return results;
}

/* Stretches those pre-calculated reference levels out to the desired
* size using log-linear interpolation */
if ((len + 1) != inputs.size()) {
results = logLinearInterpolation(inputs, len + 1);
results = log_linear_interpolation(inputs, len + 1);
} else {
results = inputs;
}
results = inputs;
}

/* Not sure if this is strictly neccessary */
results[len] = 0.0f;

return results;
}
};
@@ -230,7 +226,7 @@ struct KarrasSchedule : SigmaSchedule {

float min_inv_rho = pow(sigma_min, (1.f / rho));
float max_inv_rho = pow(sigma_max, (1.f / rho));
for (uint32_t i = 0; i < n; i++) {
for (uint32_t i = 0; i < n; i++) {
// Eq. (5) from Karras et al 2022
result[i] = pow(max_inv_rho + (float)i / ((float)n - 1.f) * (min_inv_rho - max_inv_rho), rho);
}
1 change: 1 addition & 0 deletions model.cpp
Original file line number Diff line number Diff line change
@@ -890,6 +890,7 @@ bool ModelLoader::init_from_safetensors_file(const std::string& file_path, const

// ggml/src/ggml.c:2745
if (n_dims < 1 || n_dims > GGML_MAX_DIMS) {
LOG_ERROR("skip tensor '%s' with n_dims %d", name.c_str(), n_dims);
continue;
}

10 changes: 5 additions & 5 deletions stable-diffusion.cpp
Original file line number Diff line number Diff line change
@@ -450,11 +450,11 @@ class StableDiffusionGGML {
LOG_INFO("running with Karras schedule");
denoiser->schedule = std::make_shared<KarrasSchedule>();
break;
case AYS:
LOG_INFO("Running with Align-Your-Steps schedule");
denoiser->schedule = std::make_shared<AYSSchedule>();
denoiser->schedule->version = version;
break;
case AYS:
LOG_INFO("Running with Align-Your-Steps schedule");
denoiser->schedule = std::make_shared<AYSSchedule>();
denoiser->schedule->version = version;
break;
case DEFAULT:
// Don't touch anything.
break;