Skip to content

Commit e81efb0

Browse files
Nuzhny007Nuzhny007
committed
Update ruCLIP
1 parent b1dd7c2 commit e81efb0

4 files changed

Lines changed: 72 additions & 54 deletions

File tree

‎thirdparty/ruclip/RuCLIP.cpp‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -57,9 +57,9 @@ torch::Tensor TransformerImpl :: forward(const torch::Tensor& x)
5757

5858
void TransformerImpl :: InitializeParameters()
5959
{
60-
float proj_std = powf(Width, -0.5f) * pow(2 * Layers, -0.5f);
61-
float attn_std = powf(Width, -0.5f);
62-
float fc_std = powf(2 * Width, -0.5f);
60+
float proj_std = powf((float)Width, -0.5f) * powf(2.f * Layers, -0.5f);
61+
float attn_std = powf((float)Width, -0.5f);
62+
float fc_std = powf(2.f * Width, -0.5f);
6363

6464
for (int i = 0; i < Resblocks->size(); i++)
6565
{
@@ -90,7 +90,7 @@ VisionTransformerImpl :: VisionTransformerImpl(
9090
) : torch::nn::Module(module_name), InputResolution(input_resolution), OutputDim(output_dim)
9191
{
9292
Conv1 = torch::nn::Conv2d(torch::nn::Conv2dOptions(3, width, patch_size).stride(patch_size).bias(false));
93-
float scale = powf(width, -0.5);
93+
float scale = powf((float)width, -0.5);
9494
ClassEmbedding = scale * torch::randn(width);
9595
PositionalEmbedding = scale * torch::randn({ (int)pow(input_resolution / patch_size/*деление нацело*/, 2) + 1, width });
9696
LnPre = RCLayerNorm(std::vector<int64_t>() = { (int64_t)width });
@@ -220,4 +220,4 @@ torch::Tensor CLIPImpl :: forward(torch::Tensor input_ids, torch::Tensor pixel_v
220220
auto logits_per_text = logits_per_image.t();
221221

222222
return logits_per_image;
223-
}
223+
}

‎thirdparty/ruclip/RuCLIP.h‎

Lines changed: 17 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -165,31 +165,30 @@ inline CLIP FromPretrained(const std::filesystem::path &folder)
165165

166166
// Создание модели
167167
auto clip = CLIP("ruclip",
168-
int(config["embed_dim"]),
169-
int(config["image_resolution"]),
170-
int(config["vision_layers"]),
171-
int(config["vision_width"]),
172-
int(config["vision_patch_size"]),
173-
int(config["context_length"]),
174-
int(config["vocab_size"]),
175-
int(config["transformer_width"]),
176-
int(config["transformer_heads"]),
177-
int(config["transformer_layers"]));
178-
179-
for (auto &k : clip->named_parameters())
180-
std::cout << k.key() << std::endl;
168+
int(config["embed_dim"]),
169+
int(config["image_resolution"]),
170+
int(config["vision_layers"]),
171+
int(config["vision_width"]),
172+
int(config["vision_patch_size"]),
173+
int(config["context_length"]),
174+
int(config["vocab_size"]),
175+
int(config["transformer_width"]),
176+
int(config["transformer_heads"]),
177+
int(config["transformer_layers"]));
178+
179+
//for (auto &k : clip->named_parameters())
180+
// std::cout << k.key() << std::endl;
181181
//std::cout << "Model params count: " << Trainable::ParamsCount(clip) << std::endl;
182182

183183
// Загрузка состояния модели из файла
184-
try {
184+
try
185+
{
185186
torch::load(clip, (folder / "jit_model.zip").string());
186187
}
187-
catch (std::exception& e) {
188+
catch (std::exception& e)
189+
{
188190
std::cout << e.what() << std::endl;
189191
}
190192

191-
// "mean" : [0.48145466, 0.4578275, 0.40821073] ,
192-
// "std" : [0.26862954, 0.26130258, 0.27577711]
193-
194193
return clip;
195194
}

‎thirdparty/ruclip/RuCLIPProcessor.cpp‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#include "RuCLIPProcessor.h"
22

33
///
4-
torch::Tensor CVMatToTorchTensor(const cv::Mat img, const bool perm = true)
4+
inline torch::Tensor CVMatToTorchTensor(const cv::Mat img, const bool perm = true)
55
{
66
auto tensor_image = torch::from_blob(img.data, { img.rows, img.cols, img.channels() }, at::kByte);
77
if (perm)
@@ -12,7 +12,7 @@ torch::Tensor CVMatToTorchTensor(const cv::Mat img, const bool perm = true)
1212
}
1313

1414
///
15-
cv::Mat TorchTensorToCVMat(const torch::Tensor tensor_image, const bool perm = true)
15+
inline cv::Mat TorchTensorToCVMat(const torch::Tensor tensor_image, const bool perm = true)
1616
{
1717
auto t = tensor_image.detach().squeeze().cpu();
1818
if (perm)
@@ -33,11 +33,11 @@ RuCLIPProcessor :: RuCLIPProcessor(
3333
) : ImageSize(image_size), TextSeqLength(text_seq_length), NormMean(norm_mean), NormStd(norm_std)
3434
{
3535
vkcom::Status status;
36-
Tokenizer = new vkcom::BaseEncoder(tokenizer_path, -1, &status);
36+
Tokenizer = std::make_unique<vkcom::BaseEncoder>(tokenizer_path, -1, &status);
3737
}
3838

3939
///!!!Локали-юникоды
40-
torch::Tensor RuCLIPProcessor :: EncodeText(/*std::vector<*/std::string &text)
40+
torch::Tensor RuCLIPProcessor :: EncodeText(const/*std::vector<*/std::string &text)
4141
{
4242
std::vector<std::vector<int32_t>> ret_ids;
4343
vkcom::Status status;
@@ -61,6 +61,13 @@ torch::Tensor RuCLIPProcessor :: EncodeText(/*std::vector<*/std::string &text)
6161
return PrepareTokens(it);
6262
}
6363

64+
torch::Tensor RuCLIPProcessor::EncodeImage(const cv::Mat& img)
65+
{
66+
torch::Tensor img_tensor = CVMatToTorchTensor(img, true);
67+
img_tensor = torch::data::transforms::Normalize<>(NormMean, NormStd)(img_tensor);
68+
return img_tensor;
69+
}
70+
6471
torch::Tensor RuCLIPProcessor :: PrepareTokens(/*std::vector<*/std::vector<int32_t> tokens) //Передаю по значению чтобы внутри иметь дело с копией
6572
{
6673
torch::Tensor result;
@@ -107,7 +114,7 @@ std::pair<torch::Tensor, torch::Tensor> RuCLIPProcessor::operator()(const std::v
107114
//img_tensor.clone();
108115
images_tensors.push_back(img_tensor);
109116
}
110-
return std::make_pair(/*torch::pad_sequence*/torch::stack(texts_tensors), torch::pad_sequence(images_tensors).squeeze(0));
117+
return std::make_pair(!texts_tensors.empty()?/*torch::pad_sequence*/torch::stack(texts_tensors):torch::Tensor(), torch::pad_sequence(images_tensors).squeeze(0));
111118
}
112119

113120
///

‎thirdparty/ruclip/RuCLIPProcessor.h‎

Lines changed: 38 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,21 @@ class RuCLIPProcessor
1919
const std::vector<double> norm_std = { 0.26862954, 0.26130258, 0.27577711 });
2020

2121
///!!!Локали-юникоды
22-
torch::Tensor EncodeText(/*std::vector<*/std::string &text);
22+
torch::Tensor EncodeText(const /*std::vector<*/std::string &text);
2323
torch::Tensor PrepareTokens(/*std::vector<*/std::vector<int32_t> tokens); //Передаю по значению чтобы внутри иметь дело с копией
24+
torch::Tensor EncodeImage(const cv::Mat& img);
2425
std::pair <torch::Tensor, torch::Tensor> operator()(const std::vector <std::string>& texts, const std::vector <cv::Mat>& images);
2526
std::pair <torch::Tensor, torch::Tensor> operator()(const std::vector <cv::Mat>& images);
2627

2728
void CacheText(const std::vector <std::string>& texts);
2829

30+
///
31+
int GetImageSize() const noexcept
32+
{
33+
return ImageSize;
34+
}
35+
36+
///
2937
static RuCLIPProcessor FromPretrained(const std::filesystem::path &folder)
3038
{
3139
std::filesystem::path tokenizer_path = folder / "bpe.model";
@@ -34,13 +42,14 @@ class RuCLIPProcessor
3442
std::ifstream f(folder / "config.json");
3543
json config = json::parse(f);
3644

37-
return RuCLIPProcessor(
38-
tokenizer_path.string(),
39-
int(config["image_resolution"]),
40-
int(config["context_length"]),
41-
{ 0.48145466, 0.4578275, 0.40821073 }, //config.get("mean"),
42-
{ 0.26862954, 0.26130258, 0.27577711 } //config.get("std")
43-
);
45+
auto mean = config["mean"].template get<std::vector<double>>();
46+
auto std = config["std"].template get<std::vector<double>>();
47+
48+
return RuCLIPProcessor(tokenizer_path.string(),
49+
int(config["image_resolution"]),
50+
int(config["context_length"]),
51+
mean,
52+
std);
4453
}
4554

4655
private:
@@ -52,12 +61,12 @@ class RuCLIPProcessor
5261
const int TextSeqLength{ 77 };
5362
std::vector<double> NormMean;
5463
std::vector<double> NormStd;
55-
vkcom::BaseEncoder* Tokenizer = nullptr;
64+
std::unique_ptr<vkcom::BaseEncoder> Tokenizer;
5665

5766
std::vector<torch::Tensor> m_textsTensors;
5867
};
5968

60-
//relevancy for batch size == 1 at this moment, float lv = result.index({0,0}).item<float>();
69+
///relevancy for batch size == 1 at this moment, float lv = result.index({0,0}).item<float>();
6170
///
6271
///std::vector<torch::Tensor> canon_texts_tensors;
6372
///canon_texts_tensors.push_back(ClipProcessor->EncodeText(std::string("объект")));
@@ -75,21 +84,24 @@ class RuCLIPProcessor
7584
///float lv = rel.index({0,0}).item<float>();
7685
inline torch::Tensor Relevancy(torch::Tensor embeds, torch::Tensor positives, torch::Tensor negatives)
7786
{
78-
auto embeds2 = torch::cat({ positives, negatives });
79-
auto logits = /*scale * */torch::mm(embeds, embeds2.t()); // [batch_size x phrases]
80-
auto positive_vals = logits.index({ "...", torch::indexing::Slice(0, 1) }); // [batch_size x 1]
81-
auto negative_vals = logits.index({ "...", torch::indexing::Slice(1, torch::indexing::None) }); // [batch_size x negative_phrase_n]
82-
auto repeated_pos = positive_vals.repeat({ 1, negatives.sizes()[0] }); // [batch_size x negative_phrase_n]
83-
auto sims = torch::stack({ repeated_pos, negative_vals }, -1); // [batch_size x negative_phrase_n x 2]
84-
auto smx = torch::softmax(10 * sims, -1); // [batch_size x negative_phrase_n x 2]
85-
auto best_id = smx.index({ "...", 0 }).argmin(1); // [batch_size x 2]
86-
auto result = torch::gather(smx, 1, best_id.index({
87-
"...", torch::indexing::None, torch::indexing::None
88-
}).expand(
89-
{ best_id.sizes()[0], negatives.sizes()[0], 2
90-
})
91-
).index(
92-
{ torch::indexing::Slice(), 0, torch::indexing::Slice()
93-
});// [batch_size x 2]
87+
std::cout << "Relevancy: 0" << std::endl;
88+
auto embeds2 = torch::cat({positives, negatives});
89+
std::cout << "Relevancy: 1" << std::endl;
90+
auto logits = /*scale * */torch::mm(embeds, embeds2.t()); //[batch_size x phrases]
91+
std::cout << "Relevancy: 2" << std::endl;
92+
auto positive_vals = logits.index({"...", torch::indexing::Slice(0, positives.sizes()[0])}); // [batch_size x 1]
93+
std::cout << "Relevancy: 3" << std::endl;
94+
auto negative_vals = logits.index({"...", torch::indexing::Slice(positives.sizes()[0], torch::indexing::None)}); // [batch_size x negative_phrase_n]
95+
std::cout << "Relevancy: 4" << std::endl;
96+
auto repeated_pos = positive_vals.repeat({1, negatives.sizes()[0]}); //[batch_size x negative_phrase_n]
97+
std::cout << "Relevancy: 5: repeated_pos: " << repeated_pos.sizes() << ", negative_vals: " << negative_vals.sizes() << std::endl;
98+
auto sims = torch::stack({repeated_pos, negative_vals}, -1); //[batch_size x negative_phrase_n x 2]
99+
std::cout << "Relevancy: 6" << std::endl;
100+
auto smx = torch::softmax(10 * sims, -1); // [batch_size x negative_phrase_n x 2]
101+
std::cout << "Relevancy: 7" << std::endl;
102+
auto best_id = smx.index({"...", 0}).argmin(1); // [batch_size x 2]
103+
std::cout << "Relevancy: 8" << std::endl;
104+
auto result = torch::gather(smx, 1, best_id.index({"...", torch::indexing::None, torch::indexing::None}).expand({best_id.sizes()[0], negatives.sizes()[0], 2})
105+
).index({torch::indexing::Slice(), 0, torch::indexing::Slice()});// [batch_size x 2]
94106
return result;
95107
}

0 commit comments

Comments
 (0)