be/src/exprs/function/ai/ai_functions.cpp
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #include "core/column/column_array_view.h" |
19 | | #include "exprs/function/ai/ai_classify.h" |
20 | | #include "exprs/function/ai/ai_extract.h" |
21 | | #include "exprs/function/ai/ai_filter.h" |
22 | | #include "exprs/function/ai/ai_fix_grammar.h" |
23 | | #include "exprs/function/ai/ai_generate.h" |
24 | | #include "exprs/function/ai/ai_mask.h" |
25 | | #include "exprs/function/ai/ai_sentiment.h" |
26 | | #include "exprs/function/ai/ai_similarity.h" |
27 | | #include "exprs/function/ai/ai_summarize.h" |
28 | | #include "exprs/function/ai/ai_translate.h" |
29 | | #include "exprs/function/ai/embed.h" |
30 | | #include "exprs/function/simple_function_factory.h" |
31 | | |
32 | | namespace doris { |
33 | | static Status format_labels(const ColumnPtr& labels_column, size_t row_num, |
34 | 8 | std::string_view function_name, std::string& labels_str) { |
35 | 8 | auto readable_column = check_column_const_set_readability(*labels_column, row_num); |
36 | 8 | if (!is_column<ColumnArray>(*readable_column.first)) { |
37 | 0 | return Status::InternalError( |
38 | 0 | "labels argument for {} must be Array(String) or Array(Varchar)", function_name); |
39 | 0 | } |
40 | | |
41 | 8 | auto labels_view = ColumnArrayView<TYPE_STRING>::create(labels_column); |
42 | 8 | auto labels = labels_view[row_num]; |
43 | 8 | labels_str = "["; |
44 | 8 | bool is_first_label = true; |
45 | 29 | for (size_t i = 0; i < labels.size(); ++i) { |
46 | 21 | if (labels.is_null_at(i)) { |
47 | 4 | continue; |
48 | 4 | } |
49 | 17 | if (!is_first_label) { |
50 | 9 | labels_str += ", "; |
51 | 9 | } |
52 | 17 | StringRef label = labels.value_at(i); |
53 | 17 | labels_str += "\""; |
54 | 17 | labels_str.append(label.data, label.size); |
55 | 17 | labels_str += "\""; |
56 | 17 | is_first_label = false; |
57 | 17 | } |
58 | 8 | labels_str += "]"; |
59 | 8 | return Status::OK(); |
60 | 8 | } |
61 | | |
62 | | Status FunctionAIClassify::build_prompt(const Columns& prompt_columns, size_t row_num, |
63 | 4 | std::string& prompt) const { |
64 | | // Get the text column |
65 | 4 | StringRef text = prompt_columns[0]->get_data_at(row_num); |
66 | 4 | std::string text_str = std::string(text.data, text.size); |
67 | | |
68 | 4 | std::string labels_str; |
69 | 4 | RETURN_IF_ERROR(format_labels(prompt_columns[1], row_num, name, labels_str)); |
70 | | |
71 | 4 | prompt = "Labels: " + labels_str + "\nText: " + text_str; |
72 | | |
73 | 4 | return Status::OK(); |
74 | 4 | } |
75 | | |
76 | | Status FunctionAIExtract::build_prompt(const Columns& prompt_columns, size_t row_num, |
77 | 2 | std::string& prompt) const { |
78 | | // Get the text column |
79 | 2 | StringRef text = prompt_columns[0]->get_data_at(row_num); |
80 | 2 | std::string text_str = std::string(text.data, text.size); |
81 | | |
82 | 2 | std::string labels_str; |
83 | 2 | RETURN_IF_ERROR(format_labels(prompt_columns[1], row_num, name, labels_str)); |
84 | | |
85 | 2 | prompt = "Labels: " + labels_str + "\nText: " + text_str; |
86 | | |
87 | 2 | return Status::OK(); |
88 | 2 | } |
89 | | |
90 | | Status FunctionAIGenerate::build_prompt(const Columns& prompt_columns, size_t row_num, |
91 | 6 | std::string& prompt) const { |
92 | 6 | StringRef text_ref = prompt_columns[0]->get_data_at(row_num); |
93 | 6 | prompt = std::string(text_ref.data, text_ref.size); |
94 | | |
95 | 6 | return Status::OK(); |
96 | 6 | } |
97 | | |
98 | | Status FunctionAIMask::build_prompt(const Columns& prompt_columns, size_t row_num, |
99 | 2 | std::string& prompt) const { |
100 | | // Get the text column |
101 | 2 | StringRef text = prompt_columns[0]->get_data_at(row_num); |
102 | 2 | std::string text_str = std::string(text.data, text.size); |
103 | | |
104 | 2 | std::string labels_str; |
105 | 2 | RETURN_IF_ERROR(format_labels(prompt_columns[1], row_num, name, labels_str)); |
106 | | |
107 | 2 | prompt = "Labels: " + labels_str + "\nText: " + text_str; |
108 | | |
109 | 2 | return Status::OK(); |
110 | 2 | } |
111 | | |
112 | | Status FunctionAISimilarity::build_prompt(const Columns& prompt_columns, size_t row_num, |
113 | 26 | std::string& prompt) const { |
114 | | // text1 |
115 | 26 | StringRef text_1 = prompt_columns[0]->get_data_at(row_num); |
116 | 26 | std::string text_str_1 = std::string(text_1.data, text_1.size); |
117 | | |
118 | | // text2 |
119 | 26 | StringRef text_2 = prompt_columns[1]->get_data_at(row_num); |
120 | 26 | std::string text_str_2 = std::string(text_2.data, text_2.size); |
121 | | |
122 | 26 | prompt = "Text 1: " + text_str_1 + "\nText 2: " + text_str_2; |
123 | | |
124 | 26 | return Status::OK(); |
125 | 26 | } |
126 | | |
127 | | Status FunctionAITranslate::build_prompt(const Columns& prompt_columns, size_t row_num, |
128 | 1 | std::string& prompt) const { |
129 | | // text |
130 | 1 | StringRef text = prompt_columns[0]->get_data_at(row_num); |
131 | 1 | std::string text_str = std::string(text.data, text.size); |
132 | | |
133 | | // target language |
134 | 1 | StringRef lang = prompt_columns[1]->get_data_at(row_num); |
135 | 1 | std::string target_lang = std::string(lang.data, lang.size); |
136 | | |
137 | 1 | prompt = "Translate the following text to " + target_lang + ".\nText: " + text_str; |
138 | | |
139 | 1 | return Status::OK(); |
140 | 1 | } |
141 | | |
142 | 1 | void register_function_ai(SimpleFunctionFactory& factory) { |
143 | 1 | factory.register_function<FunctionEmbed>(); |
144 | 1 | factory.register_function<FunctionAIClassify>(); |
145 | 1 | factory.register_function<FunctionAIExtract>(); |
146 | 1 | factory.register_function<FunctionAIFilter>(); |
147 | 1 | factory.register_function<FunctionAIFixGrammar>(); |
148 | 1 | factory.register_function<FunctionAIGenerate>(); |
149 | 1 | factory.register_function<FunctionAIMask>(); |
150 | 1 | factory.register_function<FunctionAISentiment>(); |
151 | 1 | factory.register_function<FunctionAISimilarity>(); |
152 | 1 | factory.register_function<FunctionAISummarize>(); |
153 | 1 | factory.register_function<FunctionAITranslate>(); |
154 | 1 | } |
155 | | |
156 | | } // namespace doris |