Coverage Report

Created: 2026-07-07 17:25

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
be/src/exprs/function/ai/ai_functions.cpp
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#include "core/column/column_array.h"
19
#include "exprs/function/ai/ai_classify.h"
20
#include "exprs/function/ai/ai_extract.h"
21
#include "exprs/function/ai/ai_filter.h"
22
#include "exprs/function/ai/ai_fix_grammar.h"
23
#include "exprs/function/ai/ai_generate.h"
24
#include "exprs/function/ai/ai_mask.h"
25
#include "exprs/function/ai/ai_sentiment.h"
26
#include "exprs/function/ai/ai_similarity.h"
27
#include "exprs/function/ai/ai_summarize.h"
28
#include "exprs/function/ai/ai_translate.h"
29
#include "exprs/function/ai/embed.h"
30
#include "exprs/function/simple_function_factory.h"
31
32
namespace doris {
33
Status FunctionAIClassify::build_prompt(const Block& block, const ColumnNumbers& arguments,
34
2
                                        size_t row_num, std::string& prompt) const {
35
    // Get the text column
36
2
    const ColumnWithTypeAndName& text_column = block.get_by_position(arguments[1]);
37
2
    StringRef text = text_column.column->get_data_at(row_num);
38
2
    std::string text_str = std::string(text.data, text.size);
39
40
    // Get the labels array column
41
2
    const ColumnWithTypeAndName& labels_column = block.get_by_position(arguments[2]);
42
2
    const auto& [array_column, array_row_num] =
43
2
            check_column_const_set_readability(*labels_column.column, row_num);
44
2
    const auto* col_array = check_and_get_column<ColumnArray>(*array_column);
45
2
    if (col_array == nullptr) {
46
0
        return Status::InternalError(
47
0
                "labels argument for {} must be Array(String) or Array(Varchar)", name);
48
0
    }
49
50
2
    std::vector<std::string> label_values;
51
2
    const auto& data = col_array->get_data();
52
2
    const auto& offsets = col_array->get_offsets();
53
2
    size_t start = array_row_num > 0 ? offsets[array_row_num - 1] : 0;
54
2
    size_t end = offsets[array_row_num];
55
8
    for (size_t i = start; i < end; ++i) {
56
6
        Field field;
57
6
        data.get(i, field);
58
6
        label_values.emplace_back(field.template get<TYPE_STRING>());
59
6
    }
60
61
2
    std::string labels_str = "[";
62
8
    for (size_t i = 0; i < label_values.size(); ++i) {
63
6
        if (i > 0) {
64
4
            labels_str += ", ";
65
4
        }
66
6
        labels_str += "\"" + label_values[i] + "\"";
67
6
    }
68
2
    labels_str += "]";
69
70
2
    prompt = "Labels: " + labels_str + "\nText: " + text_str;
71
72
2
    return Status::OK();
73
2
}
74
75
Status FunctionAIExtract::build_prompt(const Block& block, const ColumnNumbers& arguments,
76
2
                                       size_t row_num, std::string& prompt) const {
77
    // Get the text column
78
2
    const ColumnWithTypeAndName& text_column = block.get_by_position(arguments[1]);
79
2
    StringRef text = text_column.column->get_data_at(row_num);
80
2
    std::string text_str = std::string(text.data, text.size);
81
82
    // Get the labels array column
83
2
    const ColumnWithTypeAndName& labels_column = block.get_by_position(arguments[2]);
84
2
    const auto& [array_column, array_row_num] =
85
2
            check_column_const_set_readability(*labels_column.column, row_num);
86
2
    const auto* col_array = check_and_get_column<ColumnArray>(*array_column);
87
2
    if (col_array == nullptr) {
88
0
        return Status::InternalError(
89
0
                "labels argument for {} must be Array(String) or Array(Varchar)", name);
90
0
    }
91
92
2
    std::vector<std::string> label_values;
93
2
    const auto& offsets = col_array->get_offsets();
94
2
    const auto& data = col_array->get_data();
95
2
    size_t start = array_row_num > 0 ? offsets[array_row_num - 1] : 0;
96
2
    size_t end = offsets[array_row_num];
97
6
    for (size_t i = start; i < end; ++i) {
98
4
        Field field;
99
4
        data.get(i, field);
100
4
        label_values.emplace_back(field.template get<TYPE_STRING>());
101
4
    }
102
103
2
    std::string labels_str = "[";
104
6
    for (size_t i = 0; i < label_values.size(); ++i) {
105
4
        if (i > 0) {
106
2
            labels_str += ", ";
107
2
        }
108
4
        labels_str += "\"" + label_values[i] + "\"";
109
4
    }
110
2
    labels_str += "]";
111
112
2
    prompt = "Labels: " + labels_str + "\nText: " + text_str;
113
114
2
    return Status::OK();
115
2
}
116
117
Status FunctionAIGenerate::build_prompt(const Block& block, const ColumnNumbers& arguments,
118
2
                                        size_t row_num, std::string& prompt) const {
119
2
    const ColumnWithTypeAndName& text_column = block.get_by_position(arguments[1]);
120
2
    StringRef text_ref = text_column.column->get_data_at(row_num);
121
2
    prompt = std::string(text_ref.data, text_ref.size);
122
123
2
    return Status::OK();
124
2
}
125
126
Status FunctionAIMask::build_prompt(const Block& block, const ColumnNumbers& arguments,
127
2
                                    size_t row_num, std::string& prompt) const {
128
    // Get the text column
129
2
    const ColumnWithTypeAndName& text_column = block.get_by_position(arguments[1]);
130
2
    StringRef text = text_column.column->get_data_at(row_num);
131
2
    std::string text_str = std::string(text.data, text.size);
132
133
    // Get the labels array column
134
2
    const ColumnWithTypeAndName& labels_column = block.get_by_position(arguments[2]);
135
2
    const auto& [array_column, array_row_num] =
136
2
            check_column_const_set_readability(*labels_column.column, row_num);
137
2
    const auto* col_array = check_and_get_column<ColumnArray>(*array_column);
138
2
    if (col_array == nullptr) {
139
0
        return Status::InternalError(
140
0
                "labels argument for {} must be Array(String) or Array(Varchar)", name);
141
0
    }
142
143
2
    std::vector<std::string> label_values;
144
2
    const auto& offsets = col_array->get_offsets();
145
2
    const auto& data = col_array->get_data();
146
2
    size_t start = array_row_num > 0 ? offsets[array_row_num - 1] : 0;
147
2
    size_t end = offsets[array_row_num];
148
6
    for (size_t i = start; i < end; ++i) {
149
4
        Field field;
150
4
        data.get(i, field);
151
4
        label_values.emplace_back(field.template get<TYPE_STRING>());
152
4
    }
153
154
2
    std::string labels_str = "[";
155
6
    for (size_t i = 0; i < label_values.size(); ++i) {
156
4
        if (i > 0) {
157
2
            labels_str += ", ";
158
2
        }
159
4
        labels_str += "\"" + label_values[i] + "\"";
160
4
    }
161
2
    labels_str += "]";
162
163
2
    prompt = "Labels: " + labels_str + "\nText: " + text_str;
164
165
2
    return Status::OK();
166
2
}
167
168
Status FunctionAISimilarity::build_prompt(const Block& block, const ColumnNumbers& arguments,
169
48
                                          size_t row_num, std::string& prompt) const {
170
    // text1
171
48
    const ColumnWithTypeAndName& text_column_1 = block.get_by_position(arguments[1]);
172
48
    StringRef text_1 = text_column_1.column.get()->get_data_at(row_num);
173
48
    std::string text_str_1 = std::string(text_1.data, text_1.size);
174
175
    // text2
176
48
    const ColumnWithTypeAndName& text_column_2 = block.get_by_position(arguments[2]);
177
48
    StringRef text_2 = text_column_2.column.get()->get_data_at(row_num);
178
48
    std::string text_str_2 = std::string(text_2.data, text_2.size);
179
180
48
    prompt = "Text 1: " + text_str_1 + "\nText 2: " + text_str_2;
181
182
48
    return Status::OK();
183
48
}
184
185
Status FunctionAITranslate::build_prompt(const Block& block, const ColumnNumbers& arguments,
186
2
                                         size_t row_num, std::string& prompt) const {
187
    // text
188
2
    const ColumnWithTypeAndName& text_column = block.get_by_position(arguments[1]);
189
2
    StringRef text = text_column.column.get()->get_data_at(row_num);
190
2
    std::string text_str = std::string(text.data, text.size);
191
192
    // target language
193
2
    const ColumnWithTypeAndName& lang_column = block.get_by_position(arguments[2]);
194
2
    StringRef lang = lang_column.column.get()->get_data_at(row_num);
195
2
    std::string target_lang = std::string(lang.data, lang.size);
196
197
2
    prompt = "Translate the following text to " + target_lang + ".\nText: " + text_str;
198
199
2
    return Status::OK();
200
2
}
201
202
2
void register_function_ai(SimpleFunctionFactory& factory) {
203
2
    factory.register_function<FunctionEmbed>();
204
2
    factory.register_function<FunctionAIClassify>();
205
2
    factory.register_function<FunctionAIExtract>();
206
2
    factory.register_function<FunctionAIFilter>();
207
2
    factory.register_function<FunctionAIFixGrammar>();
208
2
    factory.register_function<FunctionAIGenerate>();
209
2
    factory.register_function<FunctionAIMask>();
210
2
    factory.register_function<FunctionAISentiment>();
211
2
    factory.register_function<FunctionAISimilarity>();
212
2
    factory.register_function<FunctionAISummarize>();
213
2
    factory.register_function<FunctionAITranslate>();
214
2
}
215
216
} // namespace doris