Coverage Report

Created: 2024-11-21 15:53

/root/doris/be/src/util/url_parser.cpp
Line
Count
Source (jump to first uncovered line)
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#include "util/url_parser.h"
19
20
#include <ctype.h>
21
#include <stdint.h>
22
23
#include <algorithm>
24
#include <string>
25
26
#include "runtime/string_search.hpp"
27
#include "vec/common/string_ref.h"
28
29
namespace doris {
30
31
const StringRef UrlParser::_s_url_authority(const_cast<char*>("AUTHORITY"), 9);
32
const StringRef UrlParser::_s_url_file(const_cast<char*>("FILE"), 4);
33
const StringRef UrlParser::_s_url_host(const_cast<char*>("HOST"), 4);
34
const StringRef UrlParser::_s_url_path(const_cast<char*>("PATH"), 4);
35
const StringRef UrlParser::_s_url_protocol(const_cast<char*>("PROTOCOL"), 8);
36
const StringRef UrlParser::_s_url_query(const_cast<char*>("QUERY"), 5);
37
const StringRef UrlParser::_s_url_ref(const_cast<char*>("REF"), 3);
38
const StringRef UrlParser::_s_url_userinfo(const_cast<char*>("USERINFO"), 8);
39
const StringRef UrlParser::_s_url_port(const_cast<char*>("PORT"), 4);
40
const StringRef UrlParser::_s_protocol(const_cast<char*>("://"), 3);
41
const StringRef UrlParser::_s_at(const_cast<char*>("@"), 1);
42
const StringRef UrlParser::_s_slash(const_cast<char*>("/"), 1);
43
const StringRef UrlParser::_s_colon(const_cast<char*>(":"), 1);
44
const StringRef UrlParser::_s_question(const_cast<char*>("?"), 1);
45
const StringRef UrlParser::_s_hash(const_cast<char*>("#"), 1);
46
const StringSearch UrlParser::_s_protocol_search(&_s_protocol);
47
const StringSearch UrlParser::_s_at_search(&_s_at);
48
const StringSearch UrlParser::_s_slash_search(&_s_slash);
49
const StringSearch UrlParser::_s_colon_search(&_s_colon);
50
const StringSearch UrlParser::_s_question_search(&_s_question);
51
const StringSearch UrlParser::_s_hash_search(&_s_hash);
52
53
80
bool UrlParser::parse_url(const StringRef& url, UrlPart part, StringRef* result) {
54
80
    result->data = nullptr;
55
80
    result->size = 0;
56
    // Remove leading and trailing spaces.
57
80
    StringRef trimmed_url = url.trim();
58
59
    // All parts require checking for the _s_protocol.
60
80
    int32_t protocol_pos = _s_protocol_search.search(&trimmed_url);
61
80
    if (protocol_pos < 0) {
62
8
        return false;
63
8
    }
64
65
    // Positioned to first char after '://'.
66
72
    StringRef protocol_end = trimmed_url.substring(protocol_pos + _s_protocol.size);
67
68
72
    switch (part) {
69
8
    case AUTHORITY: {
70
        // Find first '/'.
71
8
        int32_t end_pos = _s_slash_search.search(&protocol_end);
72
8
        *result = protocol_end.substring(0, end_pos);
73
8
        break;
74
0
    }
75
76
8
    case FILE:
77
16
    case PATH: {
78
        // Find first '/'.
79
16
        int32_t start_pos = _s_slash_search.search(&protocol_end);
80
81
16
        if (start_pos < 0) {
82
            // Return empty string. This is what Hive does.
83
0
            return true;
84
0
        }
85
86
16
        StringRef path_start = protocol_end.substring(start_pos);
87
16
        int32_t end_pos;
88
89
16
        if (part == FILE) {
90
            // End _s_at '#'.
91
8
            end_pos = _s_hash_search.search(&path_start);
92
8
        } else {
93
            // End string _s_at next '?' or '#'.
94
8
            end_pos = _s_question_search.search(&path_start);
95
96
8
            if (end_pos < 0) {
97
                // No '?' was found, look for '#'.
98
8
                end_pos = _s_hash_search.search(&path_start);
99
8
            }
100
8
        }
101
102
16
        *result = path_start.substring(0, end_pos);
103
16
        break;
104
16
    }
105
106
12
    case HOST: {
107
        // Find '@'.
108
12
        int32_t start_pos = _s_at_search.search(&protocol_end);
109
110
12
        if (start_pos < 0) {
111
            // No '@' was found, i.e., no user:pass info was given, start after _s_protocol.
112
12
            start_pos = 0;
113
12
        } else {
114
            // Skip '@'.
115
0
            start_pos += _s_at.size;
116
0
        }
117
118
12
        StringRef host_start = protocol_end.substring(start_pos);
119
        // Find first '?'.
120
12
        int32_t query_start_pos = _s_question_search.search(&host_start);
121
12
        if (query_start_pos > 0) {
122
12
            host_start = host_start.substring(0, query_start_pos);
123
12
        }
124
        // Find ':' to strip out port.
125
12
        int32_t end_pos = _s_colon_search.search(&host_start);
126
127
12
        if (end_pos < 0) {
128
            // No port was given. search for '/' to determine ending position.
129
12
            end_pos = _s_slash_search.search(&host_start);
130
12
        }
131
132
12
        *result = host_start.substring(0, end_pos);
133
12
        break;
134
16
    }
135
136
8
    case PROTOCOL: {
137
8
        *result = trimmed_url.substring(0, protocol_pos);
138
8
        break;
139
16
    }
140
141
12
    case QUERY: {
142
        // Find first '?'.
143
12
        int32_t start_pos = _s_question_search.search(&protocol_end);
144
145
12
        if (start_pos < 0) {
146
            // Indicate no query was found.
147
0
            return false;
148
0
        }
149
150
12
        StringRef query_start = protocol_end.substring(start_pos + _s_question.size);
151
        // End string _s_at next '#'.
152
12
        int32_t end_pos = _s_hash_search.search(&query_start);
153
12
        *result = query_start.substring(0, end_pos);
154
12
        break;
155
12
    }
156
157
8
    case REF: {
158
        // Find '#'.
159
8
        int32_t start_pos = _s_hash_search.search(&protocol_end);
160
161
8
        if (start_pos < 0) {
162
            // Indicate no user and pass were given.
163
8
            return false;
164
8
        }
165
166
0
        *result = protocol_end.substring(start_pos + _s_hash.size);
167
0
        break;
168
8
    }
169
170
0
    case USERINFO: {
171
        // Find '@'.
172
0
        int32_t end_pos = _s_at_search.search(&protocol_end);
173
174
0
        if (end_pos < 0) {
175
            // Indicate no user and pass were given.
176
0
            return false;
177
0
        }
178
179
0
        *result = protocol_end.substring(0, end_pos);
180
0
        break;
181
0
    }
182
183
8
    case PORT: {
184
        // Find '@'.
185
8
        int32_t start_pos = _s_at_search.search(&protocol_end);
186
187
8
        if (start_pos < 0) {
188
            // No '@' was found, i.e., no user:pass info was given, start after _s_protocol.
189
8
            start_pos = 0;
190
8
        } else {
191
            // Skip '@'.
192
0
            start_pos += _s_at.size;
193
0
        }
194
195
8
        StringRef host_start = protocol_end.substring(start_pos);
196
        // Find ':' to strip out port.
197
8
        int32_t end_pos = _s_colon_search.search(&host_start);
198
        //no port found
199
8
        if (end_pos < 0) {
200
4
            return false;
201
4
        }
202
203
4
        StringRef port_start_str = protocol_end.substring(end_pos + _s_colon.size);
204
4
        int32_t port_end_pos = _s_slash_search.search(&port_start_str);
205
        //if '/' not found, try to find '?'
206
4
        if (port_end_pos < 0) {
207
0
            port_end_pos = _s_question_search.search(&port_start_str);
208
0
        }
209
4
        *result = port_start_str.substring(0, port_end_pos);
210
4
        break;
211
8
    }
212
213
0
    case INVALID:
214
0
        return false;
215
72
    }
216
217
60
    return true;
218
72
}
219
220
bool UrlParser::parse_url_key(const StringRef& url, UrlPart part, const StringRef& key,
221
32
                              StringRef* result) {
222
    // Part must be query to ask for a specific query key.
223
32
    if (part != QUERY) {
224
8
        return false;
225
8
    }
226
227
    // Remove leading and trailing spaces.
228
24
    StringRef trimmed_url = url.trim();
229
230
    // Search for the key in the url, ignoring malformed URLs for now.
231
24
    StringSearch key_search(&key);
232
233
24
    while (trimmed_url.size > 0) {
234
        // Search for the key in the current substring.
235
24
        int32_t key_pos = key_search.search(&trimmed_url);
236
24
        bool match = true;
237
238
24
        if (key_pos < 0) {
239
8
            return false;
240
8
        }
241
242
        // Key pos must be != 0 because it must be preceded by a '?' or a '&'.
243
        // Check that the char before key_pos is either '?' or '&'.
244
16
        if (key_pos == 0 ||
245
16
            (trimmed_url.data[key_pos - 1] != '?' && trimmed_url.data[key_pos - 1] != '&')) {
246
0
            match = false;
247
0
        }
248
249
        // Advance substring beyond matching key.
250
16
        trimmed_url = trimmed_url.substring(key_pos + key.size);
251
252
16
        if (!match) {
253
0
            continue;
254
0
        }
255
256
16
        if (trimmed_url.size <= 0) {
257
0
            break;
258
0
        }
259
260
        // Next character must be '=', otherwise the match cannot be a key in the query part.
261
16
        if (trimmed_url.data[0] != '=') {
262
0
            continue;
263
0
        }
264
265
16
        int32_t pos = 1;
266
267
        // Find ending position of key's value by matching '#' or '&'.
268
32
        while (pos < trimmed_url.size) {
269
32
            switch (trimmed_url.data[pos]) {
270
16
            case '#':
271
16
            case '&':
272
16
                *result = trimmed_url.substring(1, pos - 1);
273
16
                return true;
274
32
            }
275
276
16
            ++pos;
277
16
        }
278
279
        // Ending position is end of string.
280
0
        *result = trimmed_url.substring(1);
281
0
        return true;
282
16
    }
283
284
0
    return false;
285
24
}
286
287
112
UrlParser::UrlPart UrlParser::get_url_part(const StringRef& part) {
288
    // Quick filter on requested URL part, based on first character.
289
    // Hive requires the requested URL part to be all upper case.
290
112
    std::string part_str = part.to_string();
291
112
    transform(part_str.begin(), part_str.end(), part_str.begin(), ::toupper);
292
112
    StringRef newPart = StringRef(part_str);
293
112
    switch (newPart.data[0]) {
294
8
    case 'A': {
295
8
        if (!newPart.eq(_s_url_authority)) {
296
0
            return INVALID;
297
0
        }
298
299
8
        return AUTHORITY;
300
8
    }
301
302
8
    case 'F': {
303
8
        if (!newPart.eq(_s_url_file)) {
304
0
            return INVALID;
305
0
        }
306
307
8
        return FILE;
308
8
    }
309
310
28
    case 'H': {
311
28
        if (!newPart.eq(_s_url_host)) {
312
0
            return INVALID;
313
0
        }
314
315
28
        return HOST;
316
28
    }
317
318
24
    case 'P': {
319
24
        if (newPart.eq(_s_url_path)) {
320
8
            return PATH;
321
16
        } else if (newPart.eq(_s_url_protocol)) {
322
8
            return PROTOCOL;
323
8
        } else if (newPart.eq(_s_url_port)) {
324
8
            return PORT;
325
8
        } else {
326
0
            return INVALID;
327
0
        }
328
24
    }
329
330
36
    case 'Q': {
331
36
        if (!newPart.eq(_s_url_query)) {
332
0
            return INVALID;
333
0
        }
334
335
36
        return QUERY;
336
36
    }
337
338
8
    case 'R': {
339
8
        if (!newPart.eq(_s_url_ref)) {
340
0
            return INVALID;
341
0
        }
342
343
8
        return REF;
344
8
    }
345
346
0
    case 'U': {
347
0
        if (!newPart.eq(_s_url_userinfo)) {
348
0
            return INVALID;
349
0
        }
350
351
0
        return USERINFO;
352
0
    }
353
354
0
    default:
355
0
        return INVALID;
356
112
    }
357
112
}
358
359
40
StringRef UrlParser::extract_url(StringRef url, StringRef name) {
360
40
    StringRef result("", 0);
361
    // Remove leading and trailing spaces.
362
40
    StringRef trimmed_url = url.trim();
363
    // find '?'
364
40
    int32_t question_pos = _s_question_search.search(&trimmed_url);
365
40
    if (question_pos < 0) {
366
        // this url no parameters.
367
        // Example: https://doris.apache.org/
368
4
        return result;
369
4
    }
370
371
    // find '#'
372
36
    int32_t hash_pos = _s_hash_search.search(&trimmed_url);
373
36
    StringRef sub_url;
374
36
    if (hash_pos < 0) {
375
8
        sub_url = trimmed_url.substring(question_pos + 1, trimmed_url.size - question_pos - 1);
376
28
    } else {
377
28
        sub_url = trimmed_url.substring(question_pos + 1, hash_pos - question_pos - 1);
378
28
    }
379
380
    // find '&' and '=', and extract target parameter
381
    // Example: k1=aa&k2=bb&k3=cc&test=dd
382
36
    int64_t and_pod;
383
36
    auto len = sub_url.size;
384
36
    StringRef key_url;
385
80
    while (true) {
386
80
        if (len <= 0) {
387
16
            break;
388
16
        }
389
64
        and_pod = sub_url.find_first_of('&');
390
64
        if (and_pod != -1) {
391
40
            key_url = sub_url.substring(0, and_pod);
392
40
            sub_url = sub_url.substring(and_pod + 1, len - and_pod - 1);
393
40
        } else {
394
24
            auto end_pos = sub_url.find_first_of('#');
395
24
            key_url = end_pos == -1 ? sub_url : sub_url.substring(0, end_pos);
396
24
            sub_url = result;
397
24
        }
398
64
        len = sub_url.size;
399
400
64
        auto eq_pod = key_url.find_first_of('=');
401
64
        if (eq_pod == -1) {
402
            // invalid url. like: k1&k2=bb
403
4
            continue;
404
4
        }
405
60
        int32_t key_len = key_url.size;
406
60
        auto key = key_url.substring(0, eq_pod);
407
60
        if (name == key) {
408
20
            return key_url.substring(eq_pod + 1, key_len - eq_pod - 1);
409
20
        }
410
60
    }
411
16
    return result;
412
36
}
413
} // namespace doris