/root/doris/be/src/util/s3_uri.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | #include "util/s3_uri.h" |
19 | | |
20 | | #include <algorithm> |
21 | | #include <vector> |
22 | | |
23 | | #include "gutil/strings/split.h" |
24 | | #include "gutil/strings/strip.h" |
25 | | |
26 | | namespace doris { |
27 | | |
28 | | const std::string S3URI::_SCHEME_S3 = "s3"; |
29 | | const std::string S3URI::_SCHEME_HTTP = "http"; |
30 | | const std::string S3URI::_SCHEME_HTTPS = "https"; |
31 | | const std::string S3URI::_SCHEME_DELIM = "://"; |
32 | | const std::string S3URI::_PATH_DELIM = "/"; |
33 | | const std::string S3URI::_QUERY_DELIM = "?"; |
34 | | const std::string S3URI::_FRAGMENT_DELIM = "#"; |
35 | | |
36 | | /// eg: |
37 | | /// s3://bucket1/path/to/file.txt |
38 | | /// _schema: s3 |
39 | | /// _bucket: bucket1 |
40 | | /// _key: path/to/file.txt |
41 | 12 | Status S3URI::parse() { |
42 | 12 | if (_location.empty()) { |
43 | 0 | return Status::InvalidArgument("location is empty"); |
44 | 0 | } |
45 | 12 | std::vector<std::string> scheme_split = strings::Split(_location, _SCHEME_DELIM); |
46 | 12 | std::string rest; |
47 | 12 | if (scheme_split.size() == 2) { |
48 | 11 | if (scheme_split[0] == _SCHEME_S3) { |
49 | | // has scheme, eg: s3://bucket1/path/to/file.txt |
50 | 5 | rest = scheme_split[1]; |
51 | 5 | std::vector<std::string> authority_split = |
52 | 5 | strings::Split(rest, strings::delimiter::Limit(_PATH_DELIM, 1)); |
53 | 5 | if (authority_split.size() < 1) { |
54 | 0 | return Status::InvalidArgument("Invalid S3 URI: {}", _location); |
55 | 0 | } |
56 | 5 | _bucket = authority_split[0]; |
57 | | // support s3://bucket1 |
58 | 5 | _key = authority_split.size() == 1 ? "/" : authority_split[1]; |
59 | 6 | } else if (scheme_split[0] == _SCHEME_HTTP || scheme_split[0] == _SCHEME_HTTPS) { |
60 | | // has scheme, eg: http(s)://host/bucket1/path/to/file.txt |
61 | 5 | rest = scheme_split[1]; |
62 | 5 | std::vector<std::string> authority_split = |
63 | 5 | strings::Split(rest, strings::delimiter::Limit(_PATH_DELIM, 2)); |
64 | 5 | if (authority_split.size() != 3) { |
65 | 2 | return Status::InvalidArgument("Invalid S3 HTTP URI: {}", _location); |
66 | 2 | } |
67 | | // authority_split[1] is host |
68 | 3 | _bucket = authority_split[1]; |
69 | 3 | _key = authority_split[2]; |
70 | 3 | } else { |
71 | 1 | return Status::InvalidArgument("Invalid S3 URI: {}", _location); |
72 | 1 | } |
73 | 11 | } else if (scheme_split.size() == 1) { |
74 | | // no scheme, eg: path/to/file.txt |
75 | 1 | _bucket = ""; // unknown |
76 | 1 | _key = _location; |
77 | 1 | } else { |
78 | 0 | return Status::InvalidArgument("Invalid S3 URI: {}", _location); |
79 | 0 | } |
80 | 9 | StripWhiteSpace(&_key); |
81 | 9 | if (_key.empty()) { |
82 | 2 | return Status::InvalidArgument("Invalid S3 key: {}", _location); |
83 | 2 | } |
84 | | // Strip query and fragment if they exist |
85 | 7 | std::vector<std::string> _query_split = strings::Split(_key, _QUERY_DELIM); |
86 | 7 | std::vector<std::string> _fragment_split = strings::Split(_query_split[0], _FRAGMENT_DELIM); |
87 | 7 | _key = _fragment_split[0]; |
88 | 7 | return Status::OK(); |
89 | 9 | } |
90 | | |
91 | 0 | std::string S3URI::to_string() const { |
92 | 0 | return _location; |
93 | 0 | } |
94 | | |
95 | | } // end namespace doris |