Coverage Report

Created: 2025-07-25 20:20

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/root/doris/be/src/olap/olap_common.h
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
#pragma once
19
20
#include <gen_cpp/Types_types.h>
21
#include <netinet/in.h>
22
23
#include <atomic>
24
#include <charconv>
25
#include <cstdint>
26
#include <functional>
27
#include <list>
28
#include <map>
29
#include <memory>
30
#include <mutex>
31
#include <ostream>
32
#include <sstream>
33
#include <string>
34
#include <typeinfo>
35
#include <unordered_map>
36
#include <unordered_set>
37
#include <utility>
38
39
#include "common/cast_set.h"
40
#include "common/config.h"
41
#include "common/exception.h"
42
#include "io/io_common.h"
43
#include "olap/inverted_index_stats.h"
44
#include "olap/olap_define.h"
45
#include "olap/rowset/rowset_fwd.h"
46
#include "util/countdown_latch.h"
47
#include "util/hash_util.hpp"
48
#include "util/time.h"
49
#include "util/uid_util.h"
50
51
namespace doris {
52
#include "common/compile_check_begin.h"
53
static constexpr int64_t MAX_ROWSET_ID = 1L << 56;
54
static constexpr int64_t LOW_56_BITS = 0x00ffffffffffffff;
55
56
using SchemaHash = int32_t;
57
using int128_t = __int128;
58
using uint128_t = unsigned __int128;
59
60
using TabletUid = UniqueId;
61
62
enum CompactionType { BASE_COMPACTION = 1, CUMULATIVE_COMPACTION = 2, FULL_COMPACTION = 3 };
63
64
enum DataDirType {
65
    SPILL_DISK_DIR,
66
    OLAP_DATA_DIR,
67
    DATA_CACHE_DIR,
68
};
69
70
struct DataDirInfo {
71
    std::string path;
72
    size_t path_hash = 0;
73
    int64_t disk_capacity = 1; // actual disk capacity
74
    int64_t available = 0;     // available space, in bytes unit
75
    int64_t local_used_capacity = 0;
76
    int64_t remote_used_capacity = 0;
77
    int64_t trash_used_capacity = 0;
78
    bool is_used = false;                                      // whether available mark
79
    TStorageMedium::type storage_medium = TStorageMedium::HDD; // Storage medium type: SSD|HDD
80
    DataDirType data_dir_type = DataDirType::OLAP_DATA_DIR;
81
    std::string metric_name;
82
};
83
84
// Sort DataDirInfo by available space.
85
struct DataDirInfoLessAvailability {
86
7
    bool operator()(const DataDirInfo& left, const DataDirInfo& right) const {
87
7
        return left.available < right.available;
88
7
    }
89
};
90
91
struct TabletInfo {
92
    TabletInfo(TTabletId in_tablet_id, UniqueId in_uid)
93
0
            : tablet_id(in_tablet_id), tablet_uid(in_uid) {}
94
95
0
    bool operator<(const TabletInfo& right) const {
96
0
        if (tablet_id != right.tablet_id) {
97
0
            return tablet_id < right.tablet_id;
98
0
        } else {
99
0
            return tablet_uid < right.tablet_uid;
100
0
        }
101
0
    }
102
103
0
    std::string to_string() const {
104
0
        std::stringstream ss;
105
0
        ss << tablet_id << "." << tablet_uid.to_string();
106
0
        return ss.str();
107
0
    }
108
109
    TTabletId tablet_id;
110
    UniqueId tablet_uid;
111
};
112
113
struct TabletSize {
114
    TabletSize(TTabletId in_tablet_id, size_t in_tablet_size)
115
0
            : tablet_id(in_tablet_id), tablet_size(in_tablet_size) {}
116
117
    TTabletId tablet_id;
118
    size_t tablet_size;
119
};
120
121
// Define all data types supported by Field.
122
// If new filed_type is defined, not only new TypeInfo may need be defined,
123
// but also some functions like get_type_info in types.cpp need to be changed.
124
enum class FieldType {
125
    OLAP_FIELD_TYPE_TINYINT = 1, // MYSQL_TYPE_TINY
126
    OLAP_FIELD_TYPE_UNSIGNED_TINYINT = 2,
127
    OLAP_FIELD_TYPE_SMALLINT = 3, // MYSQL_TYPE_SHORT
128
    OLAP_FIELD_TYPE_UNSIGNED_SMALLINT = 4,
129
    OLAP_FIELD_TYPE_INT = 5, // MYSQL_TYPE_LONG
130
    OLAP_FIELD_TYPE_UNSIGNED_INT = 6,
131
    OLAP_FIELD_TYPE_BIGINT = 7, // MYSQL_TYPE_LONGLONG
132
    OLAP_FIELD_TYPE_UNSIGNED_BIGINT = 8,
133
    OLAP_FIELD_TYPE_LARGEINT = 9,
134
    OLAP_FIELD_TYPE_FLOAT = 10,  // MYSQL_TYPE_FLOAT
135
    OLAP_FIELD_TYPE_DOUBLE = 11, // MYSQL_TYPE_DOUBLE
136
    OLAP_FIELD_TYPE_DISCRETE_DOUBLE = 12,
137
    OLAP_FIELD_TYPE_CHAR = 13,     // MYSQL_TYPE_STRING
138
    OLAP_FIELD_TYPE_DATE = 14,     // MySQL_TYPE_NEWDATE
139
    OLAP_FIELD_TYPE_DATETIME = 15, // MySQL_TYPE_DATETIME
140
    OLAP_FIELD_TYPE_DECIMAL = 16,  // DECIMAL, using different store format against MySQL
141
    OLAP_FIELD_TYPE_VARCHAR = 17,
142
143
    OLAP_FIELD_TYPE_STRUCT = 18,  // Struct
144
    OLAP_FIELD_TYPE_ARRAY = 19,   // ARRAY
145
    OLAP_FIELD_TYPE_MAP = 20,     // Map
146
    OLAP_FIELD_TYPE_UNKNOWN = 21, // UNKNOW OLAP_FIELD_TYPE_STRING
147
    OLAP_FIELD_TYPE_NONE = 22,
148
    OLAP_FIELD_TYPE_HLL = 23,
149
    OLAP_FIELD_TYPE_BOOL = 24,
150
    OLAP_FIELD_TYPE_BITMAP = 25,
151
    OLAP_FIELD_TYPE_STRING = 26,
152
    OLAP_FIELD_TYPE_QUANTILE_STATE = 27,
153
    OLAP_FIELD_TYPE_DATEV2 = 28,
154
    OLAP_FIELD_TYPE_DATETIMEV2 = 29,
155
    OLAP_FIELD_TYPE_TIMEV2 = 30,
156
    OLAP_FIELD_TYPE_DECIMAL32 = 31,
157
    OLAP_FIELD_TYPE_DECIMAL64 = 32,
158
    OLAP_FIELD_TYPE_DECIMAL128I = 33,
159
    OLAP_FIELD_TYPE_JSONB = 34,
160
    OLAP_FIELD_TYPE_VARIANT = 35,
161
    OLAP_FIELD_TYPE_AGG_STATE = 36,
162
    OLAP_FIELD_TYPE_DECIMAL256 = 37,
163
    OLAP_FIELD_TYPE_IPV4 = 38,
164
    OLAP_FIELD_TYPE_IPV6 = 39,
165
};
166
167
// Define all aggregation methods supported by Field
168
// Note that in practice, not all types can use all the following aggregation methods
169
// For example, it is meaningless to use SUM for the string type (but it will not cause the program to crash)
170
// The implementation of the Field class does not perform such checks, and should be constrained when creating the table
171
enum class FieldAggregationMethod {
172
    OLAP_FIELD_AGGREGATION_NONE = 0,
173
    OLAP_FIELD_AGGREGATION_SUM = 1,
174
    OLAP_FIELD_AGGREGATION_MIN = 2,
175
    OLAP_FIELD_AGGREGATION_MAX = 3,
176
    OLAP_FIELD_AGGREGATION_REPLACE = 4,
177
    OLAP_FIELD_AGGREGATION_HLL_UNION = 5,
178
    OLAP_FIELD_AGGREGATION_UNKNOWN = 6,
179
    OLAP_FIELD_AGGREGATION_BITMAP_UNION = 7,
180
    // Replace if and only if added value is not null
181
    OLAP_FIELD_AGGREGATION_REPLACE_IF_NOT_NULL = 8,
182
    OLAP_FIELD_AGGREGATION_QUANTILE_UNION = 9,
183
    OLAP_FIELD_AGGREGATION_GENERIC = 10
184
};
185
186
enum class PushType {
187
    PUSH_NORMAL = 1,          // for broker/hadoop load, not used any more
188
    PUSH_FOR_DELETE = 2,      // for delete
189
    PUSH_FOR_LOAD_DELETE = 3, // not used any more
190
    PUSH_NORMAL_V2 = 4,       // for spark load
191
};
192
193
0
constexpr bool field_is_slice_type(const FieldType& field_type) {
194
0
    return field_type == FieldType::OLAP_FIELD_TYPE_VARCHAR ||
195
0
           field_type == FieldType::OLAP_FIELD_TYPE_CHAR ||
196
0
           field_type == FieldType::OLAP_FIELD_TYPE_STRING;
197
0
}
198
199
0
constexpr bool field_is_numeric_type(const FieldType& field_type) {
200
0
    return field_type == FieldType::OLAP_FIELD_TYPE_INT ||
201
0
           field_type == FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT ||
202
0
           field_type == FieldType::OLAP_FIELD_TYPE_BIGINT ||
203
0
           field_type == FieldType::OLAP_FIELD_TYPE_SMALLINT ||
204
0
           field_type == FieldType::OLAP_FIELD_TYPE_UNSIGNED_TINYINT ||
205
0
           field_type == FieldType::OLAP_FIELD_TYPE_UNSIGNED_SMALLINT ||
206
0
           field_type == FieldType::OLAP_FIELD_TYPE_TINYINT ||
207
0
           field_type == FieldType::OLAP_FIELD_TYPE_DOUBLE ||
208
0
           field_type == FieldType::OLAP_FIELD_TYPE_FLOAT ||
209
0
           field_type == FieldType::OLAP_FIELD_TYPE_DATE ||
210
0
           field_type == FieldType::OLAP_FIELD_TYPE_DATEV2 ||
211
0
           field_type == FieldType::OLAP_FIELD_TYPE_DATETIME ||
212
0
           field_type == FieldType::OLAP_FIELD_TYPE_DATETIMEV2 ||
213
0
           field_type == FieldType::OLAP_FIELD_TYPE_LARGEINT ||
214
0
           field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL ||
215
0
           field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL32 ||
216
0
           field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL64 ||
217
0
           field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL128I ||
218
0
           field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL256 ||
219
0
           field_type == FieldType::OLAP_FIELD_TYPE_BOOL ||
220
0
           field_type == FieldType::OLAP_FIELD_TYPE_IPV4 ||
221
0
           field_type == FieldType::OLAP_FIELD_TYPE_IPV6;
222
0
}
223
224
// <start_version_id, end_version_id>, such as <100, 110>
225
//using Version = std::pair<TupleVersion, TupleVersion>;
226
227
struct Version {
228
    int64_t first;
229
    int64_t second;
230
231
1.82M
    Version(int64_t first_, int64_t second_) : first(first_), second(second_) {}
232
8.04k
    Version() : first(0), second(0) {}
233
234
0
    static Version mock() {
235
0
        // Every time SchemaChange is used for external rowing, some temporary versions (such as 999, 1000, 1001) will be written, in order to avoid Cache conflicts, temporary
236
0
        // The version number takes a BIG NUMBER plus the version number of the current SchemaChange
237
0
        return Version(1 << 28, 1 << 29);
238
0
    }
239
240
    friend std::ostream& operator<<(std::ostream& os, const Version& version);
241
242
0
    bool operator!=(const Version& rhs) const { return first != rhs.first || second != rhs.second; }
243
244
358k
    bool operator==(const Version& rhs) const { return first == rhs.first && second == rhs.second; }
245
246
0
    bool contains(const Version& other) const {
247
0
        return first <= other.first && second >= other.second;
248
0
    }
249
250
956
    std::string to_string() const { return fmt::format("[{}-{}]", first, second); }
251
};
252
253
using Versions = std::vector<Version>;
254
255
14
inline std::ostream& operator<<(std::ostream& os, const Version& version) {
256
14
    return os << version.to_string();
257
14
}
258
259
0
inline std::ostream& operator<<(std::ostream& os, const Versions& versions) {
260
0
    for (auto& version : versions) {
261
0
        os << version;
262
0
    }
263
0
    return os;
264
0
}
265
266
// used for hash-struct of hash_map<Version, Rowset*>.
267
struct HashOfVersion {
268
11.9k
    size_t operator()(const Version& version) const {
269
11.9k
        size_t seed = 0;
270
11.9k
        seed = HashUtil::hash64(&version.first, sizeof(version.first), seed);
271
11.9k
        seed = HashUtil::hash64(&version.second, sizeof(version.second), seed);
272
11.9k
        return seed;
273
11.9k
    }
274
};
275
276
// It is used to represent Graph vertex.
277
struct Vertex {
278
    int64_t value = 0;
279
    std::list<int64_t> edges;
280
281
0
    Vertex(int64_t v) : value(v) {}
282
};
283
284
class Field;
285
class WrapperField;
286
using KeyRange = std::pair<WrapperField*, WrapperField*>;
287
288
// ReaderStatistics used to collect statistics when scan data from storage
289
struct OlapReaderStatistics {
290
    int64_t io_ns = 0;
291
    int64_t compressed_bytes_read = 0;
292
293
    int64_t decompress_ns = 0;
294
    int64_t uncompressed_bytes_read = 0;
295
296
    // total read bytes in memory
297
    int64_t bytes_read = 0;
298
299
    int64_t block_fetch_ns = 0; // time of rowset reader's `next_batch()` call
300
    int64_t block_load_ns = 0;
301
    int64_t blocks_load = 0;
302
    // Not used any more, will be removed after non-vectorized code is removed
303
    int64_t block_seek_num = 0;
304
    // Not used any more, will be removed after non-vectorized code is removed
305
    int64_t block_seek_ns = 0;
306
307
    // block_load_ns
308
    //      block_init_ns
309
    //          block_init_seek_ns
310
    //          generate_row_ranges_ns
311
    //      predicate_column_read_ns
312
    //          predicate_column_read_seek_ns
313
    //      lazy_read_ns
314
    //          block_lazy_read_seek_ns
315
    int64_t block_init_ns = 0;
316
    int64_t block_init_seek_num = 0;
317
    int64_t block_init_seek_ns = 0;
318
    int64_t predicate_column_read_ns = 0;
319
    int64_t non_predicate_read_ns = 0;
320
    int64_t predicate_column_read_seek_num = 0;
321
    int64_t predicate_column_read_seek_ns = 0;
322
    int64_t lazy_read_ns = 0;
323
    int64_t block_lazy_read_seek_num = 0;
324
    int64_t block_lazy_read_seek_ns = 0;
325
326
    int64_t raw_rows_read = 0;
327
328
    int64_t rows_vec_cond_filtered = 0;
329
    int64_t rows_short_circuit_cond_filtered = 0;
330
    int64_t rows_expr_cond_filtered = 0;
331
    int64_t vec_cond_input_rows = 0;
332
    int64_t short_circuit_cond_input_rows = 0;
333
    int64_t expr_cond_input_rows = 0;
334
    int64_t rows_vec_del_cond_filtered = 0;
335
    int64_t vec_cond_ns = 0;
336
    int64_t short_cond_ns = 0;
337
    int64_t expr_filter_ns = 0;
338
    int64_t output_col_ns = 0;
339
    int64_t rows_key_range_filtered = 0;
340
    int64_t rows_stats_filtered = 0;
341
    int64_t rows_stats_rp_filtered = 0;
342
    int64_t rows_bf_filtered = 0;
343
    int64_t rows_dict_filtered = 0;
344
    // Including the number of rows filtered out according to the Delete information in the Tablet,
345
    // and the number of rows filtered for marked deleted rows under the unique key model.
346
    // This metric is mainly used to record the number of rows filtered by the delete condition in Segment V1,
347
    // and it is also used to record the replaced rows in the Unique key model in the "Reader" class.
348
    // In segmentv2, if you want to get all filtered rows, you need the sum of "rows_del_filtered" and "rows_conditions_filtered".
349
    int64_t rows_del_filtered = 0;
350
    int64_t rows_del_by_bitmap = 0;
351
    // the number of rows filtered by various column indexes.
352
    int64_t rows_conditions_filtered = 0;
353
    int64_t generate_row_ranges_by_keys_ns = 0;
354
    int64_t generate_row_ranges_by_column_conditions_ns = 0;
355
    int64_t generate_row_ranges_by_bf_ns = 0;
356
    int64_t generate_row_ranges_by_zonemap_ns = 0;
357
    int64_t generate_row_ranges_by_dict_ns = 0;
358
359
    int64_t index_load_ns = 0;
360
361
    int64_t total_pages_num = 0;
362
    int64_t cached_pages_num = 0;
363
364
    int64_t rows_bitmap_index_filtered = 0;
365
    int64_t bitmap_index_filter_timer = 0;
366
367
    int64_t rows_inverted_index_filtered = 0;
368
    int64_t inverted_index_filter_timer = 0;
369
    int64_t inverted_index_query_timer = 0;
370
    int64_t inverted_index_query_cache_hit = 0;
371
    int64_t inverted_index_query_cache_miss = 0;
372
    int64_t inverted_index_query_null_bitmap_timer = 0;
373
    int64_t inverted_index_query_bitmap_copy_timer = 0;
374
    int64_t inverted_index_searcher_open_timer = 0;
375
    int64_t inverted_index_searcher_search_timer = 0;
376
    int64_t inverted_index_searcher_search_init_timer = 0;
377
    int64_t inverted_index_searcher_search_exec_timer = 0;
378
    int64_t inverted_index_searcher_cache_hit = 0;
379
    int64_t inverted_index_searcher_cache_miss = 0;
380
    int64_t inverted_index_downgrade_count = 0;
381
    InvertedIndexStatistics inverted_index_stats;
382
383
    int64_t output_index_result_column_timer = 0;
384
    // number of segment filtered by column stat when creating seg iterator
385
    int64_t filtered_segment_number = 0;
386
    // total number of segment
387
    int64_t total_segment_number = 0;
388
389
    io::FileCacheStatistics file_cache_stats;
390
    int64_t load_segments_timer = 0;
391
392
    int64_t collect_iterator_merge_next_timer = 0;
393
    int64_t collect_iterator_normal_next_timer = 0;
394
    int64_t delete_bitmap_get_agg_ns = 0;
395
396
    int64_t tablet_reader_init_timer_ns = 0;
397
    int64_t tablet_reader_capture_rs_readers_timer_ns = 0;
398
    int64_t tablet_reader_init_return_columns_timer_ns = 0;
399
    int64_t tablet_reader_init_keys_param_timer_ns = 0;
400
    int64_t tablet_reader_init_orderby_keys_param_timer_ns = 0;
401
    int64_t tablet_reader_init_conditions_param_timer_ns = 0;
402
    int64_t tablet_reader_init_delete_condition_param_timer_ns = 0;
403
    int64_t block_reader_vcollect_iter_init_timer_ns = 0;
404
    int64_t block_reader_rs_readers_init_timer_ns = 0;
405
    int64_t block_reader_build_heap_init_timer_ns = 0;
406
407
    int64_t rowset_reader_get_segment_iterators_timer_ns = 0;
408
    int64_t rowset_reader_create_iterators_timer_ns = 0;
409
    int64_t rowset_reader_init_iterators_timer_ns = 0;
410
    int64_t rowset_reader_load_segments_timer_ns = 0;
411
412
    int64_t segment_iterator_init_timer_ns = 0;
413
    int64_t segment_iterator_init_return_column_iterators_timer_ns = 0;
414
    int64_t segment_iterator_init_bitmap_index_iterators_timer_ns = 0;
415
    int64_t segment_iterator_init_index_iterators_timer_ns = 0;
416
417
    int64_t segment_create_column_readers_timer_ns = 0;
418
    int64_t segment_load_index_timer_ns = 0;
419
};
420
421
using ColumnId = uint32_t;
422
// Column unique id set
423
using UniqueIdSet = std::set<uint32_t>;
424
// Column unique Id -> column id map
425
using UniqueIdToColumnIdMap = std::map<ColumnId, ColumnId>;
426
427
// 8 bit rowset id version
428
// 56 bit, inc number from 1
429
// 128 bit backend uid, it is a uuid bit, id version
430
struct RowsetId {
431
    int8_t version = 0;
432
    int64_t hi = 0;
433
    int64_t mi = 0;
434
    int64_t lo = 0;
435
436
11.4k
    void init(std::string_view rowset_id_str) {
437
        // for new rowsetid its a 48 hex string
438
        // if the len < 48, then it is an old format rowset id
439
11.4k
        if (rowset_id_str.length() < 48) [[unlikely]] {
440
69
            int64_t high;
441
69
            auto [_, ec] = std::from_chars(rowset_id_str.data(),
442
69
                                           rowset_id_str.data() + rowset_id_str.length(), high);
443
69
            if (ec != std::errc {}) [[unlikely]] {
444
1
                if (config::force_regenerate_rowsetid_on_start_error) {
445
1
                    LOG(WARNING) << "failed to init rowset id: " << rowset_id_str;
446
1
                    high = MAX_ROWSET_ID - 1;
447
1
                } else {
448
0
                    throw Exception(
449
0
                            Status::FatalError("failed to init rowset id: {}", rowset_id_str));
450
0
                }
451
1
            }
452
69
            init(1, high, 0, 0);
453
11.3k
        } else {
454
11.3k
            int64_t high = 0;
455
11.3k
            int64_t middle = 0;
456
11.3k
            int64_t low = 0;
457
11.3k
            from_hex(&high, rowset_id_str.substr(0, 16));
458
11.3k
            from_hex(&middle, rowset_id_str.substr(16, 16));
459
11.3k
            from_hex(&low, rowset_id_str.substr(32, 16));
460
11.3k
            init(high >> 56, high & LOW_56_BITS, middle, low);
461
11.3k
        }
462
11.4k
    }
463
464
    // to compatible with old version
465
1.12k
    void init(int64_t rowset_id) { init(1, rowset_id, 0, 0); }
466
467
28.9k
    void init(int64_t id_version, int64_t high, int64_t middle, int64_t low) {
468
28.9k
        version = cast_set<int8_t>(id_version);
469
28.9k
        if (UNLIKELY(high >= MAX_ROWSET_ID)) {
470
0
            throw Exception(Status::FatalError("inc rowsetid is too large:{}", high));
471
0
        }
472
28.9k
        hi = (id_version << 56) + (high & LOW_56_BITS);
473
28.9k
        mi = middle;
474
28.9k
        lo = low;
475
28.9k
    }
476
477
46.8k
    std::string to_string() const {
478
46.8k
        if (version < 2) {
479
26.0k
            return std::to_string(hi & LOW_56_BITS);
480
26.0k
        } else {
481
20.7k
            char buf[48];
482
20.7k
            to_hex(hi, buf);
483
20.7k
            to_hex(mi, buf + 16);
484
20.7k
            to_hex(lo, buf + 32);
485
20.7k
            return {buf, 48};
486
20.7k
        }
487
46.8k
    }
488
489
    // std::unordered_map need this api
490
357k
    bool operator==(const RowsetId& rhs) const {
491
357k
        return hi == rhs.hi && mi == rhs.mi && lo == rhs.lo;
492
357k
    }
493
494
0
    bool operator!=(const RowsetId& rhs) const {
495
0
        return hi != rhs.hi || mi != rhs.mi || lo != rhs.lo;
496
0
    }
497
498
44.8M
    bool operator<(const RowsetId& rhs) const {
499
44.8M
        if (hi != rhs.hi) {
500
4.56M
            return hi < rhs.hi;
501
40.3M
        } else if (mi != rhs.mi) {
502
0
            return mi < rhs.mi;
503
40.3M
        } else {
504
40.3M
            return lo < rhs.lo;
505
40.3M
        }
506
44.8M
    }
507
508
1.10k
    friend std::ostream& operator<<(std::ostream& out, const RowsetId& rowset_id) {
509
1.10k
        out << rowset_id.to_string();
510
1.10k
        return out;
511
1.10k
    }
512
};
513
514
using RowsetIdUnorderedSet = std::unordered_set<RowsetId>;
515
516
// Extract rowset id from filename, return uninitialized rowset id if filename is invalid
517
483
inline RowsetId extract_rowset_id(std::string_view filename) {
518
483
    RowsetId rowset_id;
519
483
    if (filename.ends_with(".dat")) {
520
        // filename format: {rowset_id}_{segment_num}.dat
521
243
        auto end = filename.find('_');
522
243
        if (end == std::string::npos) {
523
0
            return rowset_id;
524
0
        }
525
243
        rowset_id.init(filename.substr(0, end));
526
243
        return rowset_id;
527
243
    }
528
240
    if (filename.ends_with(".idx")) {
529
        // filename format: {rowset_id}_{segment_num}_{index_id}.idx
530
240
        auto end = filename.find('_');
531
240
        if (end == std::string::npos) {
532
0
            return rowset_id;
533
0
        }
534
240
        rowset_id.init(filename.substr(0, end));
535
240
        return rowset_id;
536
240
    }
537
0
    return rowset_id;
538
240
}
539
540
class DeleteBitmap;
541
542
struct CalcDeleteBitmapTask {
543
    std::mutex m;
544
    Status status {Status::OK()};
545
    CountDownLatch latch {1};
546
547
60
    void set_status(Status st) {
548
60
        {
549
60
            std::unique_lock l(m);
550
60
            status = std::move(st);
551
60
        }
552
60
        latch.count_down(1);
553
60
    }
554
555
0
    Status get_status() {
556
0
        if (!latch.wait_for(
557
0
                    std::chrono::seconds(config::segcompaction_wait_for_dbm_task_timeout_s))) {
558
0
            return Status::InternalError<false>("wait for calc delete bitmap task timeout");
559
0
        };
560
0
        std::unique_lock l(m);
561
0
        return status;
562
0
    }
563
};
564
565
// merge on write context
566
struct MowContext {
567
    MowContext(int64_t version, int64_t txnid, const RowsetIdUnorderedSet& ids,
568
               std::vector<RowsetSharedPtr> rowset_ptrs, std::shared_ptr<DeleteBitmap> db)
569
8
            : max_version(version),
570
8
              txn_id(txnid),
571
8
              rowset_ids(ids),
572
8
              rowset_ptrs(std::move(rowset_ptrs)),
573
8
              delete_bitmap(std::move(db)) {}
574
575
112
    CalcDeleteBitmapTask* get_calc_dbm_task(int32_t segment_id) {
576
112
        std::lock_guard l(m);
577
112
        return &calc_dbm_tasks[segment_id];
578
112
    }
579
580
    int64_t max_version;
581
    int64_t txn_id;
582
    const RowsetIdUnorderedSet& rowset_ids;
583
    std::vector<RowsetSharedPtr> rowset_ptrs;
584
    std::shared_ptr<DeleteBitmap> delete_bitmap;
585
586
    std::mutex m;
587
    // status of calc delete bitmap task in flush phase
588
    std::unordered_map<int32_t /* origin seg id*/, CalcDeleteBitmapTask> calc_dbm_tasks;
589
};
590
591
// used for controll compaction
592
struct VersionWithTime {
593
    std::atomic<int64_t> version;
594
    int64_t update_ts;
595
596
0
    VersionWithTime() : version(0), update_ts(MonotonicMillis()) {}
597
598
0
    void update_version_monoto(int64_t new_version) {
599
0
        int64_t cur_version = version.load(std::memory_order_relaxed);
600
0
        while (cur_version < new_version) {
601
0
            if (version.compare_exchange_strong(cur_version, new_version, std::memory_order_relaxed,
602
0
                                                std::memory_order_relaxed)) {
603
0
                update_ts = MonotonicMillis();
604
0
                break;
605
0
            }
606
0
        }
607
0
    }
608
};
609
#include "common/compile_check_end.h"
610
} // namespace doris
611
612
// This intended to be a "good" hash function.  It may change from time to time.
613
template <>
614
struct std::hash<doris::RowsetId> {
615
1.79k
    size_t operator()(const doris::RowsetId& rowset_id) const {
616
1.79k
        size_t seed = 0;
617
1.79k
        seed = doris::HashUtil::xxHash64WithSeed((const char*)&rowset_id.hi, sizeof(rowset_id.hi),
618
1.79k
                                                 seed);
619
1.79k
        seed = doris::HashUtil::xxHash64WithSeed((const char*)&rowset_id.mi, sizeof(rowset_id.mi),
620
1.79k
                                                 seed);
621
1.79k
        seed = doris::HashUtil::xxHash64WithSeed((const char*)&rowset_id.lo, sizeof(rowset_id.lo),
622
1.79k
                                                 seed);
623
1.79k
        return seed;
624
1.79k
    }
625
};