MultiPartitionDesc.java
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.analysis;
import org.apache.doris.analysis.TimestampArithmeticExpr.TimeUnit;
import org.apache.doris.catalog.DynamicPartitionProperty;
import org.apache.doris.catalog.ReplicaAllocation;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.util.DynamicPartitionUtil;
import org.apache.doris.common.util.TimeUtils;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import java.time.DayOfWeek;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoField;
import java.time.temporal.WeekFields;
import java.util.List;
import java.util.Map;
// to describe the key range multi partition's information in create table stmt
public class MultiPartitionDesc implements AllPartitionDesc {
public static final String HOURS_FORMAT = "yyyyMMddHH";
public static final String HOUR_FORMAT = "yyyy-MM-dd HH";
public static final String DATES_FORMAT = "yyyyMMdd";
public static final String DATE_FORMAT = "yyyy-MM-dd";
public static final String MONTHS_FORMAT = "yyyyMM";
public static final String MONTH_FORMAT = "yyyy-MM";
public static final String YEAR_FORMAT = "yyyy";
public static final String DATETIME_FORMAT = "yyyy-MM-dd HH:mm:ss";
private LocalDateTime startTime;
private LocalDateTime endTime;
private String startString;
private String endString;
private DateTimeFormatter startDateTimeFormat;
private DateTimeFormatter endDateTimeFormat;
private Long interval;
private final PartitionKeyDesc partitionKeyDesc;
private TimeUnit timeUnitType;
private final Map<String, String> properties;
private final List<SinglePartitionDesc> singlePartitionDescList = Lists.newArrayList();
private final ImmutableSet<TimeUnit> timeUnitTypeMultiPartition = ImmutableSet.of(
TimeUnit.HOUR,
TimeUnit.DAY,
TimeUnit.WEEK,
TimeUnit.MONTH,
TimeUnit.YEAR
);
private final Integer maxAllowedLimit = Config.max_multi_partition_num;
//multi_partition_name_prefix default: p_
private final String partitionPrefix = Config.multi_partition_name_prefix;
public MultiPartitionDesc(PartitionKeyDesc partitionKeyDesc,
Map<String, String> properties) throws AnalysisException {
this.partitionKeyDesc = partitionKeyDesc;
this.properties = properties;
this.intervalTrans();
this.trans();
}
/**
* for Nereids
*/
public MultiPartitionDesc(PartitionKeyDesc partitionKeyDesc, ReplicaAllocation replicaAllocation,
Map<String, String> properties) throws AnalysisException {
this.partitionKeyDesc = partitionKeyDesc;
this.properties = properties;
this.intervalTrans();
this.trans();
for (SinglePartitionDesc desc : getSinglePartitionDescList()) {
desc.setReplicaAlloc(replicaAllocation);
desc.setAnalyzed(true);
}
}
public List<SinglePartitionDesc> getSinglePartitionDescList() throws AnalysisException {
if (singlePartitionDescList.size() == 0) {
if (this.timeUnitType == null) {
buildNumberMultiPartitionToSinglePartitionDesc();
} else {
buildTimeMultiPartitionToSinglePartitionDesc();
}
}
return singlePartitionDescList;
}
private List<SinglePartitionDesc> buildNumberMultiPartitionToSinglePartitionDesc() throws AnalysisException {
long countNum = 0;
long beginNum;
long endNum;
String partitionPrefix = this.partitionPrefix;
try {
beginNum = Long.parseLong(startString);
endNum = Long.parseLong(endString);
} catch (NumberFormatException ex) {
throw new AnalysisException("Batch build partition INTERVAL is number type "
+ "but From or TO does not type match.");
}
if (beginNum >= endNum) {
throw new AnalysisException("Batch build partition From value should less then TO value.");
}
while (beginNum < endNum) {
String partitionName = partitionPrefix + beginNum;
PartitionValue lowerPartitionValue = new PartitionValue(beginNum);
beginNum += interval;
Long currentEnd = Math.min(beginNum, endNum);
PartitionValue upperPartitionValue = new PartitionValue(currentEnd);
partitionName = partitionName + "_" + currentEnd;
PartitionKeyDesc partitionKeyDesc = PartitionKeyDesc.createFixed(
Lists.newArrayList(lowerPartitionValue),
Lists.newArrayList(upperPartitionValue)
);
singlePartitionDescList.add(
new SinglePartitionDesc(
false,
partitionName,
partitionKeyDesc,
properties)
);
countNum++;
if (countNum > maxAllowedLimit) {
throw new AnalysisException("The number of Multi partitions too much, should not exceed:"
+ maxAllowedLimit);
}
}
return singlePartitionDescList;
}
private List<SinglePartitionDesc> buildTimeMultiPartitionToSinglePartitionDesc() throws AnalysisException {
String partitionName;
long countNum = 0;
int startDayOfWeek = 1;
int startDayOfMonth = 1;
String partitionPrefix = this.partitionPrefix;
LocalDateTime startTime = this.startTime;
if (properties != null) {
if (properties.containsKey(DynamicPartitionProperty.START_DAY_OF_WEEK)) {
String dayOfWeekStr = properties.get(DynamicPartitionProperty.START_DAY_OF_WEEK);
try {
DynamicPartitionUtil.checkStartDayOfWeek(dayOfWeekStr);
} catch (DdlException e) {
throw new AnalysisException(e.getMessage());
}
startDayOfWeek = Integer.parseInt(dayOfWeekStr);
}
if (properties.containsKey(DynamicPartitionProperty.START_DAY_OF_MONTH)) {
String dayOfMonthStr = properties.get(DynamicPartitionProperty.START_DAY_OF_MONTH);
try {
DynamicPartitionUtil.checkStartDayOfMonth(dayOfMonthStr);
} catch (DdlException e) {
throw new AnalysisException(e.getMessage());
}
startDayOfMonth = Integer.parseInt(dayOfMonthStr);
}
if (properties.containsKey(DynamicPartitionProperty.CREATE_HISTORY_PARTITION)) {
properties.put(DynamicPartitionProperty.CREATE_HISTORY_PARTITION, "false");
}
if (properties.containsKey(DynamicPartitionProperty.PREFIX)) {
partitionPrefix = properties.get(DynamicPartitionProperty.PREFIX);
try {
DynamicPartitionUtil.checkPrefix(partitionPrefix);
} catch (DdlException e) {
throw new AnalysisException(e.getMessage());
}
}
}
WeekFields weekFields = WeekFields.of(DayOfWeek.of(startDayOfWeek), 1);
while (startTime.isBefore(this.endTime)) {
PartitionValue lowerPartitionValue = new PartitionValue(
startTime.format(dateTypeFormat(partitionKeyDesc.getLowerValues().get(0).getStringValue()))
);
switch (this.timeUnitType) {
case HOUR:
partitionName = partitionPrefix + startTime.format(DateTimeFormatter.ofPattern(HOURS_FORMAT));
startTime = startTime.plusHours(interval);
break;
case DAY:
partitionName = partitionPrefix + startTime.format(DateTimeFormatter.ofPattern(DATES_FORMAT));
startTime = startTime.plusDays(interval);
break;
case WEEK:
LocalDate localDate = LocalDate.of(startTime.getYear(), startTime.getMonthValue(),
startTime.getDayOfMonth());
int weekOfYear = localDate.get(weekFields.weekOfYear());
partitionName = String.format("%s%s_%02d", partitionPrefix,
startTime.format(DateTimeFormatter.ofPattern(YEAR_FORMAT)), weekOfYear);
startTime = startTime.with(ChronoField.DAY_OF_WEEK, startDayOfMonth);
startTime = startTime.plusWeeks(interval);
break;
case MONTH:
partitionName = partitionPrefix + startTime.format(DateTimeFormatter.ofPattern(MONTHS_FORMAT));
startTime = startTime.withDayOfMonth(startDayOfMonth);
startTime = startTime.plusMonths(interval);
break;
case YEAR:
partitionName = partitionPrefix + startTime.format(DateTimeFormatter.ofPattern(YEAR_FORMAT));
startTime = startTime.withDayOfYear(1);
startTime = startTime.plusYears(interval);
break;
default:
throw new AnalysisException("Multi build partition does not support time interval type: "
+ this.timeUnitType);
}
if (this.timeUnitType != TimeUnit.DAY && startTime.isAfter(this.endTime)) {
startTime = this.endTime;
}
PartitionValue upperPartitionValue = new PartitionValue(
startTime.format(dateTypeFormat(partitionKeyDesc.getUpperValues().get(0).getStringValue()))
);
PartitionKeyDesc partitionKeyDesc = PartitionKeyDesc.createFixed(
Lists.newArrayList(lowerPartitionValue),
Lists.newArrayList(upperPartitionValue)
);
singlePartitionDescList.add(
new SinglePartitionDesc(
false,
partitionName,
partitionKeyDesc,
properties)
);
countNum++;
if (countNum > maxAllowedLimit) {
throw new AnalysisException("The number of Multi partitions too much, should not exceed:"
+ maxAllowedLimit);
}
}
return singlePartitionDescList;
}
private void trans() throws AnalysisException {
if (partitionKeyDesc.getLowerValues().size() != 1 || partitionKeyDesc.getUpperValues().size() != 1) {
throw new AnalysisException("partition column number in multi partition clause must be one "
+ "but start column size is " + partitionKeyDesc.getLowerValues().size()
+ ", end column size is " + partitionKeyDesc.getUpperValues().size() + ".");
}
this.startString = partitionKeyDesc.getLowerValues().get(0).getStringValue();
this.endString = partitionKeyDesc.getUpperValues().get(0).getStringValue();
if (this.timeUnitType == null) {
if (this.startString.compareTo(this.endString) >= 0) {
throw new AnalysisException("Multi build partition start number should less than end number.");
}
} else {
try {
this.startDateTimeFormat = dateFormat(this.timeUnitType, this.startString);
this.endDateTimeFormat = dateFormat(this.timeUnitType, this.endString);
this.startTime = TimeUtils.formatDateTimeAndFullZero(this.startString, startDateTimeFormat);
this.endTime = TimeUtils.formatDateTimeAndFullZero(this.endString, endDateTimeFormat);
} catch (Exception e) {
throw new AnalysisException("Multi build partition start or end time style is illegal.");
}
if (!this.startTime.isBefore(this.endTime)) {
throw new AnalysisException("Multi build partition start time should less than end time.");
}
}
}
private void intervalTrans() throws AnalysisException {
this.interval = partitionKeyDesc.getTimeInterval();
String timeType = partitionKeyDesc.getTimeType();
if (timeType == null) {
throw new AnalysisException("Unknown time interval type for Multi build partition.");
}
if (this.interval <= 0) {
throw new AnalysisException("Multi partition time interval must be larger than zero.");
}
if (!timeType.equals("")) {
try {
this.timeUnitType = TimeUnit.valueOf(timeType.toUpperCase());
} catch (Exception e) {
throw new AnalysisException("Multi build partition got an unknown time interval type: "
+ timeType);
}
}
if (this.timeUnitType != null && !timeUnitTypeMultiPartition.contains(this.timeUnitType)) {
throw new AnalysisException("Multi build partition does not support time interval type: "
+ this.timeUnitType);
}
}
private static DateTimeFormatter dateFormat(TimeUnit timeUnitType,
String dateTimeStr) throws AnalysisException {
DateTimeFormatter res;
switch (timeUnitType) {
case HOUR:
if (dateTimeStr.length() == 10) {
res = DateTimeFormatter.ofPattern(HOURS_FORMAT);
} else if (dateTimeStr.length() == 13) {
res = DateTimeFormatter.ofPattern(HOUR_FORMAT);
} else if (dateTimeStr.length() == 19) {
res = DateTimeFormatter.ofPattern(DATETIME_FORMAT);
} else {
throw new AnalysisException("can not probe datetime(hour) format:" + dateTimeStr);
}
break;
case DAY: case WEEK:
if (dateTimeStr.length() == 8) {
res = DateTimeFormatter.ofPattern(DATES_FORMAT);
} else if (dateTimeStr.length() == 10) {
res = DateTimeFormatter.ofPattern(DATE_FORMAT);
} else if (dateTimeStr.length() == 19) {
res = DateTimeFormatter.ofPattern(DATETIME_FORMAT);
} else {
throw new AnalysisException("can not probe datetime(day or week) format:" + dateTimeStr);
}
break;
case MONTH:
if (dateTimeStr.length() == 6) {
res = DateTimeFormatter.ofPattern(MONTHS_FORMAT);
} else if (dateTimeStr.length() == 7) {
res = DateTimeFormatter.ofPattern(MONTH_FORMAT);
} else if (dateTimeStr.length() == 10) {
res = DateTimeFormatter.ofPattern(DATE_FORMAT);
} else if (dateTimeStr.length() == 19) {
res = DateTimeFormatter.ofPattern(DATETIME_FORMAT);
} else {
throw new AnalysisException("can not probe datetime(month) format:" + dateTimeStr);
}
break;
case YEAR:
if (dateTimeStr.length() == 4) {
res = DateTimeFormatter.ofPattern(YEAR_FORMAT);
} else if (dateTimeStr.length() == 8) {
res = DateTimeFormatter.ofPattern(DATES_FORMAT);
} else if (dateTimeStr.length() == 10) {
res = DateTimeFormatter.ofPattern(DATE_FORMAT);
} else if (dateTimeStr.length() == 19) {
res = DateTimeFormatter.ofPattern(DATETIME_FORMAT);
} else {
throw new AnalysisException("can not probe datetime(year) format:" + dateTimeStr);
}
break;
default:
throw new AnalysisException("Multi build partition does not support time interval type: "
+ timeUnitType);
}
return res;
}
private DateTimeFormatter dateTypeFormat(String dateTimeStr) {
String s = DATE_FORMAT;
if (this.timeUnitType.equals(TimeUnit.HOUR) || dateTimeStr.length() == 19) {
s = DATETIME_FORMAT;
}
return DateTimeFormatter.ofPattern(s);
}
}