MTMVUtil.java
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.mtmv;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.constraint.Constraint;
import org.apache.doris.catalog.constraint.ForeignKeyConstraint;
import org.apache.doris.catalog.constraint.PrimaryKeyConstraint;
import org.apache.doris.catalog.info.TableNameInfo;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.datasource.CatalogMgr;
import org.apache.doris.job.common.JobType;
import org.apache.doris.job.common.TaskStatus;
import org.apache.doris.job.task.AbstractTask;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeExtractAndTransform;
import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal;
import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal;
import org.apache.doris.nereids.trees.expressions.literal.DecimalV3Literal;
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
import org.apache.doris.qe.ConnectContext;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
public class MTMVUtil {
private static final Logger LOG = LogManager.getLogger(MTMVUtil.class);
/**
* get Table by BaseTableInfo
*
* @param baseTableInfo
* @return
* @throws AnalysisException
*/
public static TableIf getTable(BaseTableInfo baseTableInfo) throws AnalysisException {
// for compatible old version, not have name
if (StringUtils.isEmpty(baseTableInfo.getCtlName())) {
return Env.getCurrentEnv().getCatalogMgr()
.getCatalogOrAnalysisException(baseTableInfo.getCtlId())
.getDbOrAnalysisException(baseTableInfo.getDbId())
.getTableOrAnalysisException(baseTableInfo.getTableId());
} else {
return Env.getCurrentEnv().getCatalogMgr()
.getCatalogOrAnalysisException(baseTableInfo.getCtlName())
.getDbOrAnalysisException(baseTableInfo.getDbName())
.getTableOrAnalysisException(baseTableInfo.getTableName());
}
}
public static MTMVRelatedTableIf getRelatedTable(BaseTableInfo baseTableInfo) {
TableIf relatedTable = null;
try {
relatedTable = MTMVUtil.getTable(baseTableInfo);
} catch (org.apache.doris.common.AnalysisException e) {
throw new org.apache.doris.nereids.exceptions.AnalysisException(e.getMessage(), e);
}
if (!(relatedTable instanceof MTMVRelatedTableIf)) {
throw new org.apache.doris.nereids.exceptions.AnalysisException(
"base table for partitioning only can be OlapTable or HMSTable");
}
return (MTMVRelatedTableIf) relatedTable;
}
public static MTMV getMTMV(BaseTableInfo baseTableInfo) throws AnalysisException {
TableIf table = getTable(baseTableInfo);
if (!(table instanceof MTMV)) {
throw new AnalysisException(String.format("table is not MTMV, table: %s", baseTableInfo));
}
return (MTMV) table;
}
public static MTMV getMTMV(long dbId, long mtmvId) throws DdlException, MetaNotFoundException {
Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(dbId);
return (MTMV) db.getTableOrMetaException(mtmvId, TableType.MATERIALIZED_VIEW);
}
public static List<MTMV> getDependentMtmvsByBaseTables(List<BaseTableInfo> baseTableInfos)
throws AnalysisException {
Set<BaseTableInfo> uniqueBases = new LinkedHashSet<>(baseTableInfos);
Map<Long, MTMV> mtmvById = new TreeMap<>();
for (BaseTableInfo base : uniqueBases) {
for (BaseTableInfo mtmvInfo
: Env.getCurrentEnv().getMtmvService().getRelationManager()
.getMtmvsByBaseTable(base)) {
try {
MTMV mtmv = MTMVUtil.getMTMV(mtmvInfo);
mtmvById.put(mtmv.getId(), mtmv);
} catch (AnalysisException e) {
LOG.warn("Skip stale dependent MTMV relation {} for base table {}",
mtmvInfo, base, e);
}
}
}
return new ArrayList<>(mtmvById.values());
}
public static List<BaseTableInfo> getConstraintRelatedBaseTables(
TableNameInfo tableNameInfo, Constraint constraint) {
List<BaseTableInfo> baseTables = new ArrayList<>();
baseTables.add(new BaseTableInfo(tableNameInfo));
if (constraint instanceof ForeignKeyConstraint) {
TableNameInfo referencedTableInfo = ((ForeignKeyConstraint) constraint).getReferencedTableName();
if (referencedTableInfo != null) {
baseTables.add(new BaseTableInfo(referencedTableInfo));
}
} else if (constraint instanceof PrimaryKeyConstraint) {
for (TableNameInfo fkTableInfo : ((PrimaryKeyConstraint) constraint).getForeignTableInfos()) {
baseTables.add(new BaseTableInfo(fkTableInfo));
}
}
return baseTables;
}
public static List<MTMV> getDependentMtmvsByConstraint(TableNameInfo tableNameInfo, Constraint constraint)
throws AnalysisException {
return getDependentMtmvsByBaseTables(getConstraintRelatedBaseTables(tableNameInfo, constraint));
}
public static void invalidateRewriteCaches(List<MTMV> dependentMtmvs) {
for (MTMV dependentMtmv : dependentMtmvs) {
dependentMtmv.invalidateRewriteCache();
}
}
public static TableIf getTable(List<String> names) throws AnalysisException {
if (names == null || names.size() != 3) {
throw new AnalysisException("size of names need 3, but names is:" + names);
}
return Env.getCurrentEnv().getCatalogMgr()
.getCatalogOrAnalysisException(names.get(0))
.getDbOrAnalysisException(names.get(1))
.getTableOrAnalysisException(names.get(2));
}
/**
* if base tables of mtmv contains external table
*
* @param mtmv
* @return
*/
public static boolean mtmvContainsExternalTable(MTMV mtmv) {
Set<BaseTableInfo> baseTables = mtmv.getRelation().getBaseTablesOneLevelAndFromView();
return baseTables.stream().anyMatch(baseTableInfo -> !baseTableInfo.isInternalTable());
}
/**
* Convert LiteralExpr to second
*
* @param expr
* @param dateFormatOptional
* @return
* @throws AnalysisException
*/
public static long getExprTimeSec(org.apache.doris.analysis.LiteralExpr expr, Optional<String> dateFormatOptional)
throws AnalysisException {
if (expr instanceof org.apache.doris.analysis.MaxLiteral) {
return Long.MAX_VALUE;
}
if (expr instanceof org.apache.doris.analysis.NullLiteral) {
return Long.MIN_VALUE;
}
if (expr instanceof org.apache.doris.analysis.DateLiteral) {
return ((org.apache.doris.analysis.DateLiteral) expr).unixTimestamp(TimeUtils.getTimeZone()) / 1000;
}
if (!dateFormatOptional.isPresent()) {
throw new AnalysisException("expr is not DateLiteral and DateFormat is not present.");
}
String dateFormat = dateFormatOptional.get();
Expression strToDate = DateTimeExtractAndTransform
.strToDate(new VarcharLiteral(expr.getStringValue()), new VarcharLiteral(dateFormat));
if (strToDate instanceof DateTimeV2Literal) {
return ((DecimalV3Literal) DateTimeExtractAndTransform
.unixTimestamp((DateTimeV2Literal) strToDate)).getValue().longValue();
} else if (strToDate instanceof DateV2Literal) {
return ((BigIntLiteral) DateTimeExtractAndTransform
.unixTimestamp((DateV2Literal) strToDate)).getValue();
} else {
throw new AnalysisException(
String.format("strToDate failed, stringValue: %s, dateFormat: %s",
expr.getStringValue(), dateFormat));
}
}
public static boolean allowModifyMTMVData(ConnectContext ctx) {
if (ctx == null) {
return false;
}
return ctx.getSessionVariable().isAllowModifyMaterializedViewData();
}
public static void checkModifyMTMVData(Database db, List<Long> tableIdList, ConnectContext ctx)
throws AnalysisException {
if (CollectionUtils.isEmpty(tableIdList)) {
return;
}
for (long tableId : tableIdList) {
Optional<Table> table = db.getTable(tableId);
if (table.isPresent() && table.get() instanceof MTMV && !MTMVUtil.allowModifyMTMVData(ctx)) {
throw new AnalysisException("Not allowed to perform current operation on async materialized view");
}
}
}
public static void compatibleMTMV(CatalogMgr catalogMgr) {
List<Database> dbs = catalogMgr.getInternalCatalog().getDbs();
for (Database database : dbs) {
List<Table> tables = database.getTables();
for (Table table : tables) {
if (table instanceof MTMV) {
((MTMV) table).compatible(catalogMgr);
}
}
}
}
/**
* get MTMV task num by status
*
* @param status status of task
* @return if status is null, return 0
*/
public static Integer getTaskNum(TaskStatus status) {
if (status == null) {
return 0;
}
int res = 0;
List<org.apache.doris.job.base.AbstractJob> jobList = Env.getCurrentEnv().getJobManager().queryJobs(JobType.MV);
for (org.apache.doris.job.base.AbstractJob job : jobList) {
List<AbstractTask> tasks = job.getRunningTasks();
for (AbstractTask task : tasks) {
if (task.getStatus().equals(status)) {
res++;
}
}
}
return res;
}
}