diff --git a/fe/fe-core/src/main/java/com/starrocks/backup/BackupJob.java b/fe/fe-core/src/main/java/com/starrocks/backup/BackupJob.java index 6651b47c5222b..85a11b7afe24e 100644 --- a/fe/fe-core/src/main/java/com/starrocks/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/backup/BackupJob.java @@ -707,11 +707,9 @@ private void saveMetaInfo() { localMetaInfoFilePath = metaInfoFile.getAbsolutePath(); // 3. save job info file - // save table info into BackupJobInfo only for OlapTable or MV - List olapTbls = backupMeta.getTables().values().stream() - .filter(Table::isOlapTableOrMaterializedView).collect(Collectors.toList()); - jobInfo = BackupJobInfo.fromCatalog(createTime, label, dbName, dbId, olapTbls, snapshotInfos); - LOG.warn("job info: {}. {}", jobInfo, this); + jobInfo = BackupJobInfo.fromCatalog(createTime, label, dbName, dbId, backupMeta.getTables().values(), + snapshotInfos); + LOG.debug("job info: {}. {}", jobInfo, this); File jobInfoFile = new File(jobDir, Repository.PREFIX_JOB_INFO + createTimeStr); if (!jobInfoFile.createNewFile()) { status = new Status(ErrCode.COMMON_ERROR, "Failed to create job info file: " + jobInfoFile.toString()); diff --git a/fe/fe-core/src/main/java/com/starrocks/backup/BackupJobInfo.java b/fe/fe-core/src/main/java/com/starrocks/backup/BackupJobInfo.java index 431b8603a63b1..5a80308bfc3c9 100644 --- a/fe/fe-core/src/main/java/com/starrocks/backup/BackupJobInfo.java +++ b/fe/fe-core/src/main/java/com/starrocks/backup/BackupJobInfo.java @@ -304,11 +304,16 @@ public static BackupJobInfo fromCatalog(long backupTime, String label, String db // tbls for (Table tbl : tbls) { - OlapTable olapTbl = (OlapTable) tbl; BackupTableInfo tableInfo = new BackupTableInfo(); tableInfo.id = tbl.getId(); tableInfo.name = tbl.getName(); jobInfo.tables.put(tableInfo.name, tableInfo); + + if (tbl.isOlapView()) { + continue; + } + + OlapTable olapTbl = (OlapTable) tbl; // partitions for (Partition partition : olapTbl.getPartitions()) { BackupPartitionInfo partitionInfo = new BackupPartitionInfo(); @@ -451,6 +456,11 @@ private static void genFromJson(String json, BackupJobInfo jobInfo) { } JSONObject parts = tbl.getJSONObject("partitions"); String[] partsNames = JSONObject.getNames(parts); + if (partsNames == null) { + // skip logical view + jobInfo.tables.put(tblName, tblInfo); + continue; + } for (String partName : partsNames) { BackupPartitionInfo partInfo = new BackupPartitionInfo(); partInfo.name = partName; diff --git a/fe/fe-core/src/main/java/com/starrocks/backup/RestoreJob.java b/fe/fe-core/src/main/java/com/starrocks/backup/RestoreJob.java index a91b415e5f2f3..2a72303f64d91 100644 --- a/fe/fe-core/src/main/java/com/starrocks/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/backup/RestoreJob.java @@ -47,7 +47,6 @@ import com.google.common.collect.Table.Cell; import com.google.gson.annotations.SerializedName; import com.starrocks.analysis.BrokerDesc; -import com.starrocks.analysis.TableName; import com.starrocks.backup.BackupJobInfo.BackupIndexInfo; import com.starrocks.backup.BackupJobInfo.BackupPartitionInfo; import com.starrocks.backup.BackupJobInfo.BackupPhysicalPartitionInfo; @@ -79,9 +78,7 @@ import com.starrocks.catalog.Table; import com.starrocks.catalog.Tablet; import com.starrocks.catalog.TabletMeta; -import com.starrocks.catalog.View; import com.starrocks.common.Config; -import com.starrocks.common.DdlException; import com.starrocks.common.Pair; import com.starrocks.common.UserException; import com.starrocks.common.io.Text; @@ -93,11 +90,8 @@ import com.starrocks.fs.HdfsUtil; import com.starrocks.metric.MetricRepo; import com.starrocks.persist.ColocatePersistInfo; -import com.starrocks.qe.ConnectContext; import com.starrocks.server.GlobalStateMgr; import com.starrocks.sql.analyzer.SemanticException; -import com.starrocks.sql.ast.CreateViewStmt; -import com.starrocks.sql.parser.NodePosition; import com.starrocks.task.AgentBatchTask; import com.starrocks.task.AgentTask; import com.starrocks.task.AgentTaskExecutor; @@ -538,6 +532,18 @@ private void checkAndPrepareMeta() { Preconditions.checkNotNull(remoteTbl); Table localTbl = globalStateMgr.getLocalMetastore() .getTable(db.getFullName(), jobInfo.getAliasByOriginNameIfSet(tblInfo.name)); + + if (localTbl != null && remoteTbl.isOlapView() && !localTbl.isOlapView()) { + status = new Status(ErrCode.BAD_REPLACE, + "Table: " + localTbl.getName() + " has existed and it is not a View"); + return; + } + + if (remoteTbl.isOlapView()) { + restoredTbls.add(remoteTbl); + continue; + } + if (localTbl != null) { if (localTbl instanceof OlapTable && localTbl.hasAutoIncrementColumn()) { // it must be !isReplay == true @@ -768,7 +774,7 @@ private void checkAndPrepareMeta() { backupTableInfo.getPartInfo(restorePart.getName()), true); } // set restored table's new name after all 'genFileMapping' - ((OlapTable) restoreTbl).setName(jobInfo.getAliasByOriginNameIfSet(restoreTbl.getName())); + restoreTbl.setName(jobInfo.getAliasByOriginNameIfSet(restoreTbl.getName())); } LOG.debug("finished to generate create replica tasks. {}", this); @@ -788,14 +794,6 @@ private void checkAndPrepareMeta() { return; } - // add all restored olap view into globalStateMgr - List restoredOlapViews = backupMeta.getTables().values().stream().filter(Table::isOlapView) - .map(x -> (View) x).collect(Collectors.toList()); - addRestoreOlapView(restoredOlapViews); - if (!status.ok()) { - return; - } - LOG.info("finished to prepare meta. begin to make snapshot. {}", this); // begin to make snapshots for all replicas @@ -853,38 +851,6 @@ protected void sendCreateReplicaTasks() { } } - protected void addRestoreOlapView(List restoredOlapViews) { - Database db = globalStateMgr.getLocalMetastore().getDb(dbId); - - ConnectContext context = new ConnectContext(); - context.setDatabase(db.getFullName()); - context.setGlobalStateMgr(globalStateMgr); - context.setStartTime(); - context.setThreadLocalInfo(); - - for (View restoredOlapView : restoredOlapViews) { - Table localTbl = db.getTable(restoredOlapView.getId()); - if (localTbl != null && !localTbl.isOlapView()) { - status = new Status(ErrCode.BAD_REPLACE, - "Table: " + localTbl.getName() + " has existed and it is not a View"); - return; - } - - CreateViewStmt stmt = new CreateViewStmt(false, true, new TableName(db.getFullName(), restoredOlapView.getName()), - Lists.newArrayList(), restoredOlapView.getComment(), restoredOlapView.getQueryStatement(), NodePosition.ZERO); - stmt.setColumns(restoredOlapView.getColumns()); - stmt.setInlineViewDef(restoredOlapView.getInlineViewDef()); - context.getSessionVariable().setSqlMode(restoredOlapView.getSqlMode()); - try { - GlobalStateMgr.getCurrentState().getMetadataMgr().createView(stmt); - } catch (DdlException e) { - status = new Status(ErrCode.COMMON_ERROR, - "Failed to create view for restore. err message: " + e.getMessage()); - return; - } - } - } - protected void addRestorePartitionsAndTables(Database db) { Locker locker = new Locker(); locker.lockDatabase(db.getId(), LockType.WRITE); @@ -896,6 +862,10 @@ protected void addRestorePartitionsAndTables(Database db) { // add restored tables for (Table tbl : restoredTbls) { + if (tbl.isOlapView()) { + // for logical view, force drop and replace + db.dropTable(tbl.getName()); + } if (!db.registerTableUnlocked(tbl)) { status = new Status(ErrCode.COMMON_ERROR, "Table " + tbl.getName() + " already exist in db: " + db.getOriginName()); @@ -1152,7 +1122,7 @@ private void replayCheckAndPrepareMeta() { for (BackupTableInfo tblInfo : jobInfo.tables.values()) { Table tbl = globalStateMgr.getLocalMetastore() .getTable(db.getFullName(), jobInfo.getAliasByOriginNameIfSet(tblInfo.name)); - if (tbl == null) { + if (tbl == null || tbl.isOlapView()) { continue; } OlapTable olapTbl = (OlapTable) tbl; @@ -1174,9 +1144,8 @@ private void replayCheckAndPrepareMeta() { locker.unLockDatabase(db.getId(), LockType.WRITE); } - List restoredOlapViews = backupMeta.getTables().values().stream().filter(Table::isOlapView) - .map(x -> (View) x).collect(Collectors.toList()); - addRestoreOlapView(restoredOlapViews); + // restored view need not to be added again here, because + // another edit log created by createView will done. LOG.info("replay check and prepare meta. {}", this); } @@ -1493,7 +1462,7 @@ private Status allTabletCommitted(boolean isReplay) { for (long tblId : restoredVersionInfo.rowKeySet()) { Table tbl = globalStateMgr.getLocalMetastore().getTable(db.getId(), tblId); - if (tbl == null) { + if (tbl == null || tbl.isOlapView()) { continue; } OlapTable olapTbl = (OlapTable) tbl;