From 13d33f10a83cb118e84eced2a2899fba3cf3bf50 Mon Sep 17 00:00:00 2001 From: srlch <111035020+srlch@users.noreply.github.com> Date: Wed, 23 Oct 2024 16:20:46 +0800 Subject: [PATCH] [Enhancement] Support backup restore for logical view (backport #52077) (#52237) Signed-off-by: srlch --- .../com/starrocks/backup/BackupHandler.java | 17 +- .../java/com/starrocks/backup/BackupJob.java | 44 +++-- .../java/com/starrocks/backup/RestoreJob.java | 73 ++++++- .../java/com/starrocks/backup/Status.java | 3 +- .../java/com/starrocks/catalog/Table.java | 4 + .../main/java/com/starrocks/catalog/View.java | 4 + .../com/starrocks/backup/BackupJobTest.java | 75 +++++++ .../com/starrocks/backup/CatalogMocker.java | 8 + .../com/starrocks/backup/RestoreJobTest.java | 184 +++++++++++++++++- .../starrocks/common/util/UnitTestUtil.java | 16 ++ 10 files changed, 401 insertions(+), 27 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/backup/BackupHandler.java b/fe/fe-core/src/main/java/com/starrocks/backup/BackupHandler.java index 36d0721470d9b..4ccb207c9facd 100644 --- a/fe/fe-core/src/main/java/com/starrocks/backup/BackupHandler.java +++ b/fe/fe-core/src/main/java/com/starrocks/backup/BackupHandler.java @@ -323,8 +323,15 @@ private void backup(Repository repository, Database db, BackupStmt stmt) throws ErrorReport.reportDdlException(ErrorCode.ERR_BAD_TABLE_ERROR, tblName); return; } - if (!tbl.isOlapTableOrMaterializedView()) { - ErrorReport.reportDdlException(ErrorCode.ERR_NOT_OLAP_TABLE, tblName); + if (!tbl.isSupportBackupRestore()) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, + "Table: " + tblName + " can not support backup restore, type: " + + tbl.getType()); + } + + if (tbl.isOlapView()) { + backupTbls.add(tbl); + continue; } OlapTable olapTbl = (OlapTable) tbl; @@ -441,8 +448,10 @@ private void restore(Repository repository, Database db, RestoreStmt stmt) throw if (backupMeta != null) { for (BackupTableInfo tblInfo : jobInfo.tables.values()) { Table remoteTbl = backupMeta.getTable(tblInfo.name); - if (remoteTbl.isCloudNativeTable()) { - ErrorReport.reportDdlException(ErrorCode.ERR_NOT_OLAP_TABLE, remoteTbl.getName()); + if (!remoteTbl.isSupportBackupRestore()) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, + "Table: " + remoteTbl.getName() + + " can not support backup restore, type: " + remoteTbl.getType()); } mvRestoreContext.addIntoMvBaseTableBackupInfoIfNeeded(db.getOriginName(), remoteTbl, jobInfo, tblInfo); } diff --git a/fe/fe-core/src/main/java/com/starrocks/backup/BackupJob.java b/fe/fe-core/src/main/java/com/starrocks/backup/BackupJob.java index c8850136832ab..a4c24de589130 100644 --- a/fe/fe-core/src/main/java/com/starrocks/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/backup/BackupJob.java @@ -58,9 +58,11 @@ import com.starrocks.catalog.Replica; import com.starrocks.catalog.Table; import com.starrocks.catalog.Tablet; +import com.starrocks.catalog.View; import com.starrocks.common.AnalysisException; import com.starrocks.common.Config; import com.starrocks.common.UserException; +import com.starrocks.common.io.DeepCopy; import com.starrocks.common.io.Text; import com.starrocks.common.util.TimeUtils; import com.starrocks.common.util.UUIDUtil; @@ -392,12 +394,16 @@ protected void checkBackupTables(Database db) { status = new Status(ErrCode.NOT_FOUND, "table " + tblName + " does not exist"); return; } - if (!tbl.isOlapTableOrMaterializedView()) { - status = new Status(ErrCode.COMMON_ERROR, "table " + tblName - + " is not OLAP table"); + if (!tbl.isSupportBackupRestore()) { + status = new Status(ErrCode.UNSUPPORTED, + "Table: " + tblName + " can not support backup restore, type: " + tbl.getType()); return; } + if (tbl.isOlapView()) { + continue; + } + OlapTable olapTbl = (OlapTable) tbl; if (tableRef.getPartitionNames() != null) { for (String partName : tableRef.getPartitionNames().getPartitionNames()) { @@ -466,13 +472,17 @@ private void prepareAndSendSnapshotTask() { // create snapshot tasks for (TableRef tblRef : tableRefs) { String tblName = tblRef.getName().getTbl(); - OlapTable tbl = (OlapTable) db.getTable(tblName); + Table tbl = db.getTable(tblName); + if (tbl.isOlapView()) { + continue; + } + OlapTable olapTbl = (OlapTable) tbl; List partitions = Lists.newArrayList(); if (tblRef.getPartitionNames() == null) { - partitions.addAll(tbl.getPartitions()); + partitions.addAll(olapTbl.getPartitions()); } else { for (String partName : tblRef.getPartitionNames().getPartitionNames()) { - Partition partition = tbl.getPartition(partName); + Partition partition = olapTbl.getPartition(partName); partitions.add(partition); } } @@ -483,9 +493,9 @@ private void prepareAndSendSnapshotTask() { long visibleVersion = physicalPartition.getVisibleVersion(); List indexes = physicalPartition.getMaterializedIndices(IndexExtState.VISIBLE); for (MaterializedIndex index : indexes) { - int schemaHash = tbl.getSchemaHashByIndexId(index.getId()); + int schemaHash = olapTbl.getSchemaHashByIndexId(index.getId()); for (Tablet tablet : index.getTablets()) { - prepareSnapshotTask(physicalPartition, tbl, tablet, index, visibleVersion, schemaHash); + prepareSnapshotTask(physicalPartition, olapTbl, tablet, index, visibleVersion, schemaHash); if (status != Status.OK) { return; } @@ -501,11 +511,17 @@ private void prepareAndSendSnapshotTask() { List copiedTables = Lists.newArrayList(); for (TableRef tableRef : tableRefs) { String tblName = tableRef.getName().getTbl(); - OlapTable tbl = (OlapTable) db.getTable(tblName); + Table tbl = db.getTable(tblName); + if (tbl.isOlapView()) { + View view = (View) tbl; + copiedTables.add((Table) DeepCopy.copyWithGson(view, View.class)); + continue; + } + OlapTable olapTbl = (OlapTable) tbl; // only copy visible indexes List reservedPartitions = tableRef.getPartitionNames() == null ? null : tableRef.getPartitionNames().getPartitionNames(); - OlapTable copiedTbl = tbl.selectiveCopy(reservedPartitions, true, IndexExtState.VISIBLE); + OlapTable copiedTbl = olapTbl.selectiveCopy(reservedPartitions, true, IndexExtState.VISIBLE); if (copiedTbl == null) { status = new Status(ErrCode.COMMON_ERROR, "faild to copy table: " + tblName); return; @@ -684,9 +700,11 @@ private void saveMetaInfo() { localMetaInfoFilePath = metaInfoFile.getAbsolutePath(); // 3. save job info file - jobInfo = BackupJobInfo.fromCatalog(createTime, label, dbName, dbId, backupMeta.getTables().values(), - snapshotInfos); - LOG.debug("job info: {}. {}", jobInfo, this); + // save table info into BackupJobInfo only for OlapTable or MV + List
olapTbls = backupMeta.getTables().values().stream() + .filter(Table::isOlapTableOrMaterializedView).collect(Collectors.toList()); + jobInfo = BackupJobInfo.fromCatalog(createTime, label, dbName, dbId, olapTbls, snapshotInfos); + LOG.warn("job info: {}. {}", jobInfo, this); File jobInfoFile = new File(jobDir, Repository.PREFIX_JOB_INFO + createTimeStr); if (!jobInfoFile.createNewFile()) { status = new Status(ErrCode.COMMON_ERROR, "Failed to create job info file: " + jobInfoFile.toString()); diff --git a/fe/fe-core/src/main/java/com/starrocks/backup/RestoreJob.java b/fe/fe-core/src/main/java/com/starrocks/backup/RestoreJob.java index c82c8757696f7..cbc4fc20101b3 100644 --- a/fe/fe-core/src/main/java/com/starrocks/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/backup/RestoreJob.java @@ -46,6 +46,7 @@ import com.google.common.collect.Table.Cell; import com.google.gson.annotations.SerializedName; import com.starrocks.analysis.BrokerDesc; +import com.starrocks.analysis.TableName; import com.starrocks.backup.BackupJobInfo.BackupIndexInfo; import com.starrocks.backup.BackupJobInfo.BackupPartitionInfo; import com.starrocks.backup.BackupJobInfo.BackupPhysicalPartitionInfo; @@ -76,8 +77,10 @@ import com.starrocks.catalog.Table; import com.starrocks.catalog.Tablet; import com.starrocks.catalog.TabletMeta; +import com.starrocks.catalog.View; import com.starrocks.common.AnalysisException; import com.starrocks.common.Config; +import com.starrocks.common.DdlException; import com.starrocks.common.MarkedCountDownLatch; import com.starrocks.common.Pair; import com.starrocks.common.UserException; @@ -87,7 +90,10 @@ import com.starrocks.fs.HdfsUtil; import com.starrocks.metric.MetricRepo; import com.starrocks.persist.ColocatePersistInfo; +import com.starrocks.qe.ConnectContext; import com.starrocks.server.GlobalStateMgr; +import com.starrocks.sql.ast.CreateViewStmt; +import com.starrocks.sql.parser.NodePosition; import com.starrocks.task.AgentBatchTask; import com.starrocks.task.AgentTask; import com.starrocks.task.AgentTaskExecutor; @@ -191,7 +197,7 @@ public enum RestoreJobState { private AgentBatchTask batchTask; boolean enableColocateRestore = Config.enable_colocate_restore; - + public RestoreJob() { super(JobType.RESTORE); } @@ -484,11 +490,17 @@ private void checkAndPrepareMeta() { continue; } - if (!tbl.isNativeTableOrMaterializedView()) { - status = new Status(ErrCode.COMMON_ERROR, "Only support restore OLAP table: " + tbl.getName()); + if (!tbl.isSupportBackupRestore()) { + status = new Status(ErrCode.UNSUPPORTED, + "Table: " + tbl.getName() + + " can not support backup restore, type: {}" + tbl.getType()); return; } + if (tbl.isOlapView()) { + continue; + } + OlapTable olapTbl = (OlapTable) tbl; if (olapTbl.getState() != OlapTableState.NORMAL) { status = new Status(ErrCode.COMMON_ERROR, @@ -535,9 +547,10 @@ private void checkAndPrepareMeta() { tblInfo.checkAndRecoverAutoIncrementId(localTbl); // table already exist, check schema - if (!localTbl.isNativeTableOrMaterializedView()) { - status = new Status(ErrCode.COMMON_ERROR, - "Only support retore olap table: " + localTbl.getName()); + if (!localTbl.isSupportBackupRestore()) { + status = new Status(ErrCode.UNSUPPORTED, + "Table: " + localTbl.getName() + + " can not support backup restore, type: {}" + localTbl.getType()); return; } OlapTable localOlapTbl = (OlapTable) localTbl; @@ -762,6 +775,14 @@ private void checkAndPrepareMeta() { return; } + // add all restored olap view into globalStateMgr + List restoredOlapViews = backupMeta.getTables().values().stream().filter(Table::isOlapView) + .map(x -> (View) x).collect(Collectors.toList()); + addRestoreOlapView(restoredOlapViews); + if (!status.ok()) { + return; + } + LOG.info("finished to prepare meta. begin to make snapshot. {}", this); // begin to make snapshots for all replicas @@ -819,6 +840,38 @@ protected void sendCreateReplicaTasks() { } } + protected void addRestoreOlapView(List restoredOlapViews) { + Database db = globalStateMgr.getLocalMetastore().getDb(dbId); + + ConnectContext context = new ConnectContext(); + context.setDatabase(db.getFullName()); + context.setGlobalStateMgr(globalStateMgr); + context.setStartTime(); + context.setThreadLocalInfo(); + + for (View restoredOlapView : restoredOlapViews) { + Table localTbl = db.getTable(restoredOlapView.getId()); + if (localTbl != null && !localTbl.isOlapView()) { + status = new Status(ErrCode.BAD_REPLACE, + "Table: " + localTbl.getName() + " has existed and it is not a View"); + return; + } + + CreateViewStmt stmt = new CreateViewStmt(false, true, new TableName(db.getFullName(), restoredOlapView.getName()), + Lists.newArrayList(), restoredOlapView.getComment(), restoredOlapView.getQueryStatement(), NodePosition.ZERO); + stmt.setColumns(restoredOlapView.getColumns()); + stmt.setInlineViewDef(restoredOlapView.getInlineViewDef()); + context.getSessionVariable().setSqlMode(restoredOlapView.getSqlMode()); + try { + GlobalStateMgr.getCurrentState().createView(stmt); + } catch (DdlException e) { + status = new Status(ErrCode.COMMON_ERROR, + "Failed to create view for restore. err message: " + e.getMessage()); + return; + } + } + } + protected void addRestorePartitionsAndTables(Database db) { db.writeLock(); try { @@ -1100,6 +1153,10 @@ private void replayCheckAndPrepareMeta() { db.writeUnlock(); } + List restoredOlapViews = backupMeta.getTables().values().stream().filter(Table::isOlapView) + .map(x -> (View) x).collect(Collectors.toList()); + addRestoreOlapView(restoredOlapViews); + LOG.info("replay check and prepare meta. {}", this); } @@ -1461,7 +1518,7 @@ private Status allTabletCommitted(boolean isReplay) { tblInfo.name); continue; } - if (!tbl.isNativeTableOrMaterializedView()) { + if (!tbl.isSupportBackupRestore()) { continue; } LOG.info("do post actions for table : {}", tbl.getName()); @@ -1692,7 +1749,7 @@ private void setTableStateToNormal(Database db) { continue; } - if (!tbl.isNativeTableOrMaterializedView()) { + if (!tbl.isSupportBackupRestore() || tbl.isOlapView()) { continue; } diff --git a/fe/fe-core/src/main/java/com/starrocks/backup/Status.java b/fe/fe-core/src/main/java/com/starrocks/backup/Status.java index cf77a32700dad..413bc4a2e79f8 100644 --- a/fe/fe-core/src/main/java/com/starrocks/backup/Status.java +++ b/fe/fe-core/src/main/java/com/starrocks/backup/Status.java @@ -31,7 +31,8 @@ public enum ErrCode { BAD_CONNECTION, COMMON_ERROR, OLAP_VERSION_ALREADY_MERGED, - UNSUPPORTED + UNSUPPORTED, + BAD_REPLACE } @SerializedName("ec") diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/Table.java b/fe/fe-core/src/main/java/com/starrocks/catalog/Table.java index 5ac407fe390b2..8e43c4cccd81c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/Table.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/Table.java @@ -825,4 +825,8 @@ public boolean isTable() { !type.equals(TableType.VIEW) && !type.equals(TableType.HIVE_VIEW); } + + public boolean isSupportBackupRestore() { + return isOlapTableOrMaterializedView() || isOlapView(); + } } diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/View.java b/fe/fe-core/src/main/java/com/starrocks/catalog/View.java index 5cead74e5afde..a97fca63b2d45 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/View.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/View.java @@ -136,6 +136,10 @@ public String getInlineViewDef() { return inlineViewDef; } + public long getSqlMode() { + return sqlMode; + } + /** * Initializes the originalViewDef, inlineViewDef, and queryStmt members * by parsing the expanded view definition SQL-string. diff --git a/fe/fe-core/src/test/java/com/starrocks/backup/BackupJobTest.java b/fe/fe-core/src/test/java/com/starrocks/backup/BackupJobTest.java index e13b9f86143bf..ac51ad7396c4a 100644 --- a/fe/fe-core/src/test/java/com/starrocks/backup/BackupJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/backup/BackupJobTest.java @@ -43,6 +43,7 @@ import com.starrocks.catalog.FsBroker; import com.starrocks.catalog.KeysType; import com.starrocks.catalog.OlapTable; +import com.starrocks.catalog.View; import com.starrocks.common.Config; import com.starrocks.common.FeConstants; import com.starrocks.common.jmockit.Deencapsulation; @@ -85,6 +86,7 @@ public class BackupJobTest { private BackupJob job; + private BackupJob jobView; private Database db; private long dbId = 1; @@ -94,6 +96,7 @@ public class BackupJobTest { private long tabletId = 5; private long backendId = 10000; private long version = 6; + private long viewId = 10; private long repoId = 20000; private AtomicLong id = new AtomicLong(50000); @@ -165,6 +168,8 @@ public void setUp() { Deencapsulation.setField(globalStateMgr, "backupHandler", backupHandler); db = UnitTestUtil.createDb(dbId, tblId, partId, idxId, tabletId, backendId, version, KeysType.AGG_KEYS); + View view = UnitTestUtil.createTestView(viewId); + db.registerTableUnlocked(view); new Expectations(globalStateMgr) { { @@ -217,6 +222,10 @@ Status getBrokerAddress(Long beId, GlobalStateMgr globalStateMgr, List List tableRefs = Lists.newArrayList(); tableRefs.add(new TableRef(new TableName(UnitTestUtil.DB_NAME, UnitTestUtil.TABLE_NAME), null)); job = new BackupJob("label", dbId, UnitTestUtil.DB_NAME, tableRefs, 13600 * 1000, globalStateMgr, repo.getId()); + + List viewRefs = Lists.newArrayList(); + viewRefs.add(new TableRef(new TableName(UnitTestUtil.DB_NAME, UnitTestUtil.VIEW_NAME), null)); + jobView = new BackupJob("label-view", dbId, UnitTestUtil.DB_NAME, viewRefs, 13600 * 1000, globalStateMgr, repo.getId()); } @Test @@ -360,4 +369,70 @@ public void testRunAbnormal() { Assert.assertEquals(Status.ErrCode.NOT_FOUND, job.getStatus().getErrCode()); Assert.assertEquals(BackupJobState.CANCELLED, job.getState()); } + + @Test + public void testRunViewNormal() { + // 1.pending + Assert.assertEquals(BackupJobState.PENDING, jobView.getState()); + jobView.run(); + Assert.assertEquals(Status.OK, jobView.getStatus()); + Assert.assertEquals(BackupJobState.SNAPSHOTING, jobView.getState()); + + BackupMeta backupMeta = jobView.getBackupMeta(); + Assert.assertEquals(1, backupMeta.getTables().size()); + View backupView = (View) backupMeta.getTable(UnitTestUtil.VIEW_NAME); + Assert.assertTrue(backupView != null); + Assert.assertTrue(backupView.getPartitions().isEmpty()); + + // 2. snapshoting finished, not snapshot needed + jobView.run(); + Assert.assertEquals(Status.OK, jobView.getStatus()); + Assert.assertEquals(BackupJobState.UPLOAD_SNAPSHOT, jobView.getState()); + + // 3. upload snapshots + jobView.run(); + Assert.assertEquals(Status.OK, jobView.getStatus()); + Assert.assertEquals(BackupJobState.UPLOADING, jobView.getState()); + + // 4. uploading + jobView.run(); + Assert.assertEquals(Status.OK, jobView.getStatus()); + Assert.assertEquals(BackupJobState.SAVE_META, jobView.getState()); + + // 5. save meta + jobView.run(); + Assert.assertEquals(Status.OK, jobView.getStatus()); + Assert.assertEquals(BackupJobState.UPLOAD_INFO, jobView.getState()); + File metaInfo = new File(jobView.getLocalMetaInfoFilePath()); + Assert.assertTrue(metaInfo.exists()); + File jobInfo = new File(jobView.getLocalJobInfoFilePath()); + Assert.assertTrue(jobInfo.exists()); + + BackupMeta restoreMetaInfo = null; + BackupJobInfo restoreJobInfo = null; + try { + restoreMetaInfo = BackupMeta.fromFile(jobView.getLocalMetaInfoFilePath(), FeConstants.STARROCKS_META_VERSION); + Assert.assertEquals(1, restoreMetaInfo.getTables().size()); + View view = (View) restoreMetaInfo.getTable(viewId); + Assert.assertNotNull(view); + Assert.assertNotNull(restoreMetaInfo.getTable(UnitTestUtil.VIEW_NAME)); + } catch (IOException e) { + e.printStackTrace(); + Assert.fail(); + } + + Assert.assertNull(jobView.getBackupMeta()); + Assert.assertNull(jobView.getJobInfo()); + + // 6. upload_info + jobView.run(); + Assert.assertEquals(Status.OK, jobView.getStatus()); + Assert.assertEquals(BackupJobState.FINISHED, jobView.getState()); + + try { + // test get backup info + jobView.getInfo(); + } catch (Exception ignore) { + } + } } diff --git a/fe/fe-core/src/test/java/com/starrocks/backup/CatalogMocker.java b/fe/fe-core/src/test/java/com/starrocks/backup/CatalogMocker.java index 8b79bc95b83f4..9d99424718663 100644 --- a/fe/fe-core/src/test/java/com/starrocks/backup/CatalogMocker.java +++ b/fe/fe-core/src/test/java/com/starrocks/backup/CatalogMocker.java @@ -65,6 +65,7 @@ import com.starrocks.catalog.ScalarType; import com.starrocks.catalog.SinglePartitionInfo; import com.starrocks.catalog.TabletMeta; +import com.starrocks.catalog.View; import com.starrocks.common.AnalysisException; import com.starrocks.common.DdlException; import com.starrocks.common.jmockit.Deencapsulation; @@ -138,6 +139,10 @@ public class CatalogMocker { public static final String TEST_TBL5_NAME = "test_tbl5"; public static final long TEST_TBL5_ID = 30005; + // logical view + public static final String TEST_TBL6_NAME = "test_tbl6"; + public static final long TEST_TBL6_ID = 30006; + public static final String TEST_PARTITION1_NAME = "p1"; public static final long TEST_PARTITION1_ID = 40001; public static final String TEST_PARTITION2_NAME = "p2"; @@ -587,6 +592,9 @@ public static Database mockDb() throws AnalysisException { olapTable5.addPartition(partition1); db.registerTableUnlocked(olapTable5); + + View view = new View(TEST_TBL6_ID, TEST_TBL6_NAME, TEST_TBL_BASE_SCHEMA); + db.registerTableUnlocked(view); } return db; } diff --git a/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java b/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java index a9d08f1348e61..6f7dfdfa7282f 100644 --- a/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java @@ -53,13 +53,17 @@ import com.starrocks.catalog.PhysicalPartition; import com.starrocks.catalog.Table; import com.starrocks.catalog.Tablet; +import com.starrocks.catalog.View; import com.starrocks.common.AnalysisException; import com.starrocks.common.Config; import com.starrocks.common.MarkedCountDownLatch; +import com.starrocks.common.UserException; import com.starrocks.common.jmockit.Deencapsulation; import com.starrocks.metric.MetricRepo; import com.starrocks.persist.EditLog; import com.starrocks.server.GlobalStateMgr; +import com.starrocks.server.LocalMetastore; +import com.starrocks.sql.ast.QueryStatement; import com.starrocks.system.SystemInfoService; import com.starrocks.task.AgentTask; import com.starrocks.task.AgentTaskQueue; @@ -699,5 +703,183 @@ public void testColocateRestore() { Config.enable_colocate_restore = false; } -} + @Test + public void testRestoreView() { + new Expectations() { + { + globalStateMgr.getLocalMetastore().getDb(anyLong); + minTimes = 0; + result = db; + + globalStateMgr.getNextId(); + minTimes = 0; + result = id.incrementAndGet(); + + globalStateMgr.getEditLog(); + minTimes = 0; + result = editLog; + } + }; + + new Expectations() { + { + editLog.logBackupJob((BackupJob) any); + minTimes = 0; + result = new Delegate() { + public void logBackupJob(BackupJob job) { + System.out.println("log backup job: " + job); + } + }; + } + }; + + new Expectations() { + { + repo.upload(anyString, anyString); + result = Status.OK; + minTimes = 0; + + List backupMetas = Lists.newArrayList(); + repo.getSnapshotMetaFile(label, backupMetas, -1, -1); + minTimes = 0; + result = new Delegate() { + public Status getSnapshotMetaFile(String label, List backupMetas) { + backupMetas.add(backupMeta); + return Status.OK; + } + }; + } + }; + + new MockUp() { + @Mock + boolean await(long timeout, TimeUnit unit) { + return true; + } + }; + + // gen BackupJobInfo + jobInfo = new BackupJobInfo(); + jobInfo.backupTime = System.currentTimeMillis(); + jobInfo.dbId = CatalogMocker.TEST_DB_ID; + jobInfo.dbName = CatalogMocker.TEST_DB_NAME; + jobInfo.name = label; + jobInfo.success = true; + + View restoredView = (View) db.getTable(CatalogMocker.TEST_TBL6_ID); + + new MockUp() { + @Mock + public Database getDb(String dbName) { + return db; + } + + @Mock + public Table getTable(String dbName, String tblName) { + return db.getTable(tblName); + } + + @Mock + public Table getTable(Long dbId, Long tableId) { + return db.getTable(tableId); + } + }; + + new MockUp() { + @Mock + public synchronized QueryStatement getQueryStatement() throws UserException { + return null; + } + }; + + new Expectations() { + { + systemInfoService.checkExceedDiskCapacityLimit((Multimap) any, anyBoolean); + minTimes = 0; + result = com.starrocks.common.Status.OK; + } + }; + + List
tbls = Lists.newArrayList(); + tbls.add(restoredView); + backupMeta = new BackupMeta(tbls); + + db.dropTable(restoredView.getName()); + job = new RestoreJob(label, "2018-01-01 01:01:01", db.getId(), db.getFullName(), + jobInfo, false, 3, 100000, + globalStateMgr, repo.getId(), backupMeta, new MvRestoreContext()); + job.setRepo(repo); + Assert.assertEquals(RestoreJobState.PENDING, job.getState()); + { + new MockUp() { + @Mock + public synchronized QueryStatement init() throws UserException { + return null; + } + }; + job.run(); + } + Assert.assertEquals(RestoreJobState.SNAPSHOTING, job.getState()); + Assert.assertEquals(0, job.getFileMapping().getMapping().size()); + + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(RestoreJobState.DOWNLOAD, job.getState()); + + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(RestoreJobState.DOWNLOADING, job.getState()); + + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(RestoreJobState.COMMIT, job.getState()); + + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(RestoreJobState.COMMITTING, job.getState()); + + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(RestoreJobState.FINISHED, job.getState()); + + // restore when the view already existed + job = new RestoreJob(label, "2018-01-01 01:01:01", db.getId(), db.getFullName(), + jobInfo, false, 3, 100000, + globalStateMgr, repo.getId(), backupMeta, new MvRestoreContext()); + job.setRepo(repo); + Assert.assertEquals(RestoreJobState.PENDING, job.getState()); + + { + new MockUp() { + @Mock + public synchronized QueryStatement init() throws UserException { + return null; + } + }; + job.run(); + } + Assert.assertEquals(RestoreJobState.SNAPSHOTING, job.getState()); + Assert.assertEquals(0, job.getFileMapping().getMapping().size()); + + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(RestoreJobState.DOWNLOAD, job.getState()); + + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(RestoreJobState.DOWNLOADING, job.getState()); + + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(RestoreJobState.COMMIT, job.getState()); + + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(RestoreJobState.COMMITTING, job.getState()); + + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(RestoreJobState.FINISHED, job.getState()); + } +} \ No newline at end of file diff --git a/fe/fe-core/src/test/java/com/starrocks/common/util/UnitTestUtil.java b/fe/fe-core/src/test/java/com/starrocks/common/util/UnitTestUtil.java index 7d00bfd6d4596..c7a7e1c0ae1b4 100644 --- a/fe/fe-core/src/test/java/com/starrocks/common/util/UnitTestUtil.java +++ b/fe/fe-core/src/test/java/com/starrocks/common/util/UnitTestUtil.java @@ -55,6 +55,7 @@ import com.starrocks.catalog.Table; import com.starrocks.catalog.TabletMeta; import com.starrocks.catalog.Type; +import com.starrocks.catalog.View; import com.starrocks.common.jmockit.Deencapsulation; import com.starrocks.system.Backend; import com.starrocks.thrift.TDisk; @@ -76,6 +77,7 @@ public class UnitTestUtil { public static final String MATERIALIZED_VIEW_NAME = "testMV"; public static final String PARTITION_NAME = "testTable"; public static final int SCHEMA_HASH = 0; + public static final String VIEW_NAME = "testView"; public static Database createDb(long dbId, long tableId, long partitionId, long indexId, long tabletId, long backendId, long version, KeysType type) { @@ -249,4 +251,18 @@ public static Class getInnerClass(Class c, String className) { return innerClass; } + public static View createTestView(long tableId) { + List columns = new ArrayList(); + Column temp = new Column("k1", Type.INT); + temp.setIsKey(false); + columns.add(temp); + temp = new Column("k2", Type.INT); + temp.setIsKey(false); + columns.add(temp); + + View view = new View(tableId, VIEW_NAME, columns); + view.setType(Table.TableType.VIEW); + return view; + } + }