diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeJobV2.java index 0a16991731a72..03cd6881ad555 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeJobV2.java @@ -856,7 +856,7 @@ private void onFinished(OlapTable tbl) { } // replace the origin index with shadow index, set index state as NORMAL for (Partition partition : tbl.getPartitions()) { - TStorageMedium medium = tbl.getPartitionInfo().getDataProperty(partition.getParentId()).getStorageMedium(); + TStorageMedium medium = tbl.getPartitionInfo().getDataProperty(partition.getId()).getStorageMedium(); // drop the origin index from partitions for (Map.Entry entry : indexIdMap.entrySet()) { long shadowIdxId = entry.getKey(); diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogRecycleBin.java b/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogRecycleBin.java index 2daa1375b04ac..405eaff8e3eee 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogRecycleBin.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogRecycleBin.java @@ -292,6 +292,14 @@ public synchronized List getPartitions(long tableId) { .collect(Collectors.toList()); } + public synchronized List getPhysicalPartitions(long tableId) { + return idToPartition.values().stream() + .filter(v -> (v.getTableId() == tableId)) + .map(RecyclePartitionInfo::getPartition) + .flatMap(p -> p.getSubPartitions().stream()) + .collect(Collectors.toList()); + } + /** * if we can erase this instance, we should check if anyone enable erase later. * Only used by main loop. diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/Partition.java b/fe/fe-core/src/main/java/com/starrocks/catalog/Partition.java index 733513fd854a2..f7224e4067e6d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/Partition.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/Partition.java @@ -628,8 +628,8 @@ public void setMinRetainVersion(long minRetainVersion) { this.minRetainVersion = minRetainVersion; } - public String generatePhysicalPartitionName(long physicalParitionId) { - return this.name + '_' + physicalParitionId; + public String generatePhysicalPartitionName(long physicalPartitionId) { + return this.name + '_' + physicalPartitionId; } @Override diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/TabletInvertedIndex.java b/fe/fe-core/src/main/java/com/starrocks/catalog/TabletInvertedIndex.java index 86eca94238a19..aa8d5db5eca40 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/TabletInvertedIndex.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/TabletInvertedIndex.java @@ -212,7 +212,7 @@ public void tabletReport(long backendId, Map backendTablets, replica.setLastReportVersion(backendTabletInfo.getVersion()); // check if tablet needs migration - long partitionId = tabletMeta.getPartitionId(); + long partitionId = tabletMeta.getPhysicalPartitionId(); TStorageMedium storageMedium = storageMediumMap.get(partitionId); if (storageMedium != null && backendTabletInfo.isSetStorage_medium()) { if (storageMedium != backendTabletInfo.getStorage_medium()) { @@ -240,7 +240,7 @@ public void tabletReport(long backendId, Map backendTablets, transactionMgr.getTransactionState(tabletMeta.getDbId(), transactionId); if (transactionState == null || transactionState.getTransactionStatus() == TransactionStatus.ABORTED) { - transactionsToClear.put(transactionId, tabletMeta.getPartitionId()); + transactionsToClear.put(transactionId, partitionId); LOG.debug("transaction id [{}] is not valid any more, " + "clear it from backend [{}]", transactionId, backendId); } else if (transactionState.getTransactionStatus() == @@ -267,7 +267,7 @@ public void tabletReport(long backendId, Map backendTablets, transactionState.getTransactionId()); } else { TPartitionVersionInfo versionInfo = - new TPartitionVersionInfo(tabletMeta.getPartitionId(), + new TPartitionVersionInfo(partitionId, partitionCommitInfo.getVersion(), 0); Map> txnMap = transactionsToPublish.computeIfAbsent( @@ -416,11 +416,11 @@ public void checkTabletMetaConsistency(Map creatingTableIds) { } // validate partition - long partitionId = tabletMeta.getPartitionId(); - PhysicalPartition partition = table.getPhysicalPartition(partitionId); - if (partition == null) { - partition = recycleBin.getPhysicalPartition(partitionId); - if (partition != null) { + long partitionId = tabletMeta.getPhysicalPartitionId(); + PhysicalPartition physicalPartition = table.getPhysicalPartition(partitionId); + if (physicalPartition == null) { + physicalPartition = recycleBin.getPhysicalPartition(partitionId); + if (physicalPartition != null) { isInRecycleBin = true; } else { deleteTabletByConsistencyChecker(tabletMeta, tabletId, backendId, @@ -432,7 +432,7 @@ public void checkTabletMetaConsistency(Map creatingTableIds) { // validate index long indexId = tabletMeta.getIndexId(); - MaterializedIndex index = partition.getIndex(indexId); + MaterializedIndex index = physicalPartition.getIndex(indexId); if (index == null) { deleteTabletByConsistencyChecker(tabletMeta, tabletId, backendId, "materialized index " + dbId + "." + tableId + "." + diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/TabletMeta.java b/fe/fe-core/src/main/java/com/starrocks/catalog/TabletMeta.java index 9743b026d0cf7..e0ad98240e9f0 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/TabletMeta.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/TabletMeta.java @@ -41,7 +41,6 @@ public class TabletMeta { private final long dbId; private final long tableId; - private final long partitionId; private final long physicalPartitionId; private final long indexId; @@ -59,11 +58,10 @@ public class TabletMeta { private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); - public TabletMeta(long dbId, long tableId, long partitionId, long physicalPartitionId, long indexId, int schemaHash, + public TabletMeta(long dbId, long tableId, long physicalPartitionId, long indexId, int schemaHash, TStorageMedium storageMedium, boolean isLakeTablet) { this.dbId = dbId; this.tableId = tableId; - this.partitionId = partitionId; this.physicalPartitionId = physicalPartitionId; this.indexId = indexId; @@ -75,15 +73,9 @@ public TabletMeta(long dbId, long tableId, long partitionId, long physicalPartit this.isLakeTablet = isLakeTablet; } - // for single physical partition, the physicalPartitionId is same as partitionId - public TabletMeta(long dbId, long tableId, long partitionId, long indexId, int schemaHash, - TStorageMedium storageMedium, boolean isLakeTablet) { - this(dbId, tableId, partitionId, partitionId, indexId, schemaHash, storageMedium, isLakeTablet); - } - - public TabletMeta(long dbId, long tableId, long partitionId, long indexId, int schemaHash, + public TabletMeta(long dbId, long tableId, long physicalPartitionId, long indexId, int schemaHash, TStorageMedium storageMedium) { - this(dbId, tableId, partitionId, indexId, schemaHash, storageMedium, false); + this(dbId, tableId, physicalPartitionId, indexId, schemaHash, storageMedium, false); } public long getDbId() { @@ -94,10 +86,6 @@ public long getTableId() { return tableId; } - public long getPartitionId() { - return partitionId; - } - public long getPhysicalPartitionId() { return physicalPartitionId; } @@ -164,7 +152,6 @@ public String toString() { StringBuilder sb = new StringBuilder(); sb.append("dbId=").append(dbId); sb.append(" tableId=").append(tableId); - sb.append(" partitionId=").append(partitionId); sb.append(" physicalPartitionId=").append(physicalPartitionId); sb.append(" indexId=").append(indexId); sb.append(" oldSchemaHash=").append(oldSchemaHash); diff --git a/fe/fe-core/src/main/java/com/starrocks/clone/ColocateTableBalancer.java b/fe/fe-core/src/main/java/com/starrocks/clone/ColocateTableBalancer.java index 17405f9f91742..c116664127195 100644 --- a/fe/fe-core/src/main/java/com/starrocks/clone/ColocateTableBalancer.java +++ b/fe/fe-core/src/main/java/com/starrocks/clone/ColocateTableBalancer.java @@ -824,7 +824,7 @@ private ColocateMatchResult doMatchOneGroup(GroupId groupId, TabletSchedCtx.Type.REPAIR, // physical partition id is same as partition id // since colocate table should have only one physical partition - db.getId(), tableId, partition.getId(), partition.getId(), + db.getId(), tableId, partition.getId(), index.getId(), tablet.getId(), System.currentTimeMillis()); // the tablet status will be checked and set again when being scheduled diff --git a/fe/fe-core/src/main/java/com/starrocks/clone/DiskAndTabletLoadReBalancer.java b/fe/fe-core/src/main/java/com/starrocks/clone/DiskAndTabletLoadReBalancer.java index e83c83e801de1..f0025802a79bb 100644 --- a/fe/fe-core/src/main/java/com/starrocks/clone/DiskAndTabletLoadReBalancer.java +++ b/fe/fe-core/src/main/java/com/starrocks/clone/DiskAndTabletLoadReBalancer.java @@ -548,9 +548,13 @@ private List balanceClusterDisk(ClusterLoadStatistic clusterStat if (olapTable == null) { continue; } + PhysicalPartition physicalPartition = olapTable.getPartition(tabletMeta.getPhysicalPartitionId()); + if (physicalPartition == null) { + continue; + } if (isDestBackendLocationMismatch(olapTable, hBackend.getId(), lBackend.getId(), - tabletMeta.getPartitionId(), tabletId)) { + physicalPartition.getParentId(), tabletId)) { continue; } @@ -588,7 +592,7 @@ private List balanceClusterDisk(ClusterLoadStatistic clusterStat hState.minusUsedCapacity(replica.getPathHash(), replica.getDataSize()); TabletSchedCtx schedCtx = new TabletSchedCtx(TabletSchedCtx.Type.BALANCE, - tabletMeta.getDbId(), tabletMeta.getTableId(), tabletMeta.getPartitionId(), + tabletMeta.getDbId(), tabletMeta.getTableId(), tabletMeta.getPhysicalPartitionId(), tabletMeta.getIndexId(), tabletId, System.currentTimeMillis()); schedCtx.setOrigPriority(TabletSchedCtx.Priority.LOW); @@ -803,7 +807,7 @@ private void balanceBackendDisk(TStorageMedium medium, double avgUsedPercent, // NOTICE: state has been changed, the tablet must be selected destPathUsedCap += replica.getDataSize(); srcPathUsedCap -= replica.getDataSize(); - Pair p = Pair.create(tabletMeta.getPartitionId(), tabletMeta.getIndexId()); + Pair p = Pair.create(tabletMeta.getPhysicalPartitionId(), tabletMeta.getIndexId()); // p: partition // k: partition same to p srcPathPartitionTablets.compute(p, (k, pTablets) -> { @@ -822,7 +826,7 @@ private void balanceBackendDisk(TStorageMedium medium, double avgUsedPercent, TabletSchedCtx schedCtx = new TabletSchedCtx(TabletSchedCtx.Type.BALANCE, tabletMeta.getDbId(), - tabletMeta.getTableId(), tabletMeta.getPartitionId(), + tabletMeta.getTableId(), tabletMeta.getPhysicalPartitionId(), tabletMeta.getIndexId(), tabletId, System.currentTimeMillis()); schedCtx.setOrigPriority(TabletSchedCtx.Priority.LOW); @@ -1420,7 +1424,7 @@ private TabletSchedCtx tryToBalanceTablet(Pair> srcTablets, } TabletSchedCtx schedCtx = new TabletSchedCtx(TabletSchedCtx.Type.BALANCE, - tabletMeta.getDbId(), tabletMeta.getTableId(), tabletMeta.getPartitionId(), + tabletMeta.getDbId(), tabletMeta.getTableId(), tabletMeta.getPhysicalPartitionId(), tabletMeta.getIndexId(), tabletId, System.currentTimeMillis()); schedCtx.setOrigPriority(TabletSchedCtx.Priority.LOW); @@ -1544,13 +1548,13 @@ private boolean isTabletUnhealthy(long dbId, OlapTable olapTable, Long tabletId, Locker locker = new Locker(); try { locker.lockDatabase(db, LockType.READ); - PhysicalPartition partition = globalStateMgr.getLocalMetastore() + PhysicalPartition physicalPartition = globalStateMgr.getLocalMetastore() .getPhysicalPartitionIncludeRecycleBin(olapTable, tabletMeta.getPhysicalPartitionId()); - if (partition == null) { + if (physicalPartition == null) { return true; } - MaterializedIndex index = partition.getIndex(tabletMeta.getIndexId()); + MaterializedIndex index = physicalPartition.getIndex(tabletMeta.getIndexId()); if (index == null) { return true; } @@ -1561,7 +1565,7 @@ private boolean isTabletUnhealthy(long dbId, OlapTable olapTable, Long tabletId, } short replicaNum = globalStateMgr.getLocalMetastore() - .getReplicationNumIncludeRecycleBin(olapTable.getPartitionInfo(), partition.getParentId()); + .getReplicationNumIncludeRecycleBin(olapTable.getPartitionInfo(), physicalPartition.getParentId()); if (replicaNum == (short) -1) { return true; } @@ -1570,7 +1574,7 @@ private boolean isTabletUnhealthy(long dbId, OlapTable olapTable, Long tabletId, TabletChecker.getTabletHealthStatusWithPriority( tablet, globalStateMgr.getNodeMgr().getClusterInfo(), - partition.getVisibleVersion(), + physicalPartition.getVisibleVersion(), replicaNum, aliveBeIds, olapTable.getLocation()); diff --git a/fe/fe-core/src/main/java/com/starrocks/clone/TabletChecker.java b/fe/fe-core/src/main/java/com/starrocks/clone/TabletChecker.java index 9bcb5417424c7..65a1b8770abd4 100644 --- a/fe/fe-core/src/main/java/com/starrocks/clone/TabletChecker.java +++ b/fe/fe-core/src/main/java/com/starrocks/clone/TabletChecker.java @@ -303,11 +303,11 @@ private void doCheck(boolean isUrgent) { } OlapTable olapTbl = (OlapTable) table; - for (Partition partition : GlobalStateMgr.getCurrentState().getLocalMetastore() - .getAllPartitionsIncludeRecycleBin(olapTbl)) { + for (PhysicalPartition physicalPartition : GlobalStateMgr.getCurrentState().getLocalMetastore() + .getAllPhysicalPartitionsIncludeRecycleBin(olapTbl)) { partitionChecked++; - boolean isPartitionUrgent = isPartitionUrgent(dbId, table.getId(), partition.getId()); + boolean isPartitionUrgent = isPartitionUrgent(dbId, table.getId(), physicalPartition.getId()); totStat.isUrgentPartitionHealthy = true; if ((isUrgent && !isPartitionUrgent) || (!isUrgent && isPartitionUrgent)) { continue; @@ -328,13 +328,18 @@ private void doCheck(boolean isUrgent) { .getTableIncludeRecycleBin(db, olapTbl.getId()) == null) { continue TABLE; } - if (GlobalStateMgr.getCurrentState() - .getLocalMetastore().getPartitionIncludeRecycleBin(olapTbl, partition.getId()) == null) { + if (GlobalStateMgr.getCurrentState().getLocalMetastore() + .getPhysicalPartitionIncludeRecycleBin(olapTbl, physicalPartition.getId()) == null) { continue; } } - if (partition.getState() != PartitionState.NORMAL) { + Partition logicalPartition = olapTbl.getPartition(physicalPartition.getParentId()); + if (logicalPartition == null) { + continue; + } + + if (logicalPartition.getState() != PartitionState.NORMAL) { // when alter job is in FINISHING state, partition state will be set to NORMAL, // and we can schedule the tablets in it. continue; @@ -342,12 +347,13 @@ private void doCheck(boolean isUrgent) { short replicaNum = GlobalStateMgr.getCurrentState() .getLocalMetastore() - .getReplicationNumIncludeRecycleBin(olapTbl.getPartitionInfo(), partition.getId()); + .getReplicationNumIncludeRecycleBin( + olapTbl.getPartitionInfo(), physicalPartition.getParentId()); if (replicaNum == (short) -1) { continue; } - TabletCheckerStat partitionTabletCheckerStat = doCheckOnePartition(db, olapTbl, partition, + TabletCheckerStat partitionTabletCheckerStat = doCheckOnePartition(db, olapTbl, physicalPartition, replicaNum, aliveBeIdsInCluster, isPartitionUrgent); totStat.accumulateStat(partitionTabletCheckerStat); @@ -355,9 +361,9 @@ private void doCheck(boolean isUrgent) { // if all replicas in this partition are healthy, remove this partition from // priorities. LOG.debug("partition is healthy, remove from urgent table: {}-{}-{}", - db.getId(), olapTbl.getId(), partition.getId()); + db.getId(), olapTbl.getId(), physicalPartition.getId()); removeFromUrgentTable(new RepairTabletInfo(db.getId(), - olapTbl.getId(), Lists.newArrayList(partition.getId()))); + olapTbl.getId(), Lists.newArrayList(physicalPartition.getId()))); } } // partitions } // tables @@ -402,12 +408,12 @@ public void accumulateStat(TabletCheckerStat stat) { } } - private TabletCheckerStat doCheckOnePartition(Database db, OlapTable olapTbl, Partition partition, + private TabletCheckerStat doCheckOnePartition(Database db, OlapTable olapTbl, PhysicalPartition physicalPartition, int replicaNum, List aliveBeIdsInCluster, boolean isPartitionUrgent) { TabletCheckerStat partitionTabletCheckerStat = new TabletCheckerStat(); // Tablet in SHADOW index can not be repaired or balanced - for (PhysicalPartition physicalPartition : partition.getSubPartitions()) { + if (physicalPartition != null) { for (MaterializedIndex idx : physicalPartition.getMaterializedIndices( IndexExtState.VISIBLE)) { for (Tablet tablet : idx.getTablets()) { @@ -452,7 +458,7 @@ private TabletCheckerStat doCheckOnePartition(Database db, OlapTable olapTbl, Pa TabletSchedCtx tabletSchedCtx = new TabletSchedCtx( TabletSchedCtx.Type.REPAIR, - db.getId(), olapTbl.getId(), partition.getId(), + db.getId(), olapTbl.getId(), physicalPartition.getId(), idx.getId(), tablet.getId(), System.currentTimeMillis()); // the tablet status will be set again when being scheduled @@ -525,7 +531,7 @@ private void cleanInvalidUrgentTable() { } Set parts = tblEntry.getValue(); - parts = parts.stream().filter(p -> (tbl.getPartition(p.partId) != null && !p.isTimeout())).collect( + parts = parts.stream().filter(p -> (tbl.getPhysicalPartition(p.partId) != null && !p.isTimeout())).collect( Collectors.toSet()); if (parts.isEmpty()) { deletedUrgentTable.add(Pair.create(dbId, tblId)); @@ -629,14 +635,15 @@ public static RepairTabletInfo getRepairTabletInfo(String dbName, String tblName OlapTable olapTable = (OlapTable) tbl; if (partitions == null || partitions.isEmpty()) { - partIds = olapTable.getPartitions().stream().map(Partition::getId).collect(Collectors.toList()); + partIds = olapTable.getPhysicalPartitions().stream().map(PhysicalPartition::getId).collect(Collectors.toList()); } else { for (String partName : partitions) { Partition partition = olapTable.getPartition(partName); if (partition == null) { throw new DdlException("Partition does not exist: " + partName); } - partIds.add(partition.getId()); + partIds.addAll(partition.getSubPartitions().stream() + .map(PhysicalPartition::getId).collect(Collectors.toList())); } } } finally { diff --git a/fe/fe-core/src/main/java/com/starrocks/clone/TabletSchedCtx.java b/fe/fe-core/src/main/java/com/starrocks/clone/TabletSchedCtx.java index 08ab413764260..b6a17b449f3d2 100644 --- a/fe/fe-core/src/main/java/com/starrocks/clone/TabletSchedCtx.java +++ b/fe/fe-core/src/main/java/com/starrocks/clone/TabletSchedCtx.java @@ -179,7 +179,6 @@ public enum State { private final long dbId; private final long tblId; - private final long partitionId; private final long physicalPartitionId; private final long indexId; private final long tabletId; @@ -231,12 +230,11 @@ public enum State { */ private int replicaNum; - public TabletSchedCtx(Type type, long dbId, long tblId, long partId, long physicalPartitionId, + public TabletSchedCtx(Type type, long dbId, long tblId, long physicalPartitionId, long idxId, long tabletId, long createTime) { this.type = type; this.dbId = dbId; this.tblId = tblId; - this.partitionId = partId; this.physicalPartitionId = physicalPartitionId; this.indexId = idxId; this.tabletId = tabletId; @@ -246,19 +244,12 @@ public TabletSchedCtx(Type type, long dbId, long tblId, long partId, long physic } @VisibleForTesting - public TabletSchedCtx(Type type, long dbId, long tblId, long partId, - long idxId, long tabletId, long createTime) { - this(type, dbId, tblId, partId, partId, idxId, tabletId, createTime); - } - - @VisibleForTesting - public TabletSchedCtx(Type type, long dbId, long tblId, long partId, + public TabletSchedCtx(Type type, long dbId, long tblId, long physicalPartitionId, long idxId, long tabletId, long createTime, SystemInfoService infoService) { this.type = type; this.dbId = dbId; this.tblId = tblId; - this.partitionId = partId; - this.physicalPartitionId = partId; + this.physicalPartitionId = physicalPartitionId; this.indexId = idxId; this.tabletId = tabletId; this.createTime = createTime; @@ -335,10 +326,6 @@ public long getTblId() { return tblId; } - public long getPartitionId() { - return partitionId; - } - public long getPhysicalPartitionId() { return physicalPartitionId; } @@ -936,6 +923,10 @@ public CreateReplicaTask createEmptyReplicaAndTask() throws SchedException { if (olapTable == null) { throw new SchedException(Status.UNRECOVERABLE, "table " + tblId + " does not exist"); } + PhysicalPartition physicalPartition = olapTable.getPhysicalPartition(physicalPartitionId); + if (physicalPartition == null) { + throw new SchedException(Status.UNRECOVERABLE, "physical partition " + physicalPartitionId + " does not exist"); + } MaterializedIndexMeta indexMeta = olapTable.getIndexMetaByIndexId(indexId); if (indexMeta == null) { throw new SchedException(Status.UNRECOVERABLE, "materialized view " + indexId + " does not exist"); @@ -966,7 +957,7 @@ public CreateReplicaTask createEmptyReplicaAndTask() throws SchedException { .setStorageMedium(TStorageMedium.HDD) .setEnablePersistentIndex(olapTable.enablePersistentIndex()) .setPrimaryIndexCacheExpireSec(olapTable.primaryIndexCacheExpireSec()) - .setTabletType(olapTable.getPartitionInfo().getTabletType(partitionId)) + .setTabletType(olapTable.getPartitionInfo().getTabletType(physicalPartition.getParentId())) .setCompressionType(olapTable.getCompressionType()) .setCompressionLevel(olapTable.getCompressionLevel()) .setRecoverySource(RecoverySource.SCHEDULER) @@ -1055,7 +1046,7 @@ public void finishCloneTask(CloneTask cloneTask, TFinishTaskRequest request) short replicationNum = globalStateMgr.getLocalMetastore() - .getReplicationNumIncludeRecycleBin(olapTable.getPartitionInfo(), partitionId); + .getReplicationNumIncludeRecycleBin(olapTable.getPartitionInfo(), partition.getParentId()); if (replicationNum == (short) -1) { throw new SchedException(Status.UNRECOVERABLE, "invalid replication number"); } diff --git a/fe/fe-core/src/main/java/com/starrocks/clone/TabletScheduler.java b/fe/fe-core/src/main/java/com/starrocks/clone/TabletScheduler.java index 16b124b84ef64..516280faae9db 100644 --- a/fe/fe-core/src/main/java/com/starrocks/clone/TabletScheduler.java +++ b/fe/fe-core/src/main/java/com/starrocks/clone/TabletScheduler.java @@ -413,7 +413,7 @@ public synchronized void changeTabletsPriorityToVeryHigh(long dbId, long tblId, PriorityQueue newPendingTablets = new PriorityQueue<>(); for (TabletSchedCtx tabletCtx : pendingTablets) { if (tabletCtx.getDbId() == dbId && tabletCtx.getTblId() == tblId - && partitionIds.contains(tabletCtx.getPartitionId())) { + && partitionIds.contains(tabletCtx.getPhysicalPartitionId())) { tabletCtx.setOrigPriority(Priority.VERY_HIGH); } newPendingTablets.add(tabletCtx); @@ -554,9 +554,10 @@ protected boolean checkIfTabletExpired(TabletSchedCtx ctx, CatalogRecycleBin rec LOG.warn("discard ctx because table {} will erase soon: {}", tableId, ctx); return true; } - long partitionId = ctx.getPartitionId(); - if (recycleBin.getPartition(partitionId) != null - && !recycleBin.ensureEraseLater(partitionId, currentTimeMs)) { + long partitionId = ctx.getPhysicalPartitionId(); + PhysicalPartition physicalPartition = recycleBin.getPhysicalPartition(partitionId); + if (physicalPartition != null + && !recycleBin.ensureEraseLater(physicalPartition.getParentId(), currentTimeMs)) { LOG.warn("discard ctx because partition {} will erase soon: {}", partitionId, ctx); return true; } @@ -713,30 +714,32 @@ private void scheduleTablet(TabletSchedCtx tabletCtx, AgentBatchTask batchTask) OlapTableState tableState = tbl.getState(); - Partition partition = GlobalStateMgr.getCurrentState() - .getLocalMetastore().getPartitionIncludeRecycleBin(tbl, tabletCtx.getPartitionId()); - if (partition == null) { - throw new SchedException(Status.UNRECOVERABLE, "partition does not exist"); + PhysicalPartition physicalPartition = GlobalStateMgr.getCurrentState() + .getLocalMetastore().getPhysicalPartitionIncludeRecycleBin(tbl, tabletCtx.getPhysicalPartitionId()); + if (physicalPartition == null) { + throw new SchedException(Status.UNRECOVERABLE, "physical partition " + + tabletCtx.getPhysicalPartitionId() + "does not exist"); } - short replicaNum = GlobalStateMgr.getCurrentState() - .getLocalMetastore().getReplicationNumIncludeRecycleBin(tbl.getPartitionInfo(), partition.getId()); + Partition logicalPartition = GlobalStateMgr.getCurrentState().getLocalMetastore() + .getPartitionIncludeRecycleBin(tbl, physicalPartition.getParentId()); + if (logicalPartition == null) { + throw new SchedException(Status.UNRECOVERABLE, "partition " + + physicalPartition.getParentId() + "does not exist"); + } + + short replicaNum = GlobalStateMgr.getCurrentState().getLocalMetastore() + .getReplicationNumIncludeRecycleBin(tbl.getPartitionInfo(), physicalPartition.getParentId()); if (replicaNum == (short) -1) { throw new SchedException(Status.UNRECOVERABLE, "invalid replication number"); } DataProperty dataProperty = GlobalStateMgr.getCurrentState().getLocalMetastore() - .getDataPropertyIncludeRecycleBin(tbl.getPartitionInfo(), partition.getId()); + .getDataPropertyIncludeRecycleBin(tbl.getPartitionInfo(), physicalPartition.getParentId()); if (dataProperty == null) { throw new SchedException(Status.UNRECOVERABLE, "partition data property not exist"); } - PhysicalPartition physicalPartition = partition.getSubPartition(tabletCtx.getPhysicalPartitionId()); - if (physicalPartition == null) { - throw new SchedException(Status.UNRECOVERABLE, "physical partition " - + tabletCtx.getPhysicalPartitionId() + "does not exist"); - } - MaterializedIndex idx = physicalPartition.getIndex(tabletCtx.getIndexId()); if (idx == null) { throw new SchedException(Status.UNRECOVERABLE, "index does not exist"); @@ -786,7 +789,7 @@ private void scheduleTablet(TabletSchedCtx tabletCtx, AgentBatchTask batchTask) } if (statusPair.first != TabletHealthStatus.VERSION_INCOMPLETE - && (partition.getState() != PartitionState.NORMAL || tableState != OlapTableState.NORMAL) + && (logicalPartition.getState() != PartitionState.NORMAL || tableState != OlapTableState.NORMAL) && tableState != OlapTableState.WAITING_STABLE) { // If table is under ALTER process(before FINISHING), do not allow to add or delete replica. // VERSION_INCOMPLETE will repair the replica in place, which is allowed. @@ -2058,7 +2061,7 @@ public TGetTabletScheduleResponse getTabletSchedule(TGetTabletScheduleRequest re if (tabletId != -1) { all = all.filter(t -> t.getTabletId() == tabletId); } else if (partitionId != -1) { - all = all.filter(t -> t.getPartitionId() == partitionId); + all = all.filter(t -> t.getPhysicalPartitionId() == partitionId); } else if (tableId != -1) { all = all.filter(t -> t.getTblId() == tableId); } @@ -2086,16 +2089,10 @@ private void checkMetaExist(TabletSchedCtx ctx) throws SchedException { throw new SchedException(Status.UNRECOVERABLE, "table " + ctx.getTblId() + " dose not exist"); } - Partition partition = - GlobalStateMgr.getCurrentState().getLocalMetastore().getPartitionIncludeRecycleBin(tbl, ctx.getPartitionId()); - if (partition == null) { - throw new SchedException(Status.UNRECOVERABLE, "partition " + ctx.getPartitionId() + " dose not exist"); - } - - PhysicalPartition physicalPartition = partition.getSubPartition(ctx.getPhysicalPartitionId()); + PhysicalPartition physicalPartition = GlobalStateMgr.getCurrentState().getLocalMetastore() + .getPhysicalPartitionIncludeRecycleBin(tbl, ctx.getPhysicalPartitionId()); if (physicalPartition == null) { - throw new SchedException(Status.UNRECOVERABLE, - "physical partition " + ctx.getPhysicalPartitionId() + " dose not exist"); + throw new SchedException(Status.UNRECOVERABLE, "partition " + ctx.getPhysicalPartitionId() + " dose not exist"); } MaterializedIndex idx = physicalPartition.getIndex(ctx.getIndexId()); diff --git a/fe/fe-core/src/main/java/com/starrocks/common/proc/StatisticProcDir.java b/fe/fe-core/src/main/java/com/starrocks/common/proc/StatisticProcDir.java index 64e79600ddabd..3422a23651149 100644 --- a/fe/fe-core/src/main/java/com/starrocks/common/proc/StatisticProcDir.java +++ b/fe/fe-core/src/main/java/com/starrocks/common/proc/StatisticProcDir.java @@ -152,8 +152,8 @@ public ProcResult fetchResult() throws AnalysisException { for (Partition partition : olapTable.getAllPartitions()) { short replicationNum = olapTable.getPartitionInfo().getReplicationNum(partition.getId()); ++dbPartitionNum; - for (PhysicalPartition physicalParition : partition.getSubPartitions()) { - for (MaterializedIndex materializedIndex : physicalParition + for (PhysicalPartition physicalPartition : partition.getSubPartitions()) { + for (MaterializedIndex materializedIndex : physicalPartition .getMaterializedIndices(IndexExtState.VISIBLE)) { ++dbIndexNum; for (Tablet tablet : materializedIndex.getTablets()) { @@ -170,7 +170,7 @@ public ProcResult fetchResult() throws AnalysisException { } Pair res = TabletChecker.getTabletHealthStatusWithPriority( - localTablet, infoService, physicalParition.getVisibleVersion(), + localTablet, infoService, physicalPartition.getVisibleVersion(), replicationNum, aliveBeIdsInCluster, olapTable.getLocation()); // here we treat REDUNDANT as HEALTHY, for user-friendly. diff --git a/fe/fe-core/src/main/java/com/starrocks/consistency/CheckConsistencyJob.java b/fe/fe-core/src/main/java/com/starrocks/consistency/CheckConsistencyJob.java index 7e42d9d2512ff..59179f1b71784 100644 --- a/fe/fe-core/src/main/java/com/starrocks/consistency/CheckConsistencyJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/consistency/CheckConsistencyJob.java @@ -40,8 +40,9 @@ import com.starrocks.catalog.Database; import com.starrocks.catalog.LocalTablet; import com.starrocks.catalog.MaterializedIndex; +import com.starrocks.catalog.MetaObject; import com.starrocks.catalog.OlapTable; -import com.starrocks.catalog.Partition; +import com.starrocks.catalog.PhysicalPartition; import com.starrocks.catalog.Replica; import com.starrocks.catalog.Replica.ReplicaState; import com.starrocks.catalog.Table; @@ -147,20 +148,20 @@ public boolean sendTasks() { LockType.READ)) { OlapTable olapTable = (OlapTable) table; - Partition partition = olapTable.getPartition(tabletMeta.getPartitionId()); - if (partition == null) { - LOG.debug("partition[{}] does not exist", tabletMeta.getPartitionId()); + PhysicalPartition physicalPartition = olapTable.getPartition(tabletMeta.getPhysicalPartitionId()); + if (physicalPartition == null) { + LOG.debug("partition[{}] does not exist", tabletMeta.getPhysicalPartitionId()); return false; } // check partition's replication num. if 1 replication. skip - short replicationNum = olapTable.getPartitionInfo().getReplicationNum(partition.getId()); + short replicationNum = olapTable.getPartitionInfo().getReplicationNum(physicalPartition.getParentId()); if (replicationNum == (short) 1) { - LOG.debug("partition[{}]'s replication num is 1. skip consistency check", partition.getId()); + LOG.debug("partition[{}]'s replication num is 1. skip consistency check", physicalPartition.getParentId()); return false; } - MaterializedIndex index = partition.getIndex(tabletMeta.getIndexId()); + MaterializedIndex index = physicalPartition.getIndex(tabletMeta.getIndexId()); if (index == null) { LOG.debug("index[{}] does not exist", tabletMeta.getIndexId()); return false; @@ -172,7 +173,7 @@ public boolean sendTasks() { return false; } - checkedVersion = partition.getVisibleVersion(); + checkedVersion = physicalPartition.getVisibleVersion(); checkedSchemaHash = olapTable.getSchemaHashByIndexId(tabletMeta.getIndexId()); int sentTaskReplicaNum = 0; @@ -191,7 +192,7 @@ public boolean sendTasks() { CheckConsistencyTask task = new CheckConsistencyTask(null, replica.getBackendId(), tabletMeta.getDbId(), tabletMeta.getTableId(), - tabletMeta.getPartitionId(), + tabletMeta.getPhysicalPartitionId(), tabletMeta.getIndexId(), tabletId, checkedSchemaHash, checkedVersion); @@ -272,13 +273,13 @@ public synchronized int tryFinishJob() { new AutoCloseableLock(new Locker(), db, Lists.newArrayList(table.getId()), LockType.WRITE)) { OlapTable olapTable = (OlapTable) table; - Partition partition = olapTable.getPartition(tabletMeta.getPartitionId()); - if (partition == null) { - LOG.warn("partition[{}] does not exist", tabletMeta.getPartitionId()); + PhysicalPartition physicalPartition = olapTable.getPartition(tabletMeta.getPhysicalPartitionId()); + if (physicalPartition == null) { + LOG.warn("partition[{}] does not exist", tabletMeta.getPhysicalPartitionId()); return -1; } - MaterializedIndex index = partition.getIndex(tabletMeta.getIndexId()); + MaterializedIndex index = physicalPartition.getIndex(tabletMeta.getIndexId()); if (index == null) { LOG.warn("index[{}] does not exist", tabletMeta.getIndexId()); return -1; @@ -356,7 +357,9 @@ public synchronized int tryFinishJob() { long lastCheckTime = System.currentTimeMillis(); db.setLastCheckTime(lastCheckTime); olapTable.setLastCheckTime(lastCheckTime); - partition.setLastCheckTime(lastCheckTime); + if (physicalPartition instanceof MetaObject) { + ((MetaObject) physicalPartition).setLastCheckTime(lastCheckTime); + } index.setLastCheckTime(lastCheckTime); tablet.setLastCheckTime(lastCheckTime); tablet.setIsConsistent(isConsistent); @@ -365,7 +368,7 @@ public synchronized int tryFinishJob() { tablet.setCheckedVersion(checkedVersion); // log - ConsistencyCheckInfo info = new ConsistencyCheckInfo(db.getId(), table.getId(), partition.getId(), + ConsistencyCheckInfo info = new ConsistencyCheckInfo(db.getId(), table.getId(), physicalPartition.getId(), index.getId(), tabletId, lastCheckTime, checkedVersion, isConsistent); journalTask = GlobalStateMgr.getCurrentState().getEditLog().logFinishConsistencyCheckNoWait(info); diff --git a/fe/fe-core/src/main/java/com/starrocks/consistency/ConsistencyChecker.java b/fe/fe-core/src/main/java/com/starrocks/consistency/ConsistencyChecker.java index 9aaf27a0ecab1..7c74b361e3b4f 100644 --- a/fe/fe-core/src/main/java/com/starrocks/consistency/ConsistencyChecker.java +++ b/fe/fe-core/src/main/java/com/starrocks/consistency/ConsistencyChecker.java @@ -320,32 +320,32 @@ protected List chooseTablets() { // sort partitions Queue partitionQueue = new PriorityQueue<>(Math.max(table.getAllPhysicalPartitions().size(), 1), COMPARATOR); - for (PhysicalPartition partition : table.getPhysicalPartitions()) { + for (PhysicalPartition physicalPartition : table.getPhysicalPartitions()) { // check partition's replication num. if 1 replication. skip - if (table.getPartitionInfo().getReplicationNum(partition.getParentId()) == (short) 1) { - LOG.debug("partition[{}]'s replication num is 1. ignore", partition.getParentId()); + if (table.getPartitionInfo().getReplicationNum(physicalPartition.getParentId()) == (short) 1) { + LOG.debug("partition[{}]'s replication num is 1. ignore", physicalPartition.getParentId()); continue; } // check if this partition has no data - if (partition.getVisibleVersion() == Partition.PARTITION_INIT_VERSION) { - LOG.debug("partition[{}]'s version is {}. ignore", partition.getId(), + if (physicalPartition.getVisibleVersion() == Partition.PARTITION_INIT_VERSION) { + LOG.debug("partition[{}]'s version is {}. ignore", physicalPartition.getId(), Partition.PARTITION_INIT_VERSION); continue; } - if (partition instanceof Partition) { - partitionQueue.add((Partition) partition); - } else if (partition instanceof PhysicalPartitionImpl) { - partitionQueue.add((PhysicalPartitionImpl) partition); + if (physicalPartition instanceof Partition) { + partitionQueue.add((Partition) physicalPartition); + } else if (physicalPartition instanceof PhysicalPartitionImpl) { + partitionQueue.add((PhysicalPartitionImpl) physicalPartition); } } while ((chosenOne = partitionQueue.poll()) != null) { - PhysicalPartition partition = (PhysicalPartition) chosenOne; + PhysicalPartition physicalPartition = (PhysicalPartition) chosenOne; // sort materializedIndices List visibleIndexes = - partition.getMaterializedIndices(IndexExtState.VISIBLE); + physicalPartition.getMaterializedIndices(IndexExtState.VISIBLE); Queue indexQueue = new PriorityQueue<>(Math.max(visibleIndexes.size(), 1), COMPARATOR); indexQueue.addAll(visibleIndexes); @@ -367,15 +367,15 @@ protected List chooseTablets() { } // check if version has already been checked - if (partition.getVisibleVersion() == tablet.getCheckedVersion()) { + if (physicalPartition.getVisibleVersion() == tablet.getCheckedVersion()) { if (tablet.isConsistent()) { LOG.debug("tablet[{}]'s version[{}-{}] has been checked. ignore", chosenTabletId, tablet.getCheckedVersion(), - partition.getVisibleVersion()); + physicalPartition.getVisibleVersion()); } } else { LOG.info("chose tablet[{}-{}-{}-{}-{}] to check consistency", db.getId(), - table.getId(), partition.getId(), index.getId(), chosenTabletId); + table.getId(), physicalPartition.getId(), index.getId(), chosenTabletId); chosenTablets.add(chosenTabletId); } @@ -429,12 +429,12 @@ public void replayFinishConsistencyCheck(ConsistencyCheckInfo info, GlobalStateM try (AutoCloseableLock ignore = new AutoCloseableLock(new Locker(), db, Lists.newArrayList(table.getId()), LockType.WRITE)) { - Partition partition = table.getPartition(info.getPartitionId()); - if (partition == null) { + PhysicalPartition physicalPartition = table.getPhysicalPartition(info.getPhysicalPartitionId()); + if (physicalPartition == null) { LOG.warn("replay finish consistency check failed, partition is null, info: {}", info); return; } - MaterializedIndex index = partition.getIndex(info.getIndexId()); + MaterializedIndex index = physicalPartition.getIndex(info.getIndexId()); if (index == null) { LOG.warn("replay finish consistency check failed, index is null, info: {}", info); return; @@ -448,7 +448,9 @@ public void replayFinishConsistencyCheck(ConsistencyCheckInfo info, GlobalStateM long lastCheckTime = info.getLastCheckTime(); db.setLastCheckTime(lastCheckTime); table.setLastCheckTime(lastCheckTime); - partition.setLastCheckTime(lastCheckTime); + if (physicalPartition instanceof MetaObject) { + ((MetaObject) physicalPartition).setLastCheckTime(lastCheckTime); + } index.setLastCheckTime(lastCheckTime); tablet.setLastCheckTime(lastCheckTime); tablet.setCheckedVersion(info.getCheckedVersion()); diff --git a/fe/fe-core/src/main/java/com/starrocks/lake/Utils.java b/fe/fe-core/src/main/java/com/starrocks/lake/Utils.java index 14ed7713e35b6..1cdf0c03eccac 100644 --- a/fe/fe-core/src/main/java/com/starrocks/lake/Utils.java +++ b/fe/fe-core/src/main/java/com/starrocks/lake/Utils.java @@ -82,8 +82,8 @@ public static Map> groupTabletID(Collection partitio Map> groupMap = new HashMap<>(); for (Partition partition : partitions) { - for (PhysicalPartition physicalParition : partition.getSubPartitions()) { - for (MaterializedIndex index : physicalParition.getMaterializedIndices(indexState)) { + for (PhysicalPartition physicalPartition : partition.getSubPartitions()) { + for (MaterializedIndex index : physicalPartition.getMaterializedIndices(indexState)) { for (Tablet tablet : index.getTablets()) { ComputeNode computeNode = warehouseManager.getComputeNodeAssignedToTablet( warehouseId, (LakeTablet) tablet); diff --git a/fe/fe-core/src/main/java/com/starrocks/leader/LeaderImpl.java b/fe/fe-core/src/main/java/com/starrocks/leader/LeaderImpl.java index a65d7ff1797a3..f43741001f053 100644 --- a/fe/fe-core/src/main/java/com/starrocks/leader/LeaderImpl.java +++ b/fe/fe-core/src/main/java/com/starrocks/leader/LeaderImpl.java @@ -1117,7 +1117,7 @@ private static void initTabletMeta(Tablet tablet, TTabletMeta tTabletMeta) { TabletMeta tabletMeta = GlobalStateMgr.getCurrentState().getTabletInvertedIndex().getTabletMeta(tablet.getId()); tTabletMeta.setDb_id(tabletMeta.getDbId()); tTabletMeta.setTable_id(tabletMeta.getTableId()); - tTabletMeta.setPartition_id(tabletMeta.getPartitionId()); + tTabletMeta.setPartition_id(tabletMeta.getPhysicalPartitionId()); tTabletMeta.setIndex_id(tabletMeta.getIndexId()); tTabletMeta.setStorage_medium(tabletMeta.getStorageMedium()); tTabletMeta.setOld_schema_hash(tabletMeta.getOldSchemaHash()); diff --git a/fe/fe-core/src/main/java/com/starrocks/leader/ReportHandler.java b/fe/fe-core/src/main/java/com/starrocks/leader/ReportHandler.java index 9ba100054bcf4..056e0eb5f02d8 100644 --- a/fe/fe-core/src/main/java/com/starrocks/leader/ReportHandler.java +++ b/fe/fe-core/src/main/java/com/starrocks/leader/ReportHandler.java @@ -848,11 +848,6 @@ private static void deleteFromMeta(ListMultimap tabletDeleteFromMeta continue; } - if (globalStateMgr.getLocalMetastore() - .getPartitionIncludeRecycleBin(olapTable, tabletMeta.getPartitionId()) == null) { - continue; - } - PhysicalPartition partition = globalStateMgr.getLocalMetastore() .getPhysicalPartitionIncludeRecycleBin(olapTable, partitionId); if (partition == null) { @@ -861,7 +856,7 @@ private static void deleteFromMeta(ListMultimap tabletDeleteFromMeta short replicationNum = globalStateMgr.getLocalMetastore().getReplicationNumIncludeRecycleBin(olapTable.getPartitionInfo(), - tabletMeta.getPartitionId()); + partition.getParentId()); if (replicationNum == (short) -1) { continue; } @@ -1396,7 +1391,7 @@ private static void handleSetTabletInMemory(long backendId, Map b long dbId = tabletMeta != null ? tabletMeta.getDbId() : TabletInvertedIndex.NOT_EXIST_VALUE; long tableId = tabletMeta != null ? tabletMeta.getTableId() : TabletInvertedIndex.NOT_EXIST_VALUE; long partitionId = - tabletMeta != null ? tabletMeta.getPartitionId() : TabletInvertedIndex.NOT_EXIST_VALUE; + tabletMeta != null ? tabletMeta.getPhysicalPartitionId() : TabletInvertedIndex.NOT_EXIST_VALUE; Database db = GlobalStateMgr.getCurrentState().getDb(dbId); if (db == null) { @@ -1411,11 +1406,11 @@ private static void handleSetTabletInMemory(long backendId, Map b Locker locker = new Locker(); locker.lockTablesWithIntensiveDbLock(db, Lists.newArrayList(olapTable.getId()), LockType.READ); try { - Partition partition = olapTable.getPartition(partitionId); + PhysicalPartition partition = olapTable.getPhysicalPartition(partitionId); if (partition == null) { continue; } - boolean feIsInMemory = olapTable.getPartitionInfo().getIsInMemory(partitionId); + boolean feIsInMemory = olapTable.getPartitionInfo().getIsInMemory(partition.getParentId()); if (beIsInMemory != feIsInMemory) { tabletToInMemory.add(new Pair<>(tabletId, feIsInMemory)); } @@ -1764,7 +1759,6 @@ private static void addReplica(long tabletId, TTabletInfo backendTabletInfo, lon TabletMeta tabletMeta = invertedIndex.getTabletMeta(tabletId); long dbId = tabletMeta != null ? tabletMeta.getDbId() : TabletInvertedIndex.NOT_EXIST_VALUE; long tableId = tabletMeta != null ? tabletMeta.getTableId() : TabletInvertedIndex.NOT_EXIST_VALUE; - long partitionId = tabletMeta != null ? tabletMeta.getPartitionId() : TabletInvertedIndex.NOT_EXIST_VALUE; long physicalPartitionId = tabletMeta != null ? tabletMeta.getPhysicalPartitionId() : TabletInvertedIndex.NOT_EXIST_VALUE; long indexId = tabletMeta != null ? tabletMeta.getIndexId() : TabletInvertedIndex.NOT_EXIST_VALUE; @@ -1785,22 +1779,22 @@ private static void addReplica(long tabletId, TTabletInfo backendTabletInfo, lon Locker locker = new Locker(); locker.lockTablesWithIntensiveDbLock(db, Lists.newArrayList(olapTable.getId()), LockType.WRITE); try { - if (globalStateMgr.getLocalMetastore().getPartitionIncludeRecycleBin(olapTable, partitionId) == null) { - throw new MetaNotFoundException("partition[" + partitionId + "] does not exist"); - } - short replicationNum = - globalStateMgr.getLocalMetastore() - .getReplicationNumIncludeRecycleBin(olapTable.getPartitionInfo(), partitionId); - if (replicationNum == (short) -1) { - throw new MetaNotFoundException("invalid replication number of partition [" + partitionId + "]"); - } - PhysicalPartition partition = globalStateMgr.getLocalMetastore() .getPhysicalPartitionIncludeRecycleBin(olapTable, physicalPartitionId); if (partition == null) { throw new MetaNotFoundException("physical partition[" + physicalPartitionId + "] does not exist"); } + if (globalStateMgr.getLocalMetastore().getPartitionIncludeRecycleBin(olapTable, partition.getParentId()) == null) { + throw new MetaNotFoundException("partition[" + partition.getParentId() + "] does not exist"); + } + short replicationNum = + globalStateMgr.getLocalMetastore() + .getReplicationNumIncludeRecycleBin(olapTable.getPartitionInfo(), partition.getParentId()); + if (replicationNum == (short) -1) { + throw new MetaNotFoundException("invalid replication number of partition [" + partition.getParentId() + "]"); + } + MaterializedIndex materializedIndex = partition.getIndex(indexId); if (materializedIndex == null) { throw new MetaNotFoundException("index[" + indexId + "] does not exist"); diff --git a/fe/fe-core/src/main/java/com/starrocks/persist/ConsistencyCheckInfo.java b/fe/fe-core/src/main/java/com/starrocks/persist/ConsistencyCheckInfo.java index 1c2b3a0b20d4a..0b01f9c62790c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/persist/ConsistencyCheckInfo.java +++ b/fe/fe-core/src/main/java/com/starrocks/persist/ConsistencyCheckInfo.java @@ -49,7 +49,7 @@ public class ConsistencyCheckInfo implements Writable { @SerializedName("tb") private long tableId; @SerializedName("pt") - private long partitionId; + private long physicalPartitionId; @SerializedName("idx") private long indexId; @SerializedName("tt") @@ -65,12 +65,12 @@ public ConsistencyCheckInfo() { // for persist } - public ConsistencyCheckInfo(long dbId, long tableId, long partitionId, long indexId, long tabletId, + public ConsistencyCheckInfo(long dbId, long tableId, long physicalPartitionId, long indexId, long tabletId, long lastCheckTime, long checkedVersion, boolean isConsistent) { this.dbId = dbId; this.tableId = tableId; - this.partitionId = partitionId; + this.physicalPartitionId = physicalPartitionId; this.indexId = indexId; this.tabletId = tabletId; @@ -88,8 +88,8 @@ public long getTableId() { return tableId; } - public long getPartitionId() { - return partitionId; + public long getPhysicalPartitionId() { + return physicalPartitionId; } public long getIndexId() { @@ -116,7 +116,7 @@ public boolean isConsistent() { public void write(DataOutput out) throws IOException { out.writeLong(dbId); out.writeLong(tableId); - out.writeLong(partitionId); + out.writeLong(physicalPartitionId); out.writeLong(indexId); out.writeLong(tabletId); @@ -130,7 +130,7 @@ public void write(DataOutput out) throws IOException { public void readFields(DataInput in) throws IOException { dbId = in.readLong(); tableId = in.readLong(); - partitionId = in.readLong(); + physicalPartitionId = in.readLong(); indexId = in.readLong(); tabletId = in.readLong(); diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java b/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java index 38eb7a299b2b4..9e885ebf45c77 100644 --- a/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java @@ -1560,7 +1560,7 @@ public ShowResultSet visitShowTabletStatement(ShowTabletStmt statement, ConnectC String dbName = null; Long tableId = tabletMeta != null ? tabletMeta.getTableId() : TabletInvertedIndex.NOT_EXIST_VALUE; String tableName = null; - Long partitionId = tabletMeta != null ? tabletMeta.getPartitionId() : TabletInvertedIndex.NOT_EXIST_VALUE; + Long partitionId = tabletMeta != null ? tabletMeta.getPhysicalPartitionId() : TabletInvertedIndex.NOT_EXIST_VALUE; String partitionName = null; Long indexId = tabletMeta != null ? tabletMeta.getIndexId() : TabletInvertedIndex.NOT_EXIST_VALUE; String indexName = null; @@ -1599,6 +1599,10 @@ public ShowResultSet visitShowTabletStatement(ShowTabletStmt statement, ConnectC break; } Partition partition = olapTable.getPartition(physicalPartition.getParentId()); + if (partition == null) { + isSync = false; + break; + } partitionName = partition.getName(); MaterializedIndex index = physicalPartition.getIndex(indexId); diff --git a/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/BinlogConsumeStateVO.java b/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/BinlogConsumeStateVO.java index bbad48c914164..bb45eee9ad619 100644 --- a/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/BinlogConsumeStateVO.java +++ b/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/BinlogConsumeStateVO.java @@ -51,7 +51,7 @@ public List toThrift() { TabletMeta meta = tabletIndex.getTabletMeta(key.getTabletId()); scan.setTable_id(meta.getTableId()); scan.setTablet_id(key.getTabletId()); - scan.setPartition_id(meta.getPartitionId()); + scan.setPartition_id(meta.getPhysicalPartitionId()); scan.setOffset(value.toThrift()); res.add(scan); }); diff --git a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java index e634170735149..4256624c71b36 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java @@ -364,7 +364,7 @@ public void recreateTabletInvertIndex() { .getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { long indexId = index.getId(); int schemaHash = olapTable.getSchemaHashByIndexId(indexId); - TabletMeta tabletMeta = new TabletMeta(dbId, tableId, partition.getParentId(), physicalPartitionId, + TabletMeta tabletMeta = new TabletMeta(dbId, tableId, physicalPartitionId, indexId, schemaHash, medium, table.isCloudNativeTableOrMaterializedView()); for (Tablet tablet : index.getTablets()) { long tabletId = tablet.getId(); @@ -1541,7 +1541,7 @@ private PhysicalPartition createPhysicalPartition(String name, Database db, Olap if (name == null) { name = partition.generatePhysicalPartitionName(id); } - PhysicalPartitionImpl physicalParition = new PhysicalPartitionImpl( + PhysicalPartitionImpl physicalPartition = new PhysicalPartitionImpl( id, name, partition.getId(), shardGroupId, indexMap.get(olapTable.getBaseIndexId())); PartitionInfo partitionInfo = olapTable.getPartitionInfo(); @@ -1563,15 +1563,15 @@ private PhysicalPartition createPhysicalPartition(String name, Database db, Olap tabletMeta, tabletIdSet, warehouseId); } else { createOlapTablets(olapTable, index, Replica.ReplicaState.NORMAL, distributionInfo, - physicalParition.getVisibleVersion(), replicationNum, tabletMeta, tabletIdSet); + physicalPartition.getVisibleVersion(), replicationNum, tabletMeta, tabletIdSet); } if (index.getId() != olapTable.getBaseIndexId()) { // add rollup index to partition - physicalParition.createRollupIndex(index); + physicalPartition.createRollupIndex(index); } } - return physicalParition; + return physicalPartition; } public void addSubPartitions(Database db, OlapTable table, Partition partition, @@ -1659,7 +1659,7 @@ public void replayAddSubPartition(PhysicalPartitionPersistInfoV2 info) throws Dd for (MaterializedIndex index : physicalPartition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { long indexId = index.getId(); int schemaHash = olapTable.getSchemaHashByIndexId(indexId); - TabletMeta tabletMeta = new TabletMeta(info.getDbId(), info.getTableId(), info.getPartitionId(), + TabletMeta tabletMeta = new TabletMeta(info.getDbId(), info.getTableId(), physicalPartition.getId(), index.getId(), schemaHash, olapTable.getPartitionInfo().getDataProperty( info.getPartitionId()).getStorageMedium(), false); for (Tablet tablet : index.getTablets()) { @@ -2253,7 +2253,7 @@ public void replayCreateTable(CreateTableInfo info) { .getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { long indexId = mIndex.getId(); int schemaHash = olapTable.getSchemaHashByIndexId(indexId); - TabletMeta tabletMeta = new TabletMeta(dbId, tableId, partition.getParentId(), physicalPartitionId, + TabletMeta tabletMeta = new TabletMeta(dbId, tableId, physicalPartitionId, indexId, schemaHash, medium, table.isCloudNativeTableOrMaterializedView()); for (Tablet tablet : mIndex.getTablets()) { long tabletId = tablet.getId(); @@ -2825,6 +2825,12 @@ public Collection getAllPartitionsIncludeRecycleBin(OlapTable table) return partitions; } + public Collection getAllPhysicalPartitionsIncludeRecycleBin(OlapTable table) { + Collection partitions = table.getAllPhysicalPartitions(); + partitions.addAll(recycleBin.getPhysicalPartitions(table.getId())); + return partitions; + } + // NOTE: result can be null, cause partition erase is not in db lock public DataProperty getDataPropertyIncludeRecycleBin(PartitionInfo info, long partitionId) { DataProperty dataProperty = info.getDataProperty(partitionId); @@ -4901,7 +4907,7 @@ private void setReplicaStatusInternal(long tabletId, long backendId, Replica.Rep if (!isReplay) { // Put this tablet into urgent table so that it can be repaired ASAP. stateMgr.getTabletChecker().setTabletForUrgentRepair(dbId, meta.getTableId(), - meta.getPartitionId()); + meta.getPhysicalPartitionId()); SetReplicaStatusOperationLog log = new SetReplicaStatusOperationLog(backendId, tabletId, status); GlobalStateMgr.getCurrentState().getEditLog().logSetReplicaStatus(log); diff --git a/fe/fe-core/src/main/java/com/starrocks/task/TabletMetadataUpdateAgentTaskFactory.java b/fe/fe-core/src/main/java/com/starrocks/task/TabletMetadataUpdateAgentTaskFactory.java index 26a2f6d5f54c0..400e66a964048 100644 --- a/fe/fe-core/src/main/java/com/starrocks/task/TabletMetadataUpdateAgentTaskFactory.java +++ b/fe/fe-core/src/main/java/com/starrocks/task/TabletMetadataUpdateAgentTaskFactory.java @@ -138,7 +138,7 @@ public List getTTabletMetaInfoList() { } TTabletMetaInfo metaInfo = new TTabletMetaInfo(); metaInfo.setTablet_id(tabletId); - metaInfo.setPartition_id(tabletMeta.getPartitionId()); + metaInfo.setPartition_id(tabletMeta.getPhysicalPartitionId()); metaInfo.setMeta_type(TTabletMetaType.PARTITIONID); metaInfos.add(metaInfo); // add at most 10000 tablet meta during one sync to avoid too large task diff --git a/fe/fe-core/src/main/java/com/starrocks/transaction/LakeTableTxnStateListener.java b/fe/fe-core/src/main/java/com/starrocks/transaction/LakeTableTxnStateListener.java index fd72abeebbb61..4775ef90b7cc4 100644 --- a/fe/fe-core/src/main/java/com/starrocks/transaction/LakeTableTxnStateListener.java +++ b/fe/fe-core/src/main/java/com/starrocks/transaction/LakeTableTxnStateListener.java @@ -102,11 +102,13 @@ public void preCommit(TransactionState txnState, List finished if (tabletMeta.getTableId() != table.getId()) { continue; } - if (table.getPhysicalPartition(tabletMeta.getPartitionId()) == null) { + if (table.getPhysicalPartition(tabletMeta.getPhysicalPartitionId()) == null) { // this can happen when partitionId == -1 (tablet being dropping) or partition really not exist. continue; } - dirtyPartitionSet.add(tabletMeta.getPartitionId()); + dirtyPartitionSet.add(tabletMeta.getPhysicalPartitionId()); + LOG.info("add dirty partition: {} from tablet {}", tabletMeta.getPhysicalPartitionId(), + finishedTablets.get(i).getTabletId()); // Invalid column set should union invalidDictCacheColumns.addAll(finishedTablets.get(i).getInvalidDictCacheColumns()); diff --git a/fe/fe-core/src/main/java/com/starrocks/transaction/OlapTableTxnStateListener.java b/fe/fe-core/src/main/java/com/starrocks/transaction/OlapTableTxnStateListener.java index 4822927400ccc..a89c73e95e450 100644 --- a/fe/fe-core/src/main/java/com/starrocks/transaction/OlapTableTxnStateListener.java +++ b/fe/fe-core/src/main/java/com/starrocks/transaction/OlapTableTxnStateListener.java @@ -110,7 +110,7 @@ public void preCommit(TransactionState txnState, List tabletCo if (tableId != table.getId()) { continue; } - long partitionId = tabletMeta.getPartitionId(); + long partitionId = tabletMeta.getPhysicalPartitionId(); if (table.getPhysicalPartition(partitionId) == null) { // this can happen when partitionId == -1 (tablet being dropping) // or partition really not exist. diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/TempPartitionTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/TempPartitionTest.java index c782b6e25aa69..a6f12bcc2917c 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/TempPartitionTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/TempPartitionTest.java @@ -153,7 +153,7 @@ private long getPartitionIdByTabletId(long tabletId) { if (tabletMeta == null) { return -1; } - return tabletMeta.getPartitionId(); + return tabletMeta.getPhysicalPartitionId(); } private void getPartitionNameToTabletIdMap(String tbl, boolean isTemp, Map partNameToTabletId) diff --git a/fe/fe-core/src/test/java/com/starrocks/leader/ReportHandlerTest.java b/fe/fe-core/src/test/java/com/starrocks/leader/ReportHandlerTest.java index 5b5ff4fdc1c20..b8854ad02763a 100644 --- a/fe/fe-core/src/test/java/com/starrocks/leader/ReportHandlerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/leader/ReportHandlerTest.java @@ -23,7 +23,7 @@ import com.starrocks.catalog.MaterializedIndex; import com.starrocks.catalog.OlapTable; import com.starrocks.catalog.OlapTable.OlapTableState; -import com.starrocks.catalog.Partition; +import com.starrocks.catalog.PhysicalPartition; import com.starrocks.catalog.Replica; import com.starrocks.catalog.Tablet; import com.starrocks.catalog.TabletInvertedIndex; @@ -388,7 +388,7 @@ public void testHandleMigration() throws TException { locker.unLockDatabase(db, LockType.READ); } - Partition partition = table.getPartition(tabletMeta.getPartitionId()); + PhysicalPartition partition = table.getPhysicalPartition(tabletMeta.getPhysicalPartitionId()); MaterializedIndex idx = partition.getIndex(tabletMeta.getIndexId()); LocalTablet tablet = (LocalTablet) idx.getTablet(tabletId);