Skip to content

Commit

Permalink
[BugFix] Fix tablet meta use tabletMeta uses partition_id and physica…
Browse files Browse the repository at this point in the history 
…l_partition_id at the same time to cause confusion (#52258)

Signed-off-by: meegoo <meegoo.sr@gmail.com>
(cherry picked from commit 248ee98)

# Conflicts:
#	fe/fe-core/src/main/java/com/starrocks/catalog/Partition.java
#	fe/fe-core/src/main/java/com/starrocks/clone/DiskAndTabletLoadReBalancer.java
#	fe/fe-core/src/main/java/com/starrocks/clone/TabletChecker.java
#	fe/fe-core/src/main/java/com/starrocks/clone/TabletSchedCtx.java
#	fe/fe-core/src/main/java/com/starrocks/clone/TabletScheduler.java
#	fe/fe-core/src/main/java/com/starrocks/consistency/CheckConsistencyJob.java
#	fe/fe-core/src/main/java/com/starrocks/consistency/ConsistencyChecker.java
#	fe/fe-core/src/main/java/com/starrocks/lake/Utils.java
#	fe/fe-core/src/main/java/com/starrocks/leader/ReportHandler.java
#	fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java
#	fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java
#	test/sql/test_automatic_bucket/R/test_automatic_partition
#	test/sql/test_automatic_bucket/T/test_automatic_partition
  • Loading branch information
@meegoo @mergify
meegoo authored and mergify[bot] committed Oct 25, 2024
1 parent 1dafcae commit ea5ac1b
Show file tree
Hide file tree
Showing 26 changed files with 1,458 additions and 119 deletions.
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeJobV2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -825,7 +825,7 @@ private void onFinished(OlapTable tbl) {
}
// replace the origin index with shadow index, set index state as NORMAL
for (Partition partition : tbl.getPartitions()) {
TStorageMedium medium = tbl.getPartitionInfo().getDataProperty(partition.getParentId()).getStorageMedium();
TStorageMedium medium = tbl.getPartitionInfo().getDataProperty(partition.getId()).getStorageMedium();
// drop the origin index from partitions
for (Map.Entry<Long, Long> entry : indexIdMap.entrySet()) {
long shadowIdxId = entry.getKey();
Expand Down
8 changes: 8 additions & 0 deletions fe/fe-core/src/main/java/com/starrocks/catalog/CatalogRecycleBin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,14 @@ public synchronized List<Partition> getPartitions(long tableId) {
.collect(Collectors.toList());
}

public synchronized List<PhysicalPartition> getPhysicalPartitions(long tableId) {
return idToPartition.values().stream()
.filter(v -> (v.getTableId() == tableId))
.map(RecyclePartitionInfo::getPartition)
.flatMap(p -> p.getSubPartitions().stream())
.collect(Collectors.toList());
}

/**
* if we can erase this instance, we should check if anyone enable erase later.
* Only used by main loop.
Expand Down
30 changes: 30 additions & 0 deletions fe/fe-core/src/main/java/com/starrocks/catalog/Partition.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -648,4 +648,34 @@ public long getMinRetainVersion() {
public void setMinRetainVersion(long minRetainVersion) {
this.minRetainVersion = minRetainVersion;
}
<<<<<<< HEAD
=======

public String generatePhysicalPartitionName(long physicalPartitionId) {
return this.name + '_' + physicalPartitionId;
}

@Override
public void gsonPostProcess() throws IOException {
if (dataVersion == 0) {
dataVersion = visibleVersion;
}
if (nextDataVersion == 0) {
nextDataVersion = nextVersion;
}
if (versionEpoch == 0) {
versionEpoch = nextVersionEpoch();
}
if (versionTxnType == null) {
versionTxnType = TransactionType.TXN_NORMAL;
}

for (PhysicalPartitionImpl subPartition : idToSubPartition.values()) {
if (subPartition.getName() == null) {
subPartition.setName(generatePhysicalPartitionName(subPartition.getId()));
}
nameToSubPartition.put(subPartition.getName(), subPartition);
}
}
>>>>>>> 248ee98eb9 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physical_partition_id at the same time to cause confusion (#52258))
}
19 changes: 3 additions & 16 deletions fe/fe-core/src/main/java/com/starrocks/catalog/TabletMeta.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@
public class TabletMeta {
private final long dbId;
private final long tableId;
private final long partitionId;
private final long physicalPartitionId;
private final long indexId;

Expand All @@ -59,11 +58,10 @@ public class TabletMeta {

private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();

public TabletMeta(long dbId, long tableId, long partitionId, long physicalPartitionId, long indexId, int schemaHash,
public TabletMeta(long dbId, long tableId, long physicalPartitionId, long indexId, int schemaHash,
TStorageMedium storageMedium, boolean isLakeTablet) {
this.dbId = dbId;
this.tableId = tableId;
this.partitionId = partitionId;
this.physicalPartitionId = physicalPartitionId;
this.indexId = indexId;

Expand All @@ -75,15 +73,9 @@ public TabletMeta(long dbId, long tableId, long partitionId, long physicalPartit
this.isLakeTablet = isLakeTablet;
}

// for single physical partition, the physicalPartitionId is same as partitionId
public TabletMeta(long dbId, long tableId, long partitionId, long indexId, int schemaHash,
TStorageMedium storageMedium, boolean isLakeTablet) {
this(dbId, tableId, partitionId, partitionId, indexId, schemaHash, storageMedium, isLakeTablet);
}

public TabletMeta(long dbId, long tableId, long partitionId, long indexId, int schemaHash,
public TabletMeta(long dbId, long tableId, long physicalPartitionId, long indexId, int schemaHash,
TStorageMedium storageMedium) {
this(dbId, tableId, partitionId, indexId, schemaHash, storageMedium, false);
this(dbId, tableId, physicalPartitionId, indexId, schemaHash, storageMedium, false);
}

public long getDbId() {
Expand All @@ -94,10 +86,6 @@ public long getTableId() {
return tableId;
}

public long getPartitionId() {
return partitionId;
}

public long getPhysicalPartitionId() {
return physicalPartitionId;
}
Expand Down Expand Up @@ -164,7 +152,6 @@ public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("dbId=").append(dbId);
sb.append(" tableId=").append(tableId);
sb.append(" partitionId=").append(partitionId);
sb.append(" physicalPartitionId=").append(physicalPartitionId);
sb.append(" indexId=").append(indexId);
sb.append(" oldSchemaHash=").append(oldSchemaHash);
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/com/starrocks/clone/ColocateTableBalancer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -820,7 +820,7 @@ private ColocateMatchResult doMatchOneGroup(GroupId groupId,
TabletSchedCtx.Type.REPAIR,
// physical partition id is same as partition id
// since colocate table should have only one physical partition
db.getId(), tableId, partition.getId(), partition.getId(),
db.getId(), tableId, partition.getId(),
index.getId(), tablet.getId(),
System.currentTimeMillis());
// the tablet status will be checked and set again when being scheduled
Expand Down
35 changes: 27 additions & 8 deletions fe/fe-core/src/main/java/com/starrocks/clone/DiskAndTabletLoadReBalancer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -544,9 +544,13 @@ private List<TabletSchedCtx> balanceClusterDisk(ClusterLoadStatistic clusterStat
if (olapTable == null) {
continue;
}
PhysicalPartition physicalPartition = olapTable.getPartition(tabletMeta.getPhysicalPartitionId());
if (physicalPartition == null) {
continue;
}

if (isDestBackendLocationMismatch(olapTable, hBackend.getId(), lBackend.getId(),
tabletMeta.getPartitionId(), tabletId)) {
physicalPartition.getParentId(), tabletId)) {
continue;
}

Expand Down Expand Up @@ -584,7 +588,7 @@ private List<TabletSchedCtx> balanceClusterDisk(ClusterLoadStatistic clusterStat
hState.minusUsedCapacity(replica.getPathHash(), replica.getDataSize());

TabletSchedCtx schedCtx = new TabletSchedCtx(TabletSchedCtx.Type.BALANCE,
tabletMeta.getDbId(), tabletMeta.getTableId(), tabletMeta.getPartitionId(),
tabletMeta.getDbId(), tabletMeta.getTableId(),
tabletMeta.getPhysicalPartitionId(), tabletMeta.getIndexId(),
tabletId, System.currentTimeMillis());
schedCtx.setOrigPriority(TabletSchedCtx.Priority.LOW);
Expand Down Expand Up @@ -800,7 +804,7 @@ private void balanceBackendDisk(TStorageMedium medium, double avgUsedPercent,
// NOTICE: state has been changed, the tablet must be selected
destPathUsedCap += replica.getDataSize();
srcPathUsedCap -= replica.getDataSize();
Pair<Long, Long> p = Pair.create(tabletMeta.getPartitionId(), tabletMeta.getIndexId());
Pair<Long, Long> p = Pair.create(tabletMeta.getPhysicalPartitionId(), tabletMeta.getIndexId());
// p: partition <physicalPartitionId, indexId>
// k: partition same to p
srcPathPartitionTablets.compute(p, (k, pTablets) -> {
Expand All @@ -819,7 +823,7 @@ private void balanceBackendDisk(TStorageMedium medium, double avgUsedPercent,

TabletSchedCtx schedCtx =
new TabletSchedCtx(TabletSchedCtx.Type.BALANCE, tabletMeta.getDbId(),
tabletMeta.getTableId(), tabletMeta.getPartitionId(),
tabletMeta.getTableId(),
tabletMeta.getPhysicalPartitionId(),
tabletMeta.getIndexId(), tabletId, System.currentTimeMillis());
schedCtx.setOrigPriority(TabletSchedCtx.Priority.LOW);
Expand Down Expand Up @@ -1414,7 +1418,7 @@ private TabletSchedCtx tryToBalanceTablet(Pair<Long, Set<Long>> srcTablets,
}

TabletSchedCtx schedCtx = new TabletSchedCtx(TabletSchedCtx.Type.BALANCE,
tabletMeta.getDbId(), tabletMeta.getTableId(), tabletMeta.getPartitionId(),
tabletMeta.getDbId(), tabletMeta.getTableId(),
tabletMeta.getPhysicalPartitionId(),
tabletMeta.getIndexId(), tabletId, System.currentTimeMillis());
schedCtx.setOrigPriority(TabletSchedCtx.Priority.LOW);
Expand Down Expand Up @@ -1536,14 +1540,21 @@ private boolean isTabletUnhealthy(long dbId, OlapTable olapTable, Long tabletId,
}

try {
<<<<<<< HEAD
db.readLock();
PhysicalPartition partition = globalStateMgr.getPhysicalPartitionIncludeRecycleBin(
olapTable, tabletMeta.getPhysicalPartitionId());
if (partition == null) {
=======
locker.lockDatabase(db.getId(), LockType.READ);
PhysicalPartition physicalPartition = globalStateMgr.getLocalMetastore()
.getPhysicalPartitionIncludeRecycleBin(olapTable, tabletMeta.getPhysicalPartitionId());
if (physicalPartition == null) {
>>>>>>> 248ee98eb9 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physical_partition_id at the same time to cause confusion (#52258))
return true;
}

MaterializedIndex index = partition.getIndex(tabletMeta.getIndexId());
MaterializedIndex index = physicalPartition.getIndex(tabletMeta.getIndexId());
if (index == null) {
return true;
}
Expand All @@ -1553,17 +1564,27 @@ private boolean isTabletUnhealthy(long dbId, OlapTable olapTable, Long tabletId,
return true;
}

<<<<<<< HEAD
short replicaNum = globalStateMgr
.getReplicationNumIncludeRecycleBin(olapTable.getPartitionInfo(), partition.getParentId());
=======
short replicaNum = globalStateMgr.getLocalMetastore()
.getReplicationNumIncludeRecycleBin(olapTable.getPartitionInfo(), physicalPartition.getParentId());
>>>>>>> 248ee98eb9 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physical_partition_id at the same time to cause confusion (#52258))
if (replicaNum == (short) -1) {
return true;
}

Pair<LocalTablet.TabletHealthStatus, TabletSchedCtx.Priority> statusPair =
TabletChecker.getTabletHealthStatusWithPriority(
tablet,
<<<<<<< HEAD
GlobalStateMgr.getCurrentSystemInfo(),
partition.getVisibleVersion(),
=======
globalStateMgr.getNodeMgr().getClusterInfo(),
physicalPartition.getVisibleVersion(),
>>>>>>> 248ee98eb9 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physical_partition_id at the same time to cause confusion (#52258))
replicaNum,
aliveBeIds,
olapTable.getLocation());
Expand Down Expand Up @@ -1726,8 +1747,6 @@ private Map<Pair<Long, Long>, PartitionStat> getPartitionStats(TStorageMedium me
}

pStat.skew = maxNum - minNum;
LOG.info("add partition stat: {} id: {}",
pStat, new Pair<>(physicalPartition.getId(), idx.getId()));
}
}
}
Expand Down
44 changes: 33 additions & 11 deletions fe/fe-core/src/main/java/com/starrocks/clone/TabletChecker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -299,10 +299,15 @@ private void doCheck(boolean isUrgent) {
}

OlapTable olapTbl = (OlapTable) table;
<<<<<<< HEAD
for (Partition partition : GlobalStateMgr.getCurrentState().getAllPartitionsIncludeRecycleBin(olapTbl)) {
=======
for (PhysicalPartition physicalPartition : GlobalStateMgr.getCurrentState().getLocalMetastore()
.getAllPhysicalPartitionsIncludeRecycleBin(olapTbl)) {
>>>>>>> 248ee98eb9 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physical_partition_id at the same time to cause confusion (#52258))
partitionChecked++;

boolean isPartitionUrgent = isPartitionUrgent(dbId, table.getId(), partition.getId());
boolean isPartitionUrgent = isPartitionUrgent(dbId, table.getId(), physicalPartition.getId());
totStat.isUrgentPartitionHealthy = true;
if ((isUrgent && !isPartitionUrgent) || (!isUrgent && isPartitionUrgent)) {
continue;
Expand All @@ -322,34 +327,50 @@ private void doCheck(boolean isUrgent) {
if (GlobalStateMgr.getCurrentState().getTableIncludeRecycleBin(db, olapTbl.getId()) == null) {
continue TABLE;
}
<<<<<<< HEAD
if (GlobalStateMgr.getCurrentState()
.getPartitionIncludeRecycleBin(olapTbl, partition.getId()) == null) {
=======
if (GlobalStateMgr.getCurrentState().getLocalMetastore()
.getPhysicalPartitionIncludeRecycleBin(olapTbl, physicalPartition.getId()) == null) {
>>>>>>> 248ee98eb9 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physical_partition_id at the same time to cause confusion (#52258))
continue;
}
}

if (partition.getState() != PartitionState.NORMAL) {
Partition logicalPartition = olapTbl.getPartition(physicalPartition.getParentId());
if (logicalPartition == null) {
continue;
}

if (logicalPartition.getState() != PartitionState.NORMAL) {
// when alter job is in FINISHING state, partition state will be set to NORMAL,
// and we can schedule the tablets in it.
continue;
}

short replicaNum = GlobalStateMgr.getCurrentState()
<<<<<<< HEAD
.getReplicationNumIncludeRecycleBin(olapTbl.getPartitionInfo(), partition.getId());
=======
.getLocalMetastore()
.getReplicationNumIncludeRecycleBin(
olapTbl.getPartitionInfo(), physicalPartition.getParentId());
>>>>>>> 248ee98eb9 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physical_partition_id at the same time to cause confusion (#52258))
if (replicaNum == (short) -1) {
continue;
}

TabletCheckerStat partitionTabletCheckerStat = doCheckOnePartition(db, olapTbl, partition,
TabletCheckerStat partitionTabletCheckerStat = doCheckOnePartition(db, olapTbl, physicalPartition,
replicaNum, aliveBeIdsInCluster, isPartitionUrgent);
totStat.accumulateStat(partitionTabletCheckerStat);
if (totStat.isUrgentPartitionHealthy && isPartitionUrgent) {
// if all replicas in this partition are healthy, remove this partition from
// priorities.
LOG.debug("partition is healthy, remove from urgent table: {}-{}-{}",
db.getId(), olapTbl.getId(), partition.getId());
db.getId(), olapTbl.getId(), physicalPartition.getId());
removeFromUrgentTable(new RepairTabletInfo(db.getId(),
olapTbl.getId(), Lists.newArrayList(partition.getId())));
olapTbl.getId(), Lists.newArrayList(physicalPartition.getId())));
}
} // partitions
} // tables
Expand Down Expand Up @@ -394,12 +415,12 @@ public void accumulateStat(TabletCheckerStat stat) {
}
}

private TabletCheckerStat doCheckOnePartition(Database db, OlapTable olapTbl, Partition partition,
private TabletCheckerStat doCheckOnePartition(Database db, OlapTable olapTbl, PhysicalPartition physicalPartition,
int replicaNum, List<Long> aliveBeIdsInCluster,
boolean isPartitionUrgent) {
TabletCheckerStat partitionTabletCheckerStat = new TabletCheckerStat();
// Tablet in SHADOW index can not be repaired or balanced
for (PhysicalPartition physicalPartition : partition.getSubPartitions()) {
if (physicalPartition != null) {
for (MaterializedIndex idx : physicalPartition.getMaterializedIndices(
IndexExtState.VISIBLE)) {
for (Tablet tablet : idx.getTablets()) {
Expand Down Expand Up @@ -444,7 +465,7 @@ private TabletCheckerStat doCheckOnePartition(Database db, OlapTable olapTbl, Pa

TabletSchedCtx tabletSchedCtx = new TabletSchedCtx(
TabletSchedCtx.Type.REPAIR,
db.getId(), olapTbl.getId(), partition.getId(),
db.getId(), olapTbl.getId(),
physicalPartition.getId(), idx.getId(), tablet.getId(),
System.currentTimeMillis());
// the tablet status will be set again when being scheduled
Expand Down Expand Up @@ -516,7 +537,7 @@ private void cleanInvalidUrgentTable() {
}

Set<PrioPart> parts = tblEntry.getValue();
parts = parts.stream().filter(p -> (tbl.getPartition(p.partId) != null && !p.isTimeout())).collect(
parts = parts.stream().filter(p -> (tbl.getPhysicalPartition(p.partId) != null && !p.isTimeout())).collect(
Collectors.toSet());
if (parts.isEmpty()) {
deletedUrgentTable.add(Pair.create(dbId, tblId));
Expand Down Expand Up @@ -619,14 +640,15 @@ public static RepairTabletInfo getRepairTabletInfo(String dbName, String tblName
OlapTable olapTable = (OlapTable) tbl;

if (partitions == null || partitions.isEmpty()) {
partIds = olapTable.getPartitions().stream().map(Partition::getId).collect(Collectors.toList());
partIds = olapTable.getPhysicalPartitions().stream().map(PhysicalPartition::getId).collect(Collectors.toList());
} else {
for (String partName : partitions) {
Partition partition = olapTable.getPartition(partName);
if (partition == null) {
throw new DdlException("Partition does not exist: " + partName);
}
partIds.add(partition.getId());
partIds.addAll(partition.getSubPartitions().stream()
.map(PhysicalPartition::getId).collect(Collectors.toList()));
}
}
} finally {
Expand Down
Loading

0 comments on commit ea5ac1b

Please sign in to comment.