Skip to content

Commit

Permalink
[BugFix] Fix tablet meta use tabletMeta uses partition_id and physica…
Browse files Browse the repository at this point in the history 
…l_partition_id at the same time to cause confusion

Signed-off-by: meegoo <meegoo.sr@gmail.com>
  • Loading branch information
@meegoo
meegoo committed Oct 24, 2024
1 parent 5ac1d7f commit 33dd061
Show file tree
Hide file tree
Showing 25 changed files with 190 additions and 185 deletions.
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeJobV2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -856,7 +856,7 @@ private void onFinished(OlapTable tbl) {
}
// replace the origin index with shadow index, set index state as NORMAL
for (Partition partition : tbl.getPartitions()) {
TStorageMedium medium = tbl.getPartitionInfo().getDataProperty(partition.getParentId()).getStorageMedium();
TStorageMedium medium = tbl.getPartitionInfo().getDataProperty(partition.getId()).getStorageMedium();
// drop the origin index from partitions
for (Map.Entry<Long, Long> entry : indexIdMap.entrySet()) {
long shadowIdxId = entry.getKey();
Expand Down
8 changes: 8 additions & 0 deletions fe/fe-core/src/main/java/com/starrocks/catalog/CatalogRecycleBin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,14 @@ public synchronized List<Partition> getPartitions(long tableId) {
.collect(Collectors.toList());
}

public synchronized List<PhysicalPartition> getPhysicalPartitions(long tableId) {
return idToPartition.values().stream()
.filter(v -> (v.getTableId() == tableId))
.map(RecyclePartitionInfo::getPartition)
.flatMap(p -> p.getSubPartitions().stream())
.collect(Collectors.toList());
}

/**
* if we can erase this instance, we should check if anyone enable erase later.
* Only used by main loop.
Expand Down
4 changes: 2 additions & 2 deletions fe/fe-core/src/main/java/com/starrocks/catalog/Partition.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -628,8 +628,8 @@ public void setMinRetainVersion(long minRetainVersion) {
this.minRetainVersion = minRetainVersion;
}

public String generatePhysicalPartitionName(long physicalParitionId) {
return this.name + '_' + physicalParitionId;
public String generatePhysicalPartitionName(long physicalPartitionId) {
return this.name + '_' + physicalPartitionId;
}

@Override
Expand Down
20 changes: 10 additions & 10 deletions fe/fe-core/src/main/java/com/starrocks/catalog/TabletInvertedIndex.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,7 @@ public void tabletReport(long backendId, Map<Long, TTablet> backendTablets,
replica.setLastReportVersion(backendTabletInfo.getVersion());

// check if tablet needs migration
long partitionId = tabletMeta.getPartitionId();
long partitionId = tabletMeta.getPhysicalPartitionId();
TStorageMedium storageMedium = storageMediumMap.get(partitionId);
if (storageMedium != null && backendTabletInfo.isSetStorage_medium()) {
if (storageMedium != backendTabletInfo.getStorage_medium()) {
Expand Down Expand Up @@ -240,7 +240,7 @@ public void tabletReport(long backendId, Map<Long, TTablet> backendTablets,
transactionMgr.getTransactionState(tabletMeta.getDbId(), transactionId);
if (transactionState == null ||
transactionState.getTransactionStatus() == TransactionStatus.ABORTED) {
transactionsToClear.put(transactionId, tabletMeta.getPartitionId());
transactionsToClear.put(transactionId, partitionId);
LOG.debug("transaction id [{}] is not valid any more, "
+ "clear it from backend [{}]", transactionId, backendId);
} else if (transactionState.getTransactionStatus() ==
Expand All @@ -267,7 +267,7 @@ public void tabletReport(long backendId, Map<Long, TTablet> backendTablets,
transactionState.getTransactionId());
} else {
TPartitionVersionInfo versionInfo =
new TPartitionVersionInfo(tabletMeta.getPartitionId(),
new TPartitionVersionInfo(partitionId,
partitionCommitInfo.getVersion(), 0);
Map<Long, Map<Long, TPartitionVersionInfo>> txnMap =
transactionsToPublish.computeIfAbsent(
Expand Down Expand Up @@ -341,7 +341,7 @@ private void deleteTabletByConsistencyChecker(TabletMeta tabletMeta, long tablet
* And this process will also output a report in `fe.log`, including valid number of
* tablet and number of tablet in recycle bin for each backend.
*/
public void checkTabletMetaConsistency(Map<Long, Integer> creatingTableIds) {
private void checkTabletMetaConsistency(Map<Long, Integer> creatingTableIds) {
LocalMetastore localMetastore = GlobalStateMgr.getCurrentState().getLocalMetastore();
CatalogRecycleBin recycleBin = GlobalStateMgr.getCurrentState().getRecycleBin();

Expand Down Expand Up @@ -416,11 +416,11 @@ public void checkTabletMetaConsistency(Map<Long, Integer> creatingTableIds) {
}

// validate partition
long partitionId = tabletMeta.getPartitionId();
PhysicalPartition partition = table.getPhysicalPartition(partitionId);
if (partition == null) {
partition = recycleBin.getPhysicalPartition(partitionId);
if (partition != null) {
long partitionId = tabletMeta.getPhysicalPartitionId();
PhysicalPartition physicalPartition = table.getPhysicalPartition(partitionId);
if (physicalPartition == null) {
physicalPartition = recycleBin.getPhysicalPartition(partitionId);
if (physicalPartition != null) {
isInRecycleBin = true;
} else {
deleteTabletByConsistencyChecker(tabletMeta, tabletId, backendId,
Expand All @@ -432,7 +432,7 @@ public void checkTabletMetaConsistency(Map<Long, Integer> creatingTableIds) {

// validate index
long indexId = tabletMeta.getIndexId();
MaterializedIndex index = partition.getIndex(indexId);
MaterializedIndex index = physicalPartition.getIndex(indexId);
if (index == null) {
deleteTabletByConsistencyChecker(tabletMeta, tabletId, backendId,
"materialized index " + dbId + "." + tableId + "." +
Expand Down
19 changes: 3 additions & 16 deletions fe/fe-core/src/main/java/com/starrocks/catalog/TabletMeta.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@
public class TabletMeta {
private final long dbId;
private final long tableId;
private final long partitionId;
private final long physicalPartitionId;
private final long indexId;

Expand All @@ -59,11 +58,10 @@ public class TabletMeta {

private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();

public TabletMeta(long dbId, long tableId, long partitionId, long physicalPartitionId, long indexId, int schemaHash,
public TabletMeta(long dbId, long tableId, long physicalPartitionId, long indexId, int schemaHash,
TStorageMedium storageMedium, boolean isLakeTablet) {
this.dbId = dbId;
this.tableId = tableId;
this.partitionId = partitionId;
this.physicalPartitionId = physicalPartitionId;
this.indexId = indexId;

Expand All @@ -75,15 +73,9 @@ public TabletMeta(long dbId, long tableId, long partitionId, long physicalPartit
this.isLakeTablet = isLakeTablet;
}

// for single physical partition, the physicalPartitionId is same as partitionId
public TabletMeta(long dbId, long tableId, long partitionId, long indexId, int schemaHash,
TStorageMedium storageMedium, boolean isLakeTablet) {
this(dbId, tableId, partitionId, partitionId, indexId, schemaHash, storageMedium, isLakeTablet);
}

public TabletMeta(long dbId, long tableId, long partitionId, long indexId, int schemaHash,
public TabletMeta(long dbId, long tableId, long physicalPartitionId, long indexId, int schemaHash,
TStorageMedium storageMedium) {
this(dbId, tableId, partitionId, indexId, schemaHash, storageMedium, false);
this(dbId, tableId, physicalPartitionId, indexId, schemaHash, storageMedium, false);
}

public long getDbId() {
Expand All @@ -94,10 +86,6 @@ public long getTableId() {
return tableId;
}

public long getPartitionId() {
return partitionId;
}

public long getPhysicalPartitionId() {
return physicalPartitionId;
}
Expand Down Expand Up @@ -164,7 +152,6 @@ public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("dbId=").append(dbId);
sb.append(" tableId=").append(tableId);
sb.append(" partitionId=").append(partitionId);
sb.append(" physicalPartitionId=").append(physicalPartitionId);
sb.append(" indexId=").append(indexId);
sb.append(" oldSchemaHash=").append(oldSchemaHash);
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/com/starrocks/clone/ColocateTableBalancer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -824,7 +824,7 @@ private ColocateMatchResult doMatchOneGroup(GroupId groupId,
TabletSchedCtx.Type.REPAIR,
// physical partition id is same as partition id
// since colocate table should have only one physical partition
db.getId(), tableId, partition.getId(), partition.getId(),
db.getId(), tableId, partition.getId(),
index.getId(), tablet.getId(),
System.currentTimeMillis());
// the tablet status will be checked and set again when being scheduled
Expand Down
24 changes: 14 additions & 10 deletions fe/fe-core/src/main/java/com/starrocks/clone/DiskAndTabletLoadReBalancer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -548,9 +548,13 @@ private List<TabletSchedCtx> balanceClusterDisk(ClusterLoadStatistic clusterStat
if (olapTable == null) {
continue;
}
PhysicalPartition physicalPartition = olapTable.getPartition(tabletMeta.getPhysicalPartitionId());
if (physicalPartition == null) {
continue;
}

if (isDestBackendLocationMismatch(olapTable, hBackend.getId(), lBackend.getId(),
tabletMeta.getPartitionId(), tabletId)) {
physicalPartition.getParentId(), tabletId)) {
continue;
}

Expand Down Expand Up @@ -588,7 +592,7 @@ private List<TabletSchedCtx> balanceClusterDisk(ClusterLoadStatistic clusterStat
hState.minusUsedCapacity(replica.getPathHash(), replica.getDataSize());

TabletSchedCtx schedCtx = new TabletSchedCtx(TabletSchedCtx.Type.BALANCE,
tabletMeta.getDbId(), tabletMeta.getTableId(), tabletMeta.getPartitionId(),
tabletMeta.getDbId(), tabletMeta.getTableId(),
tabletMeta.getPhysicalPartitionId(), tabletMeta.getIndexId(),
tabletId, System.currentTimeMillis());
schedCtx.setOrigPriority(TabletSchedCtx.Priority.LOW);
Expand Down Expand Up @@ -803,7 +807,7 @@ private void balanceBackendDisk(TStorageMedium medium, double avgUsedPercent,
// NOTICE: state has been changed, the tablet must be selected
destPathUsedCap += replica.getDataSize();
srcPathUsedCap -= replica.getDataSize();
Pair<Long, Long> p = Pair.create(tabletMeta.getPartitionId(), tabletMeta.getIndexId());
Pair<Long, Long> p = Pair.create(tabletMeta.getPhysicalPartitionId(), tabletMeta.getIndexId());
// p: partition <physicalPartitionId, indexId>
// k: partition same to p
srcPathPartitionTablets.compute(p, (k, pTablets) -> {
Expand All @@ -822,7 +826,7 @@ private void balanceBackendDisk(TStorageMedium medium, double avgUsedPercent,

TabletSchedCtx schedCtx =
new TabletSchedCtx(TabletSchedCtx.Type.BALANCE, tabletMeta.getDbId(),
tabletMeta.getTableId(), tabletMeta.getPartitionId(),
tabletMeta.getTableId(),
tabletMeta.getPhysicalPartitionId(),
tabletMeta.getIndexId(), tabletId, System.currentTimeMillis());
schedCtx.setOrigPriority(TabletSchedCtx.Priority.LOW);
Expand Down Expand Up @@ -1420,7 +1424,7 @@ private TabletSchedCtx tryToBalanceTablet(Pair<Long, Set<Long>> srcTablets,
}

TabletSchedCtx schedCtx = new TabletSchedCtx(TabletSchedCtx.Type.BALANCE,
tabletMeta.getDbId(), tabletMeta.getTableId(), tabletMeta.getPartitionId(),
tabletMeta.getDbId(), tabletMeta.getTableId(),
tabletMeta.getPhysicalPartitionId(),
tabletMeta.getIndexId(), tabletId, System.currentTimeMillis());
schedCtx.setOrigPriority(TabletSchedCtx.Priority.LOW);
Expand Down Expand Up @@ -1544,13 +1548,13 @@ private boolean isTabletUnhealthy(long dbId, OlapTable olapTable, Long tabletId,
Locker locker = new Locker();
try {
locker.lockDatabase(db, LockType.READ);
PhysicalPartition partition = globalStateMgr.getLocalMetastore()
PhysicalPartition physicalPartition = globalStateMgr.getLocalMetastore()
.getPhysicalPartitionIncludeRecycleBin(olapTable, tabletMeta.getPhysicalPartitionId());
if (partition == null) {
if (physicalPartition == null) {
return true;
}

MaterializedIndex index = partition.getIndex(tabletMeta.getIndexId());
MaterializedIndex index = physicalPartition.getIndex(tabletMeta.getIndexId());
if (index == null) {
return true;
}
Expand All @@ -1561,7 +1565,7 @@ private boolean isTabletUnhealthy(long dbId, OlapTable olapTable, Long tabletId,
}

short replicaNum = globalStateMgr.getLocalMetastore()
.getReplicationNumIncludeRecycleBin(olapTable.getPartitionInfo(), partition.getParentId());
.getReplicationNumIncludeRecycleBin(olapTable.getPartitionInfo(), physicalPartition.getParentId());
if (replicaNum == (short) -1) {
return true;
}
Expand All @@ -1570,7 +1574,7 @@ private boolean isTabletUnhealthy(long dbId, OlapTable olapTable, Long tabletId,
TabletChecker.getTabletHealthStatusWithPriority(
tablet,
globalStateMgr.getNodeMgr().getClusterInfo(),
partition.getVisibleVersion(),
physicalPartition.getVisibleVersion(),
replicaNum,
aliveBeIds,
olapTable.getLocation());
Expand Down
Loading

0 comments on commit 33dd061

Please sign in to comment.