Skip to content

Commit

Permalink
[UT] Fix unstable testRefreshPriority (StarRocks#50932)
Browse files Browse the repository at this point in the history 
Signed-off-by: shuming.li <[email protected]>
  • Loading branch information
@LiShuMing
LiShuMing authored Sep 11, 2024
1 parent a22f115 commit 4bd744c
Show file tree
Hide file tree
Showing 3 changed files with 89 additions and 107 deletions.
16 changes: 8 additions & 8 deletions fe/fe-core/src/test/java/com/starrocks/alter/AlterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1011,8 +1011,8 @@ public void testSwapTableWithUniqueConstraints() throws Exception {
createTable(s2);
Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test");

String replaceStmt = "ALTER TABLE test.s1 SWAP WITH test.s2";
alterTableWithNewParser(replaceStmt, true);
String replaceStmt = "ALTER TABLE s1 SWAP WITH s2";
alterTableWithNewParser(replaceStmt, false);

OlapTable tbl1 = (OlapTable) db.getTable("s1");
List<UniqueConstraint> uk1 = tbl1.getUniqueConstraints();
Expand Down Expand Up @@ -1058,8 +1058,8 @@ public void testSwapTableWithForeignConstraints1() throws Exception {
Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test");

// swap child tables
String replaceStmt = "ALTER TABLE test.s2 SWAP WITH test.s3";
alterTableWithNewParser(replaceStmt, true);
String replaceStmt = "ALTER TABLE s2 SWAP WITH s3";
alterTableWithNewParser(replaceStmt, false);

OlapTable tbl1 = (OlapTable) db.getTable("s1");
List<UniqueConstraint> uk1 = tbl1.getUniqueConstraints();
Expand Down Expand Up @@ -1133,8 +1133,8 @@ public void testSwapTableWithForeignConstraints2() throws Exception {
Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test");

// swap parent tables
String replaceStmt = "ALTER TABLE test.s2 SWAP WITH test.s1";
alterTableWithNewParser(replaceStmt, true);
String replaceStmt = "ALTER TABLE s2 SWAP WITH s1";
alterTableWithNewParser(replaceStmt, false);

OlapTable tbl1 = (OlapTable) db.getTable("s1");
List<UniqueConstraint> uk1 = tbl1.getUniqueConstraints();
Expand All @@ -1157,13 +1157,13 @@ public void testSwapTableWithForeignConstraints2() throws Exception {
BaseTableInfo parentTableInfo = fk30.getParentTableInfo();
Assert.assertTrue(parentTableInfo != null);
Assert.assertEquals("s1", parentTableInfo.getTableName());
Assert.assertEquals(tbl1.getId(), parentTableInfo.getTableId());
Assert.assertEquals(tbl2.getId(), parentTableInfo.getTableId());

// test global constraint manager
GlobalConstraintManager cm = GlobalStateMgr.getCurrentState().getGlobalConstraintManager();
Assert.assertTrue(cm != null);

Set<TableWithFKConstraint> tableWithFKConstraintSet = cm.getRefConstraints(tbl1);
Set<TableWithFKConstraint> tableWithFKConstraintSet = cm.getRefConstraints(tbl2);
Assert.assertTrue(tableWithFKConstraintSet != null);
Assert.assertTrue(tableWithFKConstraintSet.size() == 1);
Assert.assertTrue(tableWithFKConstraintSet.contains(TableWithFKConstraint.of(tbl3, fk30)));
Expand Down
161 changes: 69 additions & 92 deletions fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobMaterializedViewTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import com.starrocks.common.AnalysisException;
import com.starrocks.common.UserException;
import com.starrocks.common.jmockit.Deencapsulation;
import com.starrocks.common.util.DynamicPartitionUtil;
import com.starrocks.common.util.UnitTestUtil;
import com.starrocks.common.util.concurrent.MarkedCountDownLatch;
import com.starrocks.fs.HdfsUtil;
Expand All @@ -46,6 +47,10 @@
import com.starrocks.server.GlobalStateMgr;
import com.starrocks.server.LocalMetastore;
import com.starrocks.server.MetadataMgr;
import com.starrocks.sql.analyzer.Analyzer;
import com.starrocks.sql.ast.StatementBase;
import com.starrocks.sql.parser.AstBuilder;
import com.starrocks.sql.parser.SqlParser;
import com.starrocks.system.NodeSelector;
import com.starrocks.system.SystemInfoService;
import com.starrocks.task.AgentTask;
Expand All @@ -66,12 +71,14 @@
import mockit.MockUp;
import mockit.Mocked;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestMethodOrder;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;

import java.util.Arrays;
import java.util.List;
Expand All @@ -85,6 +92,8 @@
import static com.starrocks.common.util.UnitTestUtil.DB_NAME;
import static com.starrocks.common.util.UnitTestUtil.MATERIALIZED_VIEW_NAME;
import static com.starrocks.common.util.UnitTestUtil.TABLE_NAME;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;

@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
public class RestoreJobMaterializedViewTest {
Expand All @@ -106,12 +115,10 @@ public class RestoreJobMaterializedViewTest {

@Mocked
private GlobalStateMgr globalStateMgr;

private MockBackupHandler backupHandler;

private MockRepositoryMgr repoMgr;

private MvRestoreContext mvRestoreContext;
static MockedStatic<DynamicPartitionUtil> mockedDynamicPartitionUtil = Mockito.mockStatic(DynamicPartitionUtil.class);

// Thread is not mockable in Jmockit, use subclass instead
private final class MockBackupHandler extends BackupHandler {
Expand Down Expand Up @@ -143,14 +150,12 @@ public Repository getRepo(long repoId) {
private SystemInfoService systemInfoService;
@Mocked
private NodeSelector nodeSelector;

@Injectable
private Repository repo = new Repository(repoId, "repo", false, "bos://my_repo",
new BlobStorage("broker", Maps.newHashMap()));

private BackupMeta backupMeta;

private Object[] arrayIds;
private SqlParser sqlParser = new SqlParser(AstBuilder.getInstance());

private void setUpMocker() {
MetricRepo.init();
Expand All @@ -162,6 +167,16 @@ private void setUpMocker() {
arrayIds = new Object[ID_SIZE];
IntStream.range(0, ID_SIZE).forEach(i -> arrayIds[i] = id.getAndIncrement());

// mock DynamicPartitionUtil
mockedDynamicPartitionUtil
.when(() -> DynamicPartitionUtil.registerOrRemovePartitionScheduleInfo(anyLong(), any()))
.thenAnswer(new Answer<Void>() {
@Override
public Void answer(InvocationOnMock invocation) throws Throwable {
return null;
}
});

new Expectations() {
{
GlobalStateMgr.getCurrentState();
Expand All @@ -172,6 +187,10 @@ private void setUpMocker() {
minTimes = 0;
result = db;

globalStateMgr.getSqlParser();
minTimes = 0;
result = sqlParser;

globalStateMgr.getLocalMetastore().getTable(UnitTestUtil.DB_NAME, MATERIALIZED_VIEW_NAME);
minTimes = 0;
result = db.getTable(MATERIALIZED_VIEW_NAME);
Expand Down Expand Up @@ -257,6 +276,35 @@ BackupHandler getBackupHandler() {
public void getTProperties(String path, BrokerDesc brokerDesc, THdfsProperties tProperties) throws UserException {
}
};

new MockUp<LocalMetastore>() {
@Mock
public Database getDb(String dbName) {
return db;
}

@Mock
public Table getTable(String dbName, String tblName) {
return db.getTable(tblName);
}

@Mock
public Table getTable(Long dbId, Long tableId) {
return db.getTable(tableId);
}
};

new MockUp<Analyzer>() {
@Mock
public void analyze(StatementBase statement, ConnectContext context) {
}
};

new MockUp<MaterializedView>() {
@Mock
public void fixRelationship() {
}
};
}

@BeforeEach
Expand Down Expand Up @@ -444,92 +492,39 @@ private void checkJobRun(RestoreJob job) {
}

@Test
@Order(1)
public void testMVRestore_TestOneTable1() {
public void testMVRestore() {
RestoreJob job = createRestoreJob(ImmutableList.of(UnitTestUtil.MATERIALIZED_VIEW_NAME));

new MockUp<LocalMetastore>() {
@Mock
public Database getDb(String dbName) {
return db;
}

@Mock
public Table getTable(String dbName, String tblName) {
return db.getTable(tblName);
}

@Mock
public Table getTable(Long dbId, Long tableId) {
return db.getTable(tableId);
}
};

checkJobRun(job);
assertMVActiveEquals(MATERIALIZED_VIEW_NAME, true);
}

@Ignore
public void testMVRestore_TestOneTable2() {
RestoreJob job = createRestoreJob(ImmutableList.of(UnitTestUtil.TABLE_NAME));
checkJobRun(job);
assertMVActiveEquals(MATERIALIZED_VIEW_NAME, true);
}

@Test
@Order(3)
public void testMVRestore_TestMVWithBaseTable1() {
public void testMVRestoreMVWithBaseTable1() {
// gen BackupJobInfo
RestoreJob job = createRestoreJob(ImmutableList.of(TABLE_NAME, MATERIALIZED_VIEW_NAME));
new MockUp<LocalMetastore>() {
@Mock
public Database getDb(String dbName) {
return db;
}

@Mock
public Table getTable(String dbName, String tblName) {
return db.getTable(tblName);
}

@Mock
public Table getTable(Long dbId, Long tableId) {
return db.getTable(tableId);
}
};
// backup & restore
checkJobRun(job);
assertMVActiveEquals(MATERIALIZED_VIEW_NAME, true);
}

@Test
@Order(4)
public void testMVRestore_TestMVWithBaseTable2() {
public void testMVRestoreMVWithBaseTable2() {
// gen BackupJobInfo
RestoreJob job = createRestoreJob(ImmutableList.of(MATERIALIZED_VIEW_NAME, TABLE_NAME));
new MockUp<LocalMetastore>() {
@Mock
public Database getDb(String dbName) {
return db;
}

@Mock
public Table getTable(String dbName, String tblName) {
return db.getTable(tblName);
}

@Mock
public Table getTable(Long dbId, Long tableId) {
return db.getTable(tableId);
}
};
// backup & restore
checkJobRun(job);
assertMVActiveEquals(MATERIALIZED_VIEW_NAME, true);
}

@Ignore
public void testMVRestore_TestMVWithBaseTable3() {
@Test
public void testMVRestoreMVWithBaseTable3() {
new MockUp<MetadataMgr>() {
@Mock
public Table getTable(String catalogName, String dbName, String tblName) {
return GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tblName);
}
};
// gen BackupJobInfo
RestoreJob job1 = createRestoreJob(ImmutableList.of(TABLE_NAME));
// backup & restore
Expand All @@ -542,8 +537,7 @@ public void testMVRestore_TestMVWithBaseTable3() {
}

@Test
@Order(6)
public void testMVRestore_TestMVWithBaseTable4() {
public void testMVRestoreMVWithBaseTable4() {
new MockUp<MetadataMgr>() {
@Mock
public Table getTable(String catalogName, String dbName, String tblName) {
Expand All @@ -552,22 +546,6 @@ public Table getTable(String catalogName, String dbName, String tblName) {
};
// gen BackupJobInfo
RestoreJob job1 = createRestoreJob(ImmutableList.of(MATERIALIZED_VIEW_NAME));
new MockUp<LocalMetastore>() {
@Mock
public Database getDb(String dbName) {
return db;
}

@Mock
public Table getTable(String dbName, String tblName) {
return db.getTable(tblName);
}

@Mock
public Table getTable(Long dbId, Long tableId) {
return db.getTable(tableId);
}
};
// backup & restore
checkJobRun(job1);
assertMVActiveEquals(MATERIALIZED_VIEW_NAME, true);
Expand All @@ -584,5 +562,4 @@ private void assertMVActiveEquals(String mvName, boolean expect) {
MaterializedView mv = (MaterializedView) mvTable;
Assert.assertEquals(mv.isActive(), expect);
}
}

}
19 changes: 12 additions & 7 deletions .../src/test/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessorOlapPart2Test.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import com.starrocks.common.util.UUIDUtil;
import com.starrocks.qe.ConnectContext;
import com.starrocks.qe.SessionVariable;
import com.starrocks.scheduler.persist.TaskRunStatus;
import com.starrocks.schema.MTable;
import com.starrocks.server.GlobalStateMgr;
import com.starrocks.sql.ast.StatementBase;
Expand Down Expand Up @@ -315,13 +316,13 @@ public void testTaskRun() {
),
() -> {
starRocksAssert
.withMaterializedView("create materialized view mv_refresh_priority\n" +
.withMaterializedView("create materialized view test_task_run \n" +
"partition by date_trunc('month',k1) \n" +
"distributed by hash(k2) buckets 10\n" +
"refresh deferred manual\n" +
"properties('replication_num' = '1', 'partition_refresh_number'='1')\n" +
"as select k1, k2 from tbl6;");
String mvName = "mv_refresh_priority";
String mvName = "test_task_run";
Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(TEST_DB_NAME);
MaterializedView mv = ((MaterializedView) testDb.getTable(mvName));
TaskManager tm = GlobalStateMgr.getCurrentState().getTaskManager();
Expand Down Expand Up @@ -358,7 +359,7 @@ public void testTaskRun() {
while (taskRunScheduler.getRunningTaskCount() > 0) {
Thread.sleep(100);
}
starRocksAssert.dropMaterializedView("mv_refresh_priority");
starRocksAssert.dropMaterializedView("test_task_run");
}
);
}
Expand Down Expand Up @@ -402,12 +403,16 @@ public void testRefreshPriority() {
Task task = TaskBuilder.buildMvTask(mv, testDb.getFullName());
TaskRun taskRun = TaskRunBuilder.newBuilder(task).build();
initAndExecuteTaskRun(taskRun);

TaskRun run = taskRunScheduler.getRunnableTaskRun(taskId);
Assert.assertEquals(Constants.TaskRunPriority.HIGHEST.value(), run.getStatus().getPriority());
while (taskRunScheduler.getRunningTaskCount() > 0) {
TGetTasksParams params = new TGetTasksParams();
params.setTask_name(task.getName());
List<TaskRunStatus> statuses = tm.getMatchedTaskRunStatus(params);
while (statuses.size() != 1) {
statuses = tm.getMatchedTaskRunStatus(params);
Thread.sleep(100);
}
Assert.assertEquals(1, statuses.size());
TaskRunStatus status = statuses.get(0);
Assert.assertEquals(Constants.TaskRunPriority.HIGHEST.value(), status.getPriority());
starRocksAssert.dropMaterializedView("mv_refresh_priority");
}
);
Expand Down

0 comments on commit 4bd744c

Please sign in to comment.