From c58cb1ea518b7e03bce9a479ed9774e98f5366da Mon Sep 17 00:00:00 2001 From: Javi Merino Date: Wed, 10 Aug 2016 16:57:17 +0100 Subject: [PATCH 1/7] tests/eas/hmp_parity: don't clobber big tasks When creating the tasks for the ForkMigration test the test framework was creating the big and little tasks with the same prefix, which meant that the loop that creates the little tasks overwrote the tasks created in the previous big task loop. Use different prefixes for big and little tasks so that they are all created. 55eee3cb6f1f ("tests/eas/hmp_parity: Add descriptions for tests") described the ForkMigration behaviour as it was, instead of the spirit of the test: create as many threads as there are cores in the system. Fix the description of the test to what the test always meant to do and now actually does. --- tests/eas/hmp_parity.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/eas/hmp_parity.py b/tests/eas/hmp_parity.py index 475dbffee..f89652471 100644 --- a/tests/eas/hmp_parity.py +++ b/tests/eas/hmp_parity.py @@ -109,8 +109,8 @@ class ForkMigration(unittest.TestCase): Detailed Description ==================== - The test spawns as many threads as there are little cores. It - then checks that all threads started on a big core. + The test spawns as many threads as there are cores in the system. + It then checks that all threads started on a big core. Expected Behaviour ================== @@ -138,12 +138,14 @@ class ForkMigration(unittest.TestCase): @classmethod def populate_params(cls): + big_prefix = cls.task_prefix + "_big" for idx in range(len(cls.env.target.bl.bigs)): - task = cls.task_prefix + str(idx) + task = big_prefix + str(idx) cls.params[task] = Periodic(**BIG_WORKLOAD).get() + little_prefix = cls.task_prefix + "_little" for idx in range(len(cls.env.target.bl.littles)): - task = cls.task_prefix + str(idx) + task = little_prefix + str(idx) cls.params[task] = Periodic(**SMALL_WORKLOAD).get() @classmethod -- GitLab From cba9a6f0b80fdfff2ff721ee5f94697d13caf63d Mon Sep 17 00:00:00 2001 From: Javi Merino Date: Fri, 12 Aug 2016 19:42:00 +0100 Subject: [PATCH 2/7] tests/eas/hmp_parity: move the migrator delay to the configuration file --- tests/eas/hmp_parity.config | 1 + tests/eas/hmp_parity.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/eas/hmp_parity.config b/tests/eas/hmp_parity.config index 495241d93..b693a359e 100644 --- a/tests/eas/hmp_parity.config +++ b/tests/eas/hmp_parity.config @@ -8,6 +8,7 @@ "STEP_LOW_DCYCLE" : 10, "STEP_HIGH_DCYCLE" : 50, "EXPECTED_RESIDENCY_PCT" : 85, + "OFFLOAD_MIGRATION_MIGRATOR_DELAY": 1, "TEST_CONF" : { "modules" : [ "bl", "cpufreq" ], "tools" : [ "rt-app" ], diff --git a/tests/eas/hmp_parity.py b/tests/eas/hmp_parity.py index f89652471..7573c6ece 100644 --- a/tests/eas/hmp_parity.py +++ b/tests/eas/hmp_parity.py @@ -340,7 +340,7 @@ class OffloadMigrationAndIdlePull(unittest.TestCase): def populate_tasks(cls): migrator_workload = BIG_WORKLOAD.copy() migrator_workload["duration_s"] = 9 - migrator_workload["delay_s"] = 1 + migrator_workload["delay_s"] = OFFLOAD_MIGRATION_MIGRATOR_DELAY for idx in range(cls.num_tasks): task = "early_starters" + str(idx) -- GitLab From 5b399c512dcddcefb87b80ca611663e70d73ada2 Mon Sep 17 00:00:00 2001 From: Javi Merino Date: Mon, 15 Aug 2016 14:50:38 +0100 Subject: [PATCH 3/7] tests/eas/hmp_parity: cache the trappy.FTrace object in OffloadMigrationAndIdlePull In the OffloadMigrationAndIdlePull test we call SchedMultiAssert multiple times with the trace filename as a parameter, making each SchedMultiAssert create a trappy object for itself. Call trappy.FTrace from the test itself so that we can pass a reference to it to all the SchedMultiAssert invocations, avoiding multiple copies of effectively the same thing. --- tests/eas/hmp_parity.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/eas/hmp_parity.py b/tests/eas/hmp_parity.py index 7573c6ece..481a94c63 100644 --- a/tests/eas/hmp_parity.py +++ b/tests/eas/hmp_parity.py @@ -321,15 +321,12 @@ class OffloadMigrationAndIdlePull(unittest.TestCase): cls.offset = cls.get_offset(cls.early_starters[0]) - cls.m_assert = SchedMultiAssert( - cls.trace_file, - cls.env.topology, - execnames=cls.migrators) + cls.trace = trappy.FTrace(cls.trace_file) + cls.m_assert = SchedMultiAssert(cls.trace, cls.env.topology, + execnames=cls.migrators) + cls.e_assert = SchedMultiAssert(cls.trace, cls.env.topology, + execnames=cls.early_starters) - cls.e_assert = SchedMultiAssert( - cls.trace_file, - cls.env.topology, - execnames=cls.early_starters) cls.log_fh = open(os.path.join(cls.env.res_dir, cls.log_file), "w") @classmethod @@ -370,7 +367,7 @@ class OffloadMigrationAndIdlePull(unittest.TestCase): @classmethod def get_offset(cls, task_name): return SchedAssert( - cls.trace_file, + cls.trace, cls.env.topology, execname=task_name).getStartTime() -- GitLab From 7b7bc98763759a250362720570cf9f11908efb4c Mon Sep 17 00:00:00 2001 From: Javi Merino Date: Fri, 12 Aug 2016 19:42:33 +0100 Subject: [PATCH 4/7] tests/eas/hmp_parity: refactor the OffloadMigrationAndIdlePull test The Offload Migration test assumes that the early starters will run in the big cpus until completion and then the "migrator" tasks are going to be pulled from the littles. Recent versions of EAS may not follow this pattern exactly, even though their behaviour is still valid. It is possible for a migrator task to displace an early_starter task and run in a big cpu since it starts running, forcing the early starter to run on a little cpu until a big cpu is available. The new tests now check that the big cpus were fully busy as long as there are any big tasks still active and that all the tasks got to run in the big cpus at some point. This verifies the idle pull behaviour but still allows for early starters to be displaced. --- tests/eas/hmp_parity.config | 1 + tests/eas/hmp_parity.py | 199 +++++++++++++++++++----------------- 2 files changed, 104 insertions(+), 96 deletions(-) diff --git a/tests/eas/hmp_parity.config b/tests/eas/hmp_parity.config index b693a359e..eb3be962e 100644 --- a/tests/eas/hmp_parity.config +++ b/tests/eas/hmp_parity.config @@ -9,6 +9,7 @@ "STEP_HIGH_DCYCLE" : 50, "EXPECTED_RESIDENCY_PCT" : 85, "OFFLOAD_MIGRATION_MIGRATOR_DELAY": 1, + "EXPECTED_BIG_BUSY_TIME_PCT": 90, "TEST_CONF" : { "modules" : [ "bl", "cpufreq" ], "tools" : [ "rt-app" ], diff --git a/tests/eas/hmp_parity.py b/tests/eas/hmp_parity.py index 481a94c63..443db048c 100644 --- a/tests/eas/hmp_parity.py +++ b/tests/eas/hmp_parity.py @@ -291,9 +291,10 @@ class OffloadMigrationAndIdlePull(unittest.TestCase): This test runs twice as many tasks are there are big cpus. All these tasks are big tasks. Half of them are called "early_starter" and the other half "migrator". The migrator tasks - start 1 second after the early_starter tasks. The test checks - that when the big cpus finish executing the early starter tasks, - the scheduler moves the migrator tasks to the big cpus. + start 1 second after the early_starter tasks. As the big cpus are + fully utilized when the migrator tasks start, some tasks are + offloaded to the little cpus. As the big cpus finish their tasks, + they pull tasks from the little to complete them. Expected Behaviour ================== @@ -304,6 +305,21 @@ class OffloadMigrationAndIdlePull(unittest.TestCase): in the big cpus so they run on the little cpus. Once the big cpus finish with the early_starters, they should pull the migrator tasks and run them. + + It is possible that when the migrator tasks start they do it on + big cpus and they end up displacing the early starters. This is + acceptable behaviour. As long as big cpus are fully utilized + running big tasks, the scheduler is doing a good job. + + That is why this test doesn't test for migrations of the migrator + tasks to the bigs when we expect that the early starters have + finished. Instead, it tests that the big cpus were fully loaded + as long as there are tasks left to run in the system; that the + little cpus run tasks while the bigs are busy (offload migration); + that all tasks get a chance on a big cpu (either because they + started there or because of idle pull); and that all tasks are + finished off in a big cpu. + """ @classmethod @@ -319,14 +335,17 @@ class OffloadMigrationAndIdlePull(unittest.TestCase): local_setup(cls.env) cls.run_workload() - cls.offset = cls.get_offset(cls.early_starters[0]) - cls.trace = trappy.FTrace(cls.trace_file) cls.m_assert = SchedMultiAssert(cls.trace, cls.env.topology, execnames=cls.migrators) cls.e_assert = SchedMultiAssert(cls.trace, cls.env.topology, execnames=cls.early_starters) + all_tasks = cls.early_starters + cls.migrators + cls.a_assert = SchedMultiAssert(cls.trace, cls.env.topology, + execnames=all_tasks) + + cls.end_times = cls.calculate_end_times() cls.log_fh = open(os.path.join(cls.env.res_dir, cls.log_file), "w") @classmethod @@ -365,11 +384,15 @@ class OffloadMigrationAndIdlePull(unittest.TestCase): trace = cls.env.ftrace.get_trace(cls.trace_file) @classmethod - def get_offset(cls, task_name): - return SchedAssert( - cls.trace, - cls.env.topology, - execname=task_name).getStartTime() + def calculate_end_times(cls): + + end_times = {} + for task in cls.params.keys(): + sched_assert = SchedAssert(cls.trace, cls.env.topology, + execname=task) + end_times[task] = sched_assert.getEndTime() + + return end_times def test_first_cpu_early_starters(self): """Offload Migration and Idle Pull: Test First CPU (Early Starters)""" @@ -400,108 +423,92 @@ class OffloadMigrationAndIdlePull(unittest.TestCase): rank=self.num_tasks), msg="Not all the new 'migrator' tasks started on a big CPU") - def test_little_res_migrators(self): - "Offload Migration and Idle Pull: Test Little Residency (Migrators)" - little_residency_window = (self.offset + 1, self.offset + 5) + def test_big_cpus_fully_loaded(self): + """Offload Migration and Idle Pull: Big cpus are fully loaded as long as there are tasks left to run in the system""" + num_big_cpus = len(self.env.target.bl.bigs) - logging.info( - "Offload Migration and Idle Pull: Test Little Residency (Migrators)") + end_times = sorted(self.end_times.values()) - log_result( - self.m_assert.getResidency( - "cluster", - self.env.target.bl.littles, - percent=True, - window=little_residency_window - ), self.log_fh) + # Window of time until the first migrator finishes + window = (0, end_times[-num_big_cpus]) + busy_time = self.a_assert.getCPUBusyTime("cluster", + self.env.target.bl.bigs, + window=window, percent=True) + msg = "Big cpus were not fully loaded while there were enough big tasks to fill them" + self.assertGreater(busy_time, EXPECTED_BIG_BUSY_TIME_PCT, msg=msg) - self.assertTrue( - self.m_assert.assertResidency( - "cluster", - self.env.target.bl.littles, - EXPECTED_RESIDENCY_PCT, - operator.ge, - percent=True, - window=little_residency_window, - rank=self.num_tasks), - msg="Not all 'migrator' tasks are running on LITTLE cores for at least {}% of their execution time"\ - .format(EXPECTED_RESIDENCY_PCT)) + # As the migrators start finishing, make sure that the tasks + # that are left are running on the big cpus + for i in range(num_big_cpus-1): + big_cpus_left = num_big_cpus - i - 1 + window = (end_times[-num_big_cpus+i], end_times[-num_big_cpus+i+1]) + busy_time = self.a_assert.getCPUBusyTime("cluster", + self.env.target.bl.bigs, + window=window, percent=True) - def test_big_res_migrators(self): - "Offload Migration and Idle Pull: Test Big Residency (Migrators)" - big_residency_window = (self.offset + 5, self.offset + 10) + expected_busy_time = EXPECTED_BIG_BUSY_TIME_PCT * \ + big_cpus_left / num_big_cpus + msg = "Big tasks were not running on big cpus from {} to {}".format( + window[0], window[1]) - logging.info( - "Offload Migration and Idle Pull: Test Big Residency (Migrators)") + self.assertGreater(busy_time, expected_busy_time, msg=msg) - log_result( - self.m_assert.getResidency( - "cluster", - self.env.target.bl.bigs, - percent=True, - window=big_residency_window - ), self.log_fh) + def test_little_cpus_run_tasks(self): + """Offload Migration and Idle Pull: Little cpus run tasks while bigs are busy""" + tasks = self.params.keys() + num_offloaded_tasks = len(tasks) / 2 - self.assertTrue( - self.m_assert.assertResidency( - "cluster", - self.env.target.bl.bigs, - EXPECTED_RESIDENCY_PCT, - operator.ge, - percent=True, - window=big_residency_window, - rank=self.num_tasks), - msg="Not all 'migrator' tasks are running on big cores for at least {}% of their execution time"\ - .format(EXPECTED_RESIDENCY_PCT)) + first_task_finish_time = None + for task in tasks: + end_time = self.end_times[task] + if not first_task_finish_time or (end_time < first_task_finish_time): + first_task_finish_time = end_time + window = (OFFLOAD_MIGRATION_MIGRATOR_DELAY, first_task_finish_time) + busy_time = self.a_assert.getCPUBusyTime("cluster", + self.env.target.bl.littles, + window=window) - def test_migrators_switch(self): - "Offload Migration and Idle Pull: Test LITTLE -> BIG Idle Pull Switch (Migrators)" - switch_window = (self.offset + 4.5, self.offset + 5.5) + window_len = window[1] - window[0] + expected_busy_time = window_len * num_offloaded_tasks * \ + EXPECTED_BIG_BUSY_TIME_PCT / 100. + msg = "Little cpus did not pick up big tasks while big cpus were fully loaded" - logging.info( - "Offload Migration and Idle Pull: Test LITTLE -> BIG Idle Pull Switch (Migrators)") - log_result( - self.m_assert.assertSwitch( - "cluster", - self.env.target.bl.littles, - self.env.target.bl.bigs, - window=switch_window), self.log_fh) + self.assertGreater(busy_time, expected_busy_time, msg=msg) - self.assertTrue( - self.m_assert.assertSwitch( - "cluster", - self.env.target.bl.littles, - self.env.target.bl.bigs, - window=switch_window, - rank=self.num_tasks), - msg="Not all 'migrator' tasks are pulled by idle big cores when expected") + def test_all_tasks_run_on_a_big_cpu(self): + """Offload Migration and Idle Pull: All tasks run on a big cpu at some point + Note: this test may fail in big.LITTLE platforms in which the + little cpus are almost as performant as the big ones. - def test_big_res_early_starters(self): - """Offload Migration and Idle Pull: Test Big Residency (EarlyStarters)""" - logging.info( - "Offload Migration and Idle Pull: Test Big Residency (EarlyStarters)") + """ - big_residency_window = (self.offset, self.offset + 5) + for task in self.params.keys(): + sa = SchedAssert(self.trace, self.env.topology, execname=task) + window = (0, self.end_times[task]) + big_residency = sa.getResidency("cluster", self.env.target.bl.bigs, + window=window, percent=True) + log_result(big_residency, self.log_fh) - log_result(self.e_assert.getResidency( - "cluster", - self.env.target.bl.bigs, - percent=True, - window=big_residency_window), self.log_fh) + msg = "Task {} didn't run on a big cpu.".format(task) + self.assertGreater(big_residency, 0, msg=msg) - self.assertTrue( - self.e_assert.assertResidency( - "cluster", - self.env.target.bl.bigs, - EXPECTED_RESIDENCY_PCT, - operator.ge, - percent=True, - window=big_residency_window, - rank=self.num_tasks), - msg="Not all 'early starter' tasks are running on big cores for at least {}% of their execution time"\ - .format(EXPECTED_RESIDENCY_PCT)) + def test_all_tasks_finish_on_a_big_cpu(self): + """Offload Migration and Idle Pull: All tasks finish on a big cpu + + Note: this test may fail in big.LITTLE systems where the + little cpus' performance is comparable to the bigs' and they + can take almost the same time as a big cpu to complete a + task. + + """ + + for task in self.params.keys(): + sa = SchedAssert(self.trace, self.env.topology, execname=task) + + msg = "Task {} did not finish on a big cpu".format(task) + self.assertIn(sa.getLastCpu(), self.env.target.bl.bigs, msg=msg) class WakeMigration(unittest.TestCase): -- GitLab From d819214f6355cc00fc6e260e9547c6489dd7cff5 Mon Sep 17 00:00:00 2001 From: Javi Merino Date: Thu, 18 Aug 2016 15:13:44 +0100 Subject: [PATCH 5/7] tests/eas/hmp_parity: don't hardcode the number of tasks in WakeMigration WakeMigration creates two tasks, which is ok if the test system has two or more big cpus, but it will fail with one big cpu. We don't need to hardcode the number of tasks, we can generalize the test by creating as many as there are big cpus. --- tests/eas/hmp_parity.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/eas/hmp_parity.py b/tests/eas/hmp_parity.py index 443db048c..57d92aaa2 100644 --- a/tests/eas/hmp_parity.py +++ b/tests/eas/hmp_parity.py @@ -521,10 +521,11 @@ class WakeMigration(unittest.TestCase): Detailed Description ==================== - This test creates two tasks that alternate between being big and - small workload. They start being small load for 5 seconds, they - become big for another 5 seconds, then small for another 5 seconds - and finally big for the last 5 seconds. + This test creates as many tasks as there are big cpus. The tasks + alternate between high and low utilization. They start being + small load for 5 seconds, they become big for another 5 seconds, + then small for another 5 seconds and finally big for the last 5 + seconds. Expected Behaviour ================== @@ -558,8 +559,11 @@ class WakeMigration(unittest.TestCase): @classmethod def populate_params(cls): - cls.params[cls.task_prefix] = Step(**STEP_WORKLOAD).get() - cls.params[cls.task_prefix + "1"] = Step(**STEP_WORKLOAD).get() + num_big_cpus = len(cls.env.target.bl.bigs) + + for i in range(num_big_cpus): + task_name = "{}_{}".format(cls.task_prefix, i) + cls.params[task_name] = Step(**STEP_WORKLOAD).get() @classmethod def run_workload(cls): -- GitLab From 42326735f71163a7ac3a4d8dfb971b52a6b7dc43 Mon Sep 17 00:00:00 2001 From: Javi Merino Date: Tue, 13 Sep 2016 11:54:14 +0100 Subject: [PATCH 6/7] tests/eas/hmp_parity: don't hardcode the phase duration in WakeMigration The tests in WakeMigration hardcode the duration of the phase. Take the task duration from the configuration and give it a proper name to avoid the magic number. Document it while we are at it. --- tests/eas/hmp_parity.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/eas/hmp_parity.py b/tests/eas/hmp_parity.py index 57d92aaa2..10a4e6eca 100644 --- a/tests/eas/hmp_parity.py +++ b/tests/eas/hmp_parity.py @@ -565,6 +565,8 @@ class WakeMigration(unittest.TestCase): task_name = "{}_{}".format(cls.task_prefix, i) cls.params[task_name] = Step(**STEP_WORKLOAD).get() + cls.phase_duration = STEP_WORKLOAD["time_s"] + @classmethod def run_workload(cls): wload = RTA( @@ -602,7 +604,7 @@ class WakeMigration(unittest.TestCase): def test_little_big_switch1(self): """Wake Migration: LITTLE -> BIG: 1""" - expected_time = self.offset + 5 + expected_time = self.offset + self.phase_duration switch_window = ( expected_time - SWITCH_WINDOW_HALF, @@ -632,7 +634,11 @@ class WakeMigration(unittest.TestCase): def test_little_big_switch2(self): """Wake Migration: LITTLE -> BIG: 2""" - expected_time = self.offset + 15 + # little - big - little - big + # ^ + # We want to test that this little to big migration happens. So we skip + # the first three phases. + expected_time = self.offset + 3 * self.phase_duration switch_window = ( expected_time - SWITCH_WINDOW_HALF, @@ -688,7 +694,11 @@ class WakeMigration(unittest.TestCase): def test_big_little_switch2(self): """Wake Migration: BIG -> LITLLE: 2""" - expected_time = self.offset + 10 + # little - big - little - big + # ^ + # We want to test that this big to little migration happens. So we skip + # the first two phases. + expected_time = self.offset + 2 * self.phase_duration switch_window = ( expected_time - SWITCH_WINDOW_HALF, -- GitLab From 4c29d17aeee9018d238b1a1a0e6825b840cbdcb4 Mon Sep 17 00:00:00 2001 From: Javi Merino Date: Thu, 8 Sep 2016 14:56:18 +0100 Subject: [PATCH 7/7] tests/eas/hmp_parity: add some more interesting sched ftrace events HMP parity tests run for only a few seconds. We can collect more ftrace events on each run as they should not use too much space and can help in debugging issues. sched_energy_diff, sched_load_avg_task and sched_load_avg_cpu are not in mainline but they are part of the EAS patchset. I'm including them because it is expected that the test will be run on systems with the EAS patches applied. --- tests/eas/hmp_parity.config | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/eas/hmp_parity.config b/tests/eas/hmp_parity.config index eb3be962e..9a80413b9 100644 --- a/tests/eas/hmp_parity.config +++ b/tests/eas/hmp_parity.config @@ -15,6 +15,10 @@ "tools" : [ "rt-app" ], "ftrace" : { "events" : [ + "sched_energy_diff", + "sched_load_avg_task", + "sched_load_avg_cpu", + "sched_migrate_task", "sched_switch" ], "buffsize" : 10240 -- GitLab