diff options
Diffstat (limited to 'testing/mozbase/mozpower')
19 files changed, 2648 insertions, 0 deletions
diff --git a/testing/mozbase/mozpower/mozpower/__init__.py b/testing/mozbase/mozpower/mozpower/__init__.py new file mode 100644 index 0000000000..388ad721aa --- /dev/null +++ b/testing/mozbase/mozpower/mozpower/__init__.py @@ -0,0 +1,33 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +from .intel_power_gadget import ( + IPGEmptyFileError, + IPGMissingOutputFileError, + IPGTimeoutError, + IPGUnknownValueTypeError, +) + +from .mozpower import ( + MissingProcessorInfoError, + MozPower, + OsCpuComboMissingError, +) + +from .powerbase import IPGExecutableMissingError, PlatformUnsupportedError + + +__all__ = [ + "IPGEmptyFileError", + "IPGExecutableMissingError", + "IPGMissingOutputFileError", + "IPGTimeoutError", + "IPGUnknownValueTypeError", + "MissingProcessorInfoError", + "MozPower", + "OsCpuComboMissingError", + "PlatformUnsupportedError", +] diff --git a/testing/mozbase/mozpower/mozpower/intel_power_gadget.py b/testing/mozbase/mozpower/mozpower/intel_power_gadget.py new file mode 100644 index 0000000000..f6a32ac9ed --- /dev/null +++ b/testing/mozbase/mozpower/mozpower/intel_power_gadget.py @@ -0,0 +1,912 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import, division, unicode_literals + +import csv +import os +import subprocess +import re +import time +import threading + +from .mozpowerutils import get_logger + + +class IPGTimeoutError(Exception): + """IPGTimeoutError is raised when we cannot stop Intel Power + Gadget from running. One possble cause of this is not calling + `stop_ipg` through a `finalize_power_measurements` call. The + other possiblity is that IPG failed to stop. + """ + + pass + + +class IPGMissingOutputFileError(Exception): + """IPGMissingOutputFile is raised when a file path is given + to _clean_ipg_file but it does not exist or cannot be found + at the expected location. + """ + + pass + + +class IPGEmptyFileError(Exception): + """IPGEmptyFileError is raised when a file path is given + to _clean_ipg_file and it exists but it is empty (contains + no results to clean). + """ + + pass + + +class IPGUnknownValueTypeError(Exception): + """IPGUnknownValueTypeError is raised when a value within a + given results file (that was cleaned) cannot be converted to + its column's expected data type. + """ + + pass + + +class IntelPowerGadget(object): + """IntelPowerGadget provides methods for using Intel Power Gadget + to measure power consumption. + + :: + + from mozpower.intel_power_gadget import IntelPowerGadget + + # On Mac, the ipg_exe_path should point to '/Applications/Intel Power Gadget/PowerLog' + ipg = IntelPowerGadget(ipg_exe_path, ipg_measure_duration=600, sampling_rate=500) + + ipg.start_ipg() + + # Run tests... + + ipg.stop_ipg() + + # Now process results with IPGResultsHandler by passing it + # ipg.output_files, ipg.output_file_ext, ipg.ipg_measure_duration, + # and ipg.sampling_rate. + """ + + def __init__( + self, + exe_file_path, + ipg_measure_duration=10, + sampling_rate=1000, + output_file_ext=".txt", + file_counter=1, + output_file_path="powerlog", + logger_name="mozpower", + ): + """Initializes the IntelPowerGadget object. + + :param str exe_file_path: path to Intel Power Gadget 'PowerLog' executable. + :param int ipg_measure_duration: duration to run IPG for in seconds. + This does not dictate when the tools shuts down. It only stops when stop_ipg + is called in case the test runs for a non-deterministic amount of time. The + IPG executable requires a duration to be supplied. The machinery in place is + to ensure that IPG keeps recording as long as the experiment runs, so + multiple files may result, which is handled in + IPGResultsHandler._combine_cumulative_rows. Defaults to 10s. + :param int sampling_rate: sampling rate of measurements in milliseconds. + Defaults to 1000ms. + :param output_file_ext: file extension of data being output. Defaults to '.txt'. + :param int file_counter: dictates the start of the file numbering (used + when test time exceeds duration value). Defaults to 0. + :param str output_file_path: path to the output location combined + with the output file prefix. Defaults to current working directory using the + prefix 'powerlog'. + :param str logger_name: logging logger name. Defaults to 'mozpower'. + """ + self._logger = get_logger(logger_name) + + # Output-specific settings + self._file_counter = file_counter + self._output_files = [] + self._output_file_path = output_file_path + self._output_file_ext = output_file_ext + self._output_dir_path, self._output_file_prefix = os.path.split( + self._output_file_path + ) + + # IPG-specific settings + self._ipg_measure_duration = ipg_measure_duration # in seconds + self._sampling_rate = sampling_rate # in milliseconds + self._exe_file_path = exe_file_path + + # Setup thread for measurement gathering + self._thread = threading.Thread( + target=self.run, args=(exe_file_path, ipg_measure_duration) + ) + self._thread.daemon = True + self._running = False + + @property + def output_files(self): + """Returns the list of files produced from running IPG. + + :returns: list + """ + return self._output_files + + @property + def output_file_ext(self): + """Returns the extension of the files produced by IPG. + + :returns: str + """ + return self._output_file_ext + + @property + def output_file_prefix(self): + """Returns the prefix of the files produces by IPG. + + :returns: str + """ + return self._output_file_prefix + + @property + def output_dir_path(self): + """Returns the output directory of the files produced by IPG. + + :returns: str + """ + return self._output_dir_path + + @property + def sampling_rate(self): + """Returns the specified sampling rate. + + :returns: int + """ + return self._sampling_rate + + @property + def ipg_measure_duration(self): + """Returns the IPG measurement duration (see __init__ for description + of what this value does). + + :returns: int + """ + return self._ipg_measure_duration + + def start_ipg(self): + """Starts the thread which runs IPG to start gathering measurements.""" + self._logger.info("Starting IPG thread") + self._stop = False + self._thread.start() + + def stop_ipg(self, wait_interval=10, timeout=200): + """Stops the thread which runs IPG and waits for it to finish it's work. + + :param int wait_interval: interval time (in seconds) at which to check if + IPG is finished. + :param int timeout: time to wait until _wait_for_ipg hits a time out, + in seconds. + """ + self._logger.info("Stopping IPG thread") + self._stop = True + self._wait_for_ipg(wait_interval=wait_interval, timeout=timeout) + + def _wait_for_ipg(self, wait_interval=10, timeout=200): + """Waits for IPG to finish it's final dataset. + + :param int wait_interval: interval time (in seconds) at which to check if + IPG is finished. + :param int timeout: time to wait until this method hits a time out, + in seconds. + :raises: * IPGTimeoutError + """ + timeout_stime = time.time() + while self._running and (time.time() - timeout_stime) < timeout: + self._logger.info( + "Waiting %s sec for Intel Power Gadget to stop" % wait_interval + ) + time.sleep(wait_interval) + if self._running: + raise IPGTimeoutError("Timed out waiting for IPG to stop") + + def _get_output_file_path(self): + """Returns the next output file name to be used. Starts at 1 and increases + at each successive call. Used when the test duration exceeds the specified + duration through ipg_meaure_duration. + + :returns: str + """ + self._output_file_path = os.path.join( + self._output_dir_path, + "%s_%s_%s" + % (self._output_file_prefix, self._file_counter, self._output_file_ext), + ) + self._file_counter += 1 + self._logger.info( + "Creating new file for IPG data measurements: %s" % self._output_file_path + ) + return self._output_file_path + + def run(self, exe_file_path, ipg_measure_duration): + """Runs the IPG measurement gatherer. While stop has not been set to True, + it continuously gathers measurements into separate files that are merged + by IPGResultsHandler once the output_files are passed to it. + + :param str exe_file_path: file path of where to find the IPG executable. + :param int ipg_measure_duration: time to gather measurements for. + """ + self._logger.info("Starting to gather IPG measurements in thread") + self._running = True + + while not self._stop: + outname = self._get_output_file_path() + self._output_files.append(outname) + + try: + subprocess.check_output( + [ + exe_file_path, + "-duration", + str(ipg_measure_duration), + "-resolution", + str(self._sampling_rate), + "-file", + outname, + ], + stderr=subprocess.STDOUT, + ) + except subprocess.CalledProcessError as e: + error_log = str(e) + if e.output: + error_log = e.output.decode() + self._logger.critical( + "Error while running Intel Power Gadget: %s" % error_log + ) + + self._running = False + + +class IPGResultsHandler(object): + """IPGResultsHandler provides methods for cleaning and formatting + the files that were created by IntelPowerGadget. + + :: + + from mozpower.intel_power_gadget import IntelPowerGadget, IPGResultsHandler + + ipg = IntelPowerGadget(ipg_exe_path, ipg_measure_duration=600, sampling_rate=500) + + # Run tests - call start_ipg and stop_ipg... + + ipg_rh = IPGResultsHandler( + ipg.output_files, + ipg.output_dir_path, + ipg_measure_duration=ipg.ipg_measure_duration, + sampling_rate=ipg.sampling_rate + ) + + cleaned_data = ipg_rh.clean_ipg_data() + # you can also get the data from results after calling clean_ipg_data + cleaned_data = ipg_rh.results + + perfherder_data = ipg_rh.format_ipg_data_to_partial_perfherder( + experiment_duration, + test_name + ) + # you can also get the perfherder data from summarized_results + # after calling format_ipg_data_to_partial_perfherder + perfherder_data = ipg_rh.summarized_results + """ + + def __init__( + self, + output_files, + output_dir_path, + ipg_measure_duration=10, + sampling_rate=1000, + logger_name="mozpower", + ): + """Initializes the IPGResultsHandler object. + + :param list output_files: files output by IntelPowerGadget containing + the IPG data. + :param str output_dir_path: location to store cleaned files and merged data. + :param int ipg_measure_duration: length of time that each IPG measurement lasted + in seconds (see IntelPowerGadget for more information on this argument). + Defaults to 10s. + :param int sampling_rate: sampling rate of the measurements in milliseconds. + Defaults to 1000ms. + :param str logger_name: logging logger name. Defaults to 'mozpower'. + """ + self._logger = get_logger(logger_name) + self._results = {} + self._summarized_results = {} + self._cleaned_files = [] + self._csv_header = None + self._merged_output_path = None + self._output_file_prefix = None + self._output_file_ext = None + + self._sampling_rate = sampling_rate + self._ipg_measure_duration = ipg_measure_duration + self._output_files = output_files + self._output_dir_path = output_dir_path + + if self._output_files: + # Gather the output file extension, and prefix + # for the cleaned files, and the merged file. + single_file = self._output_files[0] + _, file = os.path.split(single_file) + self._output_file_ext = "." + file.split(".")[-1] + + # This prefix detection depends on the path names created + # by _get_output_file_path. + integer_re = re.compile(r"""(.*)_[\d+]_%s""" % self._output_file_ext) + match = integer_re.match(file) + if match: + self._output_file_prefix = match.group(1) + else: + self._output_file_prefix = file.split("_")[0] + self._logger.warning( + "Cannot find output file prefix from output file name %s" + "using the following prefix: %s" % (file, self._output_file_prefix) + ) + + @property + def results(self): + """Returns the cleaned IPG data in the form of a dict. + Each key is a measurement name with the values being the list + of measurements. All value lists are sorted in increasing time. + + :returns: dict + """ + return self._results + + @property + def summarized_results(self): + """Returns IPG data in the form of a dict that is formatted + into a perfherder data blob. + + :returns: dict + """ + return self._summarized_results + + @property + def cleaned_files(self): + """Returns a list of cleaned IPG data files that were output + after running clean_ipg_data. + + :returns: list + """ + return self._cleaned_files + + @property + def merged_output_path(self): + """Returns the path to the cleaned, and merged output file. + + :returns: str + """ + return self._merged_output_path + + def _clean_ipg_file(self, file): + """Cleans an individual IPG file and writes out a cleaned file. + + An uncleaned file looks like this (this contains a partial + sample of the time series data): + ``` + "System Time","RDTSC","Elapsed Time (sec)" + "12:11:05:769","61075218263548"," 2.002" + "12:11:06:774","61077822279584"," 3.007" + "12:11:07:778","61080424421708"," 4.011" + "12:11:08:781","61083023535972"," 5.013" + "12:11:09:784","61085623402302"," 6.016" + + "Total Elapsed Time (sec) = 10.029232" + "Measured RDTSC Frequency (GHz) = 2.592" + + "Cumulative Processor Energy_0 (Joules) = 142.337524" + "Cumulative Processor Energy_0 (mWh) = 39.538201" + "Average Processor Power_0 (Watt) = 14.192265" + + "Cumulative IA Energy_0 (Joules) = 121.888000" + "Cumulative IA Energy_0 (mWh) = 33.857778" + "Average IA Power_0 (Watt) = 12.153273" + + "Cumulative DRAM Energy_0 (Joules) = 7.453308" + "Cumulative DRAM Energy_0 (mWh) = 2.070363" + "Average DRAM Power_0 (Watt) = 0.743158" + + "Cumulative GT Energy_0 (Joules) = 0.079834" + "Cumulative GT Energy_0 (mWh) = 0.022176" + "Average GT Power_0 (Watt) = 0.007960" + ``` + + While a cleaned file looks like: + ``` + System Time,RDTSC,Elapsed Time (sec) + 12:11:05:769,61075218263548, 2.002 + 12:11:06:774,61077822279584, 3.007 + 12:11:07:778,61080424421708, 4.011 + 12:11:08:781,61083023535972, 5.013 + 12:11:09:784,61085623402302, 6.016 + ``` + + The first portion of the file before the '"Total Elapsed Time' entry is + considered as the time series which is captured as a dict in the returned + results. + + All the text starting from '"Total Elapsed Time' is removed from the file and + is considered as a summary of the experiment (which is returned). This + portion is ignored in any other processing done by IPGResultsHandler, + and is not saved, unlike the returned results value. + + Note that the new lines are removed from the summary that is returned + (as well as the results). + + :param str file: file to clean. + :returns: a tuple of (dict, list, str) for + (results, summary, clean_output_path) + see method comments above for information on what + this output contains. + :raises: * IPGMissingOutputFileError + * IPGEmptyFileError + """ + self._logger.info("Cleaning IPG data file %s" % file) + + txt = "" + if os.path.exists(file): + with open(file, "r") as f: + txt = f.read() + else: + # This should never happen, so prevent IPGResultsHandler + # from continuing to clean files if it does occur. + raise IPGMissingOutputFileError( + "The following file does not exist so it cannot be cleaned: %s " % file + ) + + if txt == "": + raise IPGEmptyFileError( + "The following file is empty so it cannot be cleaned: %s" % file + ) + + # Split the time series data from the summary + tseries, summary = re.split('"Total Elapsed Time', txt) + + # Clean the summary + summary = '"Total Elapsed Time' + summary + summary = [ + line for line in re.split(r"""\n""", summary.replace("\r", "")) if line + ] + + # Clean the time series data, store the clean rows to write out later, + # and format the rows into a dict entry for each measure. + results = {} + clean_rows = [] + csv_header = None + for c, row in enumerate( + csv.reader(tseries.split("\n"), quotechar=str('"'), delimiter=str(",")) + ): + if not row: + continue + + # Make sure we don't have any bad line endings + # contaminating the cleaned rows. + fmt_row = [ + val.replace("\n", "") + .replace("\t", "") + .replace("\r", "") + .replace("\\n", "") + .strip() + for val in row + ] + + if not fmt_row or not any(fmt_row): + continue + + clean_rows.append(fmt_row) + if c == 0: + csv_header = fmt_row + for col in fmt_row: + results[col] = [] + continue + for i, col in enumerate(fmt_row): + results[csv_header[i]].append(col) + + # Write out the cleaned data and check to make sure + # the csv header hasn't changed mid-experiment + _, fname = os.path.split(file) + clean_output_path = os.path.join( + self._output_dir_path, + fname.replace( + self._output_file_ext, + "_clean.%s" % self._output_file_ext.replace(".", ""), + ), + ) + self._logger.info("Writing cleaned IPG results to %s" % clean_output_path) + try: + with open(clean_output_path, "w") as csvfile: + writer = csv.writer(csvfile) + for count, row in enumerate(clean_rows): + if count == 0: + if self._csv_header is None: + self._csv_header = row + elif self._csv_header != row: + self._logger.warning( + "CSV Headers from IPG data have changed during the experiment " + "expected: %s; got: %s" + % (str(self._csv_header), str(row)) + ) + writer.writerow(row) + except Exception as e: + self._logger.warning( + "Could not write out cleaned results of %s to %s due to the following error" + ", skipping this step: %s" % (file, clean_output_path, str(e)) + ) + + # Check to make sure the expected number of samples + # exist in the file and that the columns have the correct + # data type. + column_datatypes = {"System Time": str, "RDTSC": int, "default": float} + # pylint --py3k W1619 + expected_samples = int( + self._ipg_measure_duration / (float(self._sampling_rate) / 1000) + ) + for key in results: + if len(results[key]) != expected_samples: + self._logger.warning( + "Unexpected number of samples in %s for column %s - " + "expected: %s, got: %s" + % (clean_output_path, key, expected_samples, len(results[key])) + ) + + dtype = column_datatypes["default"] + if key in column_datatypes: + dtype = column_datatypes[key] + + for val in results[key]: + try: + # Check if converting from a string to its expected data + # type works, if not, it's not the expected data type. + dtype(val) + except ValueError as e: + raise IPGUnknownValueTypeError( + "Cleaned file %s entry %s in column %s has unknown type " + "instead of the expected type %s - data corrupted, " + "cannot continue: %s" + % (clean_output_path, str(val), key, dtype.__name__, str(e)) + ) + + # Check to make sure that IPG measured for the expected + # amount of time. + etime = "Elapsed Time (sec)" + if etime in results: + total_time = int(float(results[etime][-1])) + if total_time != self._ipg_measure_duration: + self._logger.warning( + "Elapsed time found in file %s is different from expected length - " + "expected: %s, got: %s" + % (clean_output_path, self._ipg_measure_duration, total_time) + ) + else: + self._logger.warning( + "Missing 'Elapsed Time (sec)' in file %s" % clean_output_path + ) + + return results, summary, clean_output_path + + def _combine_cumulative_rows(self, cumulatives): + """Combine cumulative rows from multiple IPG files into + a single time series. + + :param list cumulatives: list of cumulative time series collected + over time, on for each file. Must be ordered in increasing time. + :returns: list + """ + combined_cumulatives = [] + + val_mod = 0 + for count, cumulative in enumerate(cumulatives): + # Add the previous file's maximum cumulative value + # to the current one + mod_cumulative = [val_mod + float(val) for val in cumulative] + val_mod = mod_cumulative[-1] + combined_cumulatives.extend(mod_cumulative) + + return combined_cumulatives + + def clean_ipg_data(self): + """Cleans all IPG files, or data, that was produced by an IntelPowerGadget object. + Returns a dict containing the merged data from all files. + + :returns: dict + """ + self._logger.info("Cleaning IPG data...") + + if not self._output_files: + self._logger.warning("No IPG files to clean.") + return + # If this is called a second time for the same set of files, + # then prevent it from duplicating the cleaned file entries. + if self._cleaned_files: + self._cleaned_files = [] + + # Clean each file individually and gather the results + all_results = [] + for file in self._output_files: + results, summary, output_file_path = self._clean_ipg_file(file) + self._cleaned_files.append(output_file_path) + all_results.append(results) + + # Merge all the results into a single result + combined_results = {} + for measure in all_results[0]: + lmeasure = measure.lower() + if "cumulative" not in lmeasure and "elapsed time" not in lmeasure: + # For measures which are not cumulative, or elapsed time, + # combine them without changing the data. + for count, result in enumerate(all_results): + if "system time" in lmeasure or "rdtsc" in lmeasure: + new_results = result[measure] + else: + new_results = [float(val) for val in result[measure]] + + if count == 0: + combined_results[measure] = new_results + else: + combined_results[measure].extend(new_results) + else: + # For cumulative, and elapsed time measures, we need to + # modify all values - see _combine_cumulative_rows for + # more information on this procedure. + cumulatives = [result[measure] for result in all_results] + self._logger.info( + "Combining cumulative rows for '%s' measure" % measure + ) + combined_results[measure] = self._combine_cumulative_rows(cumulatives) + + # Write merged results to a new file + merged_output_path = os.path.join( + self._output_dir_path, + "%s_merged.%s" + % (self._output_file_prefix, self._output_file_ext.replace(".", "")), + ) + + self._merged_output_path = merged_output_path + self._logger.info("Writing merged IPG results to %s" % merged_output_path) + try: + with open(merged_output_path, "w") as csvfile: + writer = csv.writer(csvfile) + writer.writerow(self._csv_header) + + # Use _csv_header list to keep the header ordering + # the same as the cleaned and raw files. + first_key = list(combined_results.keys())[0] + for row_count, _ in enumerate(combined_results[first_key]): + row = [] + for measure in self._csv_header: + row.append(combined_results[measure][row_count]) + writer.writerow(row) + except Exception as e: + self.merged_output_path = None + self._logger.warning( + "Could not write out merged results to %s due to the following error" + ", skipping this step: %s" % (merged_output_path, str(e)) + ) + + # Check that the combined results have the expected number of samples. + # pylint W16919 + expected_samples = int( + self._ipg_measure_duration / (float(self._sampling_rate) / 1000) + ) + combined_expected_samples = len(self._cleaned_files) * expected_samples + for key in combined_results: + if len(combined_results[key]) != combined_expected_samples: + self._logger.warning( + "Unexpected number of merged samples in %s for column %s - " + "expected: %s, got: %s" + % ( + merged_output_path, + key, + combined_expected_samples, + len(results[key]), + ) + ) + + self._results = combined_results + return self._results + + def format_ipg_data_to_partial_perfherder(self, duration, test_name): + """Format the merged IPG data produced by clean_ipg_data into a + partial/incomplete perfherder data blob. Returns the perfherder + data and stores it in _summarized_results. + + This perfherder data still needs more information added to it before it + can be validated against the perfherder schema. The entries returned + through here are missing many required fields. Furthermore, the 'values' + entries need to be formatted into 'subtests'. + + Here is a sample of a complete perfherder data which uses + the 'utilization' results within the subtests: + ``` + { + "framework": {"name": "raptor"}, + "type": "power", + "unit": "mWh" + "suites": [ + { + "name": "raptor-tp6-amazon-firefox-power", + "lowerIsBetter": true, + "alertThreshold": 2.0, + "subtests": [ + { + "lowerIsBetter": true, + "unit": "%", + "name": "raptor-tp6-youtube-firefox-power-cpu", + "value": 14.409090909090908, + "alertThreshold": 2.0 + }, + { + "lowerIsBetter": true, + "unit": "%", + "name": "raptor-tp6-youtube-firefox-power-gpu", + "value": 20.1, + "alertThreshold": 2.0 + }, + ] + } + ] + } + ``` + + To obtain data that is formatted to a complete perfherder data blob, + see get_complete_perfherder_data in MozPower. + + :param float duration: the actual duration of the test in case some + data needs to be cut off from the end. + :param str test_name: the name of the test. + :returns: dict + """ + self._logger.info("Formatting cleaned IPG data into partial perfherder data") + + if not self._results: + self._logger.warning( + "No merged results found - cannot format data to perfherder format." + ) + return + + def replace_measure_name(name): + """Replaces the long IPG names with shorter versions. + Returns the given name if no conversions exist. + + :param str name: name of the entry to replace. + :returns: str + """ + lname = name.lower() + if "ia " in lname: + return "processor-cores" + elif "processor " in lname: + return "processor-package" + elif "gt " in lname: + return "gpu" + elif "dram " in lname: + return "dram" + else: + return name + + # Cut out entries which surpass the test duration. + # Occurs when IPG continues past the call to stop it. + cut_results = self._results + if duration: + cutoff_index = 0 + for count, etime in enumerate(self._results["Elapsed Time (sec)"]): + if etime > duration: + cutoff_index = count + break + if cutoff_index > 0: + for measure in self._results: + cut_results[measure] = self._results[measure][:cutoff_index] + + # Get the cumulative power used in mWh + cumulative_mwh = {} + for measure in cut_results: + if "cumulative" in measure.lower() and "mwh" in measure.lower(): + cumulative_mwh[replace_measure_name(measure)] = float( + cut_results[measure][-1] + ) + + # Get the power usage rate in Watts + watt_usage = {} + for measure in cut_results: + if "watt" in measure.lower() and "limit" not in measure.lower(): + # pylint --py3k W1619 + watt_usage[replace_measure_name(measure) + "-avg"] = sum( + [float(val) for val in cut_results[measure]] + ) / len(cut_results[measure]) + watt_usage[replace_measure_name(measure) + "-max"] = max( + [float(val) for val in cut_results[measure]] + ) + + # Get average CPU and GPU utilization + average_utilization = {} + for utilization in ("CPU Utilization(%)", "GT Utilization(%)"): + if utilization not in cut_results: + self._logger.warning( + "Could not find measurements for: %s" % utilization + ) + continue + + utilized_name = utilization.lower() + if "cpu " in utilized_name: + utilized_name = "cpu" + elif "gt " in utilized_name: + utilized_name = "gpu" + + # pylint --py3k W1619 + average_utilization[utilized_name] = sum( + [float(val) for val in cut_results[utilization]] + ) / len(cut_results[utilization]) + + # Get average and maximum CPU and GPU frequency + frequency_info = {"cpu": {}, "gpu": {}} + for frequency_measure in ("CPU Frequency_0(MHz)", "GT Frequency(MHz)"): + if frequency_measure not in cut_results: + self._logger.warning( + "Could not find measurements for: %s" % frequency_measure + ) + continue + + fmeasure_name = frequency_measure.lower() + if "cpu " in fmeasure_name: + fmeasure_name = "cpu" + elif "gt " in fmeasure_name: + fmeasure_name = "gpu" + # pylint --py3k W1619 + + frequency_info[fmeasure_name]["favg"] = sum( + [float(val) for val in cut_results[frequency_measure]] + ) / len(cut_results[frequency_measure]) + + frequency_info[fmeasure_name]["fmax"] = max( + [float(val) for val in cut_results[frequency_measure]] + ) + + frequency_info[fmeasure_name]["fmin"] = min( + [float(val) for val in cut_results[frequency_measure]] + ) + + summarized_results = { + "utilization": { + "type": "power", + "test": str(test_name) + "-utilization", + "unit": "%", + "values": average_utilization, + }, + "power-usage": { + "type": "power", + "test": str(test_name) + "-cumulative", + "unit": "mWh", + "values": cumulative_mwh, + }, + "power-watts": { + "type": "power", + "test": str(test_name) + "-watts", + "unit": "W", + "values": watt_usage, + }, + "frequency-cpu": { + "type": "power", + "test": str(test_name) + "-frequency-cpu", + "unit": "MHz", + "values": frequency_info["cpu"], + }, + "frequency-gpu": { + "type": "power", + "test": str(test_name) + "-frequency-gpu", + "unit": "MHz", + "values": frequency_info["gpu"], + }, + } + + self._summarized_results = summarized_results + return self._summarized_results diff --git a/testing/mozbase/mozpower/mozpower/macintelpower.py b/testing/mozbase/mozpower/mozpower/macintelpower.py new file mode 100644 index 0000000000..191d9b9326 --- /dev/null +++ b/testing/mozbase/mozpower/mozpower/macintelpower.py @@ -0,0 +1,94 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import, unicode_literals + +import time + +from .powerbase import PowerBase +from .intel_power_gadget import IntelPowerGadget, IPGResultsHandler + + +class MacIntelPower(PowerBase): + """MacIntelPower is the OS and CPU dependent class for + power measurement gathering on Mac Intel-based hardware. + + :: + + from mozpower.macintelpower import MacIntelPower + + # duration and output_file_path are used in IntelPowerGadget + mip = MacIntelPower(ipg_measure_duration=600, output_file_path='power-testing') + + mip.initialize_power_measurements() + # Run test... + mip.finalize_power_measurements(test_name='raptor-test-name') + + perfherder_data = mip.get_perfherder_data() + """ + + def __init__(self, logger_name="mozpower", **kwargs): + """Initializes the MacIntelPower object. + + :param str logger_name: logging logger name. Defaults to 'mozpower'. + :param dict kwargs: optional keyword arguments passed to IntelPowerGadget. + """ + PowerBase.__init__(self, logger_name=logger_name, os="darwin", cpu="intel") + self.ipg = IntelPowerGadget(self.ipg_path, **kwargs) + self.ipg_results_handler = None + self.start_time = None + self.end_time = None + self.perfherder_data = {} + + def initialize_power_measurements(self): + """Starts power measurement gathering through IntelPowerGadget.""" + self._logger.info("Initializing power measurements...") + + # Start measuring + self.ipg.start_ipg() + + # Record start time to get an approximation of run time + self.start_time = time.time() + + def finalize_power_measurements( + self, test_name="power-testing", output_dir_path="", **kwargs + ): + """Stops power measurement gathering through IntelPowerGadget, cleans the data, + and produces partial perfherder formatted data that is stored in perfherder_data. + + :param str test_name: name of the test that was run. + :param str output_dir_path: directory to store output files. + :param dict kwargs: contains optional arguments to stop_ipg. + """ + self._logger.info("Finalizing power measurements...") + self.end_time = time.time() + + # Wait until Intel Power Gadget is done, then clean the data + # and then format it + self.ipg.stop_ipg(**kwargs) + + # Handle the results and format them to a partial perfherder format + if not output_dir_path: + output_dir_path = self.ipg.output_dir_path + + self.ipg_results_handler = IPGResultsHandler( + self.ipg.output_files, + output_dir_path, + ipg_measure_duration=self.ipg.ipg_measure_duration, + sampling_rate=self.ipg.sampling_rate, + logger_name=self.logger_name, + ) + + self.ipg_results_handler.clean_ipg_data() + self.perfherder_data = ( + self.ipg_results_handler.format_ipg_data_to_partial_perfherder( + self.end_time - self.start_time, test_name + ) + ) + + def get_perfherder_data(self): + """Returns the perfherder data output that was produced. + + :returns: dict + """ + return self.perfherder_data diff --git a/testing/mozbase/mozpower/mozpower/mozpower.py b/testing/mozbase/mozpower/mozpower/mozpower.py new file mode 100644 index 0000000000..97847e220a --- /dev/null +++ b/testing/mozbase/mozpower/mozpower/mozpower.py @@ -0,0 +1,376 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import, unicode_literals + +import platform +import os +import subprocess +import re + +from .macintelpower import MacIntelPower +from .mozpowerutils import average_summary, frequency_summary, get_logger, sum_summary + +OSCPU_COMBOS = { + "darwin-intel": MacIntelPower, +} + +SUMMARY_METHODS = { + "utilization": average_summary, + "power-usage": sum_summary, + "power-watts": frequency_summary, + "frequency-cpu": frequency_summary, + "frequency-gpu": frequency_summary, + "default": sum_summary, +} + + +class OsCpuComboMissingError(Exception): + """OsCpuComboMissingError is raised when we cannot find + a class responsible for the OS, and CPU combination that + was detected. + """ + + pass + + +class MissingProcessorInfoError(Exception): + """MissingProcessorInfoError is raised when we cannot find + the processor information on the machine. This is raised when + the file is missing (mentioned in the error message) or if + an exception occurs when we try to gather the information + from the file. + """ + + pass + + +class MozPower(object): + """MozPower provides an OS and CPU independent interface + for initializing, finalizing, and gathering power measurement + data from OS+CPU combo-dependent measurement classes. The combo + is detected automatically, and the correct class is instantiated + based on the OSCPU_COMBOS list. If it cannot be found, an + OsCpuComboMissingError will be raised. + + If a newly added power measurer does not have the required functions + `initialize_power_measurements`, `finalize_power_measurements`, + or `get_perfherder_data`, then a NotImplementedError will be + raised. + + Android power measurements are currently not supported by this + module. + + :: + + from mozpower import MozPower + + mp = MozPower(output_file_path='dir/power-testing') + + mp.initialize_power_measurements() + # Run test... + mp.finalize_power_measurements(test_name='raptor-test-name') + + perfherder_data = mp.get_perfherder_data() + """ + + def __init__( + self, + android=False, + logger_name="mozpower", + output_file_path="power-testing", + **kwargs + ): + """Initializes the MozPower object, detects OS and CPU (if not android), + and instatiates the appropriate combo-dependent class for measurements. + + :param bool android: run android power measurer. + :param str logger_name: logging logger name. Defaults to 'mozpower'. + :param str output_file_path: path to where output files will be stored. + Can include a prefix for the output files, i.e. 'dir/raptor-test' + would output data to 'dir' with the prefix 'raptor-test'. + Defaults to 'power-testing', the current directory using + the prefix 'power-testing'. + :param dict kwargs: optional keyword arguments passed to power measurer. + :raises: * OsCpuComboMissingError + * NotImplementedError + """ + self.measurer = None + self._os = None + self._cpu = None + self._logger = get_logger(logger_name) + + if android: + self._logger.error("Android power usage measurer has not been implemented") + raise NotImplementedError + else: + self._os = self._get_os().lower() + cpu = self._get_processor_info().lower() + + if "intel" in cpu: + self._cpu = "intel" + else: + self._cpu = "arm64" + + # OS+CPU combos are specified through strings such as 'darwin-intel' + # for mac power measurement on intel-based machines. If none exist in + # OSCPU_COMBOS, OsCpuComboMissingError will be raised. + measurer = None + oscpu_combo = "%s-%s" % (self._os, self._cpu) + if oscpu_combo in OSCPU_COMBOS: + measurer = OSCPU_COMBOS[oscpu_combo] + else: + raise OsCpuComboMissingError( + "Cannot find OS+CPU combo for %s" % oscpu_combo + ) + + if measurer: + self._logger.info( + "Intializing measurer %s for %s power measurements, see below for errors..." + % (measurer.__name__, oscpu_combo) + ) + self.measurer = measurer( + logger_name=logger_name, output_file_path=output_file_path, **kwargs + ) + + def _get_os(self): + """Returns the operating system of the machine being tested. platform.system() + returns 'darwin' on MacOS, 'windows' on Windows, and 'linux' on Linux systems. + + :returns: str + """ + return platform.system() + + def _get_processor_info(self): + """Returns the processor model type of the machine being tested. + Each OS has it's own way of storing this information. Raises + MissingProcessorInfoError if we cannot get the processor info + from the expected locations. + + :returns: str + :raises: * MissingProcessorInfoError + """ + model = "" + + if self._get_os() == "Windows": + model = platform.processor() + + elif self._get_os() == "Darwin": + proc_info_path = "/usr/sbin/sysctl" + command = [proc_info_path, "-n", "machdep.cpu.brand_string"] + + if not os.path.exists(proc_info_path): + raise MissingProcessorInfoError( + "Missing processor info file for darwin platform, " + "expecting it here %s" % proc_info_path + ) + + try: + model = subprocess.check_output(command).strip() + except subprocess.CalledProcessError as e: + error_log = str(e) + if e.output: + error_log = e.output.decode() + raise MissingProcessorInfoError( + "Error while attempting to get darwin processor information " + "from %s (exists) with the command %s: %s" + % (proc_info_path, str(command), error_log) + ) + + elif self._get_os() == "Linux": + proc_info_path = "/proc/cpuinfo" + model_re = re.compile(r""".*model name\s+[:]\s+(.*)\s+""") + + if not os.path.exists(proc_info_path): + raise MissingProcessorInfoError( + "Missing processor info file for linux platform, " + "expecting it here %s" % proc_info_path + ) + + try: + with open(proc_info_path) as cpuinfo: + for line in cpuinfo: + if not line.strip(): + continue + match = model_re.match(line) + if match: + model = match.group(1) + if not model: + raise Exception( + "No 'model name' entries found in the processor info file" + ) + except Exception as e: + raise MissingProcessorInfoError( + "Error while attempting to get linux processor information " + "from %s (exists): %s" % (proc_info_path, str(e)) + ) + + return model + + def initialize_power_measurements(self, **kwargs): + """Starts the power measurements by calling the power measurer's + `initialize_power_measurements` function. + + :param dict kwargs: keyword arguments for power measurer initialization + function if they are needed. + """ + if self.measurer is None: + return + self.measurer.initialize_power_measurements(**kwargs) + + def finalize_power_measurements(self, **kwargs): + """Stops the power measurements by calling the power measurer's + `finalize_power_measurements` function. + + :param dict kwargs: keyword arguments for power measurer finalization + function if they are needed. + """ + if self.measurer is None: + return + self.measurer.finalize_power_measurements(**kwargs) + + def get_perfherder_data(self): + """Returns the partial perfherder data output produced by the measurer. + For a complete perfherder data blob, see get_full_perfherder_data. + + :returns: dict + """ + if self.measurer is None: + return + return self.measurer.get_perfherder_data() + + def _summarize_values(self, datatype, values): + """Summarizes the given values based on the type of the + data. See SUMMARY_METHODS for the methods used for each + known data type. Defaults to using the sum of the values + when a data type cannot be found. + + Data type entries in SUMMARY_METHODS are case-sensitive. + + :param str datastype: the measurement type being summarized. + :param list values: the values to summarize. + :returns: float + """ + if datatype not in SUMMARY_METHODS: + self._logger.warning( + "Missing summary method for data type %s, defaulting to sum" % datatype + ) + datatype = "default" + + summary_func = SUMMARY_METHODS[datatype] + return summary_func(values) + + def get_full_perfherder_data( + self, framework, lowerisbetter=True, alertthreshold=2.0 + ): + """Returns a list of complete perfherder data blobs compiled from the + partial perfherder data blob returned from the measurer. Each key entry + (measurement type) in the partial perfherder data is parsed into its + own suite within a single perfherder data blob. + + For example, a partial perfherder data blob such as: + + :: + + { + 'utilization': {<perfherder_data>}, + 'power-usage': {<perfherder_data>} + } + + would produce two suites within a single perfherder data blobs - + one for utilization, and one for power-usage. + + Note that the 'values' entry must exist, otherwise the measurement + type is skipped. Furthermore, if 'name', 'unit', or 'type' is missing + we default to: + + :: + + { + 'name': 'mozpower', + 'unit': 'mWh', + 'type': 'power' + } + + Subtests produced for each sub-suite (measurement type), have the naming + pattern: <measurement_type>-<measured_name> + + Utilization of cpu would have the following name: 'utilization-cpu' + Power-usage for cpu has the following name: 'power-usage-cpu' + + :param str framework: name of the framework being tested, i.e. 'raptor'. + :param bool lowerisbetter: if set to true, low values are better than high ones. + :param float alertthreshold: determines the crossing threshold at + which an alert is generated. + :returns: dict + """ + if self.measurer is None: + return + + # Get the partial data, and the measurers name for + # logging purposes. + partial_perfherder_data = self.get_perfherder_data() + measurer_name = self.measurer.__class__.__name__ + + suites = [] + perfherder_data = {"framework": {"name": framework}, "suites": suites} + + for measurement_type in partial_perfherder_data: + self._logger.info("Summarizing %s data" % measurement_type) + dataset = partial_perfherder_data[measurement_type] + + # Skip this measurement type if the 'values' entry + # doesn't exist, and output a warning. + if "values" not in dataset: + self._logger.warning( + "Missing 'values' entry in partial perfherder data for measurement type %s " + "obtained from %s. This measurement type will not be processed." + % (measurement_type, measurer_name) + ) + continue + + # Get the settings, if they exist, otherwise output + # a warning and use a default entry. + settings = {"test": "mozpower", "unit": "mWh", "type": "power"} + + for setting in settings: + if setting in dataset: + settings[setting] = dataset[setting] + else: + self._logger.warning( + "Missing '%s' entry in partial perfherder data for measurement type %s " + "obtained from %s, using %s as the default" + % (setting, measurement_type, measurer_name, settings[setting]) + ) + + subtests = [] + suite = { + "name": "%s-%s" % (settings["test"], measurement_type), + "type": settings["type"], + "value": 0, + "subtests": subtests, + "lowerIsBetter": lowerisbetter, + "unit": settings["unit"], + "alertThreshold": alertthreshold, + } + + # Parse the 'values' entries into subtests + values = [] + for measure in dataset["values"]: + value = dataset["values"][measure] + subtest = { + "name": "%s-%s" % (measurement_type, measure), + "value": float(value), + "lowerIsBetter": lowerisbetter, + "alertThreshold": alertthreshold, + "unit": settings["unit"], + } + values.append((value, measure)) + subtests.append(subtest) + + # Summarize the data based on the measurement type + if len(values) > 0: + suite["value"] = self._summarize_values(measurement_type, values) + suites.append(suite) + + return perfherder_data diff --git a/testing/mozbase/mozpower/mozpower/mozpowerutils.py b/testing/mozbase/mozpower/mozpower/mozpowerutils.py new file mode 100644 index 0000000000..8a781c9b88 --- /dev/null +++ b/testing/mozbase/mozpower/mozpower/mozpowerutils.py @@ -0,0 +1,59 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import, division, unicode_literals + + +def get_logger(logger_name): + """Returns the logger that should be used based on logger_name. + Defaults to the logging logger if mozlog cannot be imported. + + :returns: mozlog or logging logger object + """ + logger = None + try: + import mozlog + + logger = mozlog.get_default_logger(logger_name) + except ImportError: + pass + + if logger is None: + import logging + + logging.basicConfig() + logger = logging.getLogger(logger_name) + return logger + + +def average_summary(values): + """Averages all given values. + + :param list values: list of values to average. + :returns: float + """ + # pylint --py3k W1619 + return sum([float(v[0]) for v in values]) / len(values) + + +def sum_summary(values): + """Adds all values together. + + :param list values: list of values to sum. + :returns: float + """ + return sum([float(v[0]) for v in values]) + + +def frequency_summary(values): + """Returns the average frequency as the summary value. + + :param list values: list of values to search in. + :returns: float + """ + avgfreq = 0 + for val, name in values: + if "avg" in name: + avgfreq = float(val) + break + return avgfreq diff --git a/testing/mozbase/mozpower/mozpower/powerbase.py b/testing/mozbase/mozpower/mozpower/powerbase.py new file mode 100644 index 0000000000..ecf9a218e8 --- /dev/null +++ b/testing/mozbase/mozpower/mozpower/powerbase.py @@ -0,0 +1,124 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import, unicode_literals + +import os + +from .mozpowerutils import get_logger + + +class IPGExecutableMissingError(Exception): + """IPGExecutableMissingError is raised when we cannot find + the executable for Intel Power Gadget at the expected location. + """ + + pass + + +class PlatformUnsupportedError(Exception): + """PlatformUnsupportedError is raised when we cannot find + an expected IPG path for the OS being tested. + """ + + pass + + +class PowerBase(object): + """PowerBase provides an interface for power measurement objects + that depend on the os and cpu. When using this class as a base class + the `initialize_power_measurements`, `finalize_power_measurements`, + and `get_perfherder_data` functions must be implemented, otherwise + a NotImplementedError will be raised. + + PowerBase should only be used as the base class for other + classes and should not be instantiated directly. To enforce this + restriction calling PowerBase's constructor will raise a + NonImplementedError exception. + + :: + + from mozpower.powerbase import PowerBase + + try: + pb = PowerBase() + except NotImplementedError: + print "PowerBase cannot be instantiated." + + """ + + def __init__(self, logger_name="mozpower", os=None, cpu=None): + """Initializes the PowerBase object. + + :param str logger_name: logging logger name. Defaults to 'mozpower'. + :param str os: operating system being tested. Defaults to None. + :param str cpu: cpu type being tested (either intel or arm64). Defaults to None. + :raises: * NotImplementedError + """ + if self.__class__ == PowerBase: + raise NotImplementedError + + self._logger_name = logger_name + self._logger = get_logger(logger_name) + self._os = os + self._cpu = cpu + self._ipg_path = self.get_ipg_path() + + @property + def ipg_path(self): + return self._ipg_path + + @property + def logger_name(self): + return self._logger_name + + def initialize_power_measurements(self): + """Starts power measurement gathering, must be implemented by subclass. + + :raises: * NotImplementedError + """ + raise NotImplementedError + + def finalize_power_measurements(self, test_name="power-testing", **kwargs): + """Stops power measurement gathering, must be implemented by subclass. + + :raises: * NotImplementedError + """ + raise NotImplementedError + + def get_perfherder_data(self): + """Returns the perfherder data output produced from the tests, must + be implemented by subclass. + + :raises: * NotImplementedError + """ + raise NotImplementedError + + def get_ipg_path(self): + """Returns the path to where we expect to find Intel Power Gadget + depending on the OS being tested. Raises a PlatformUnsupportedError + if the OS being tested is unsupported, and raises a IPGExecutableMissingError + if the Intel Power Gadget executable cannot be found at the expected + location. + + :returns: str + :raises: * IPGExecutableMissingError + * PlatformUnsupportedError + """ + if self._cpu != "intel": + return None + + if self._os == "darwin": + exe_path = "/Applications/Intel Power Gadget/PowerLog" + else: + raise PlatformUnsupportedError( + "%s platform currently not supported for Intel Power Gadget measurements" + % self._os + ) + if not os.path.exists(exe_path): + raise IPGExecutableMissingError( + "Intel Power Gadget is not installed, or cannot be found at the location %s" + % exe_path + ) + + return exe_path diff --git a/testing/mozbase/mozpower/setup.cfg b/testing/mozbase/mozpower/setup.cfg new file mode 100644 index 0000000000..2a9acf13da --- /dev/null +++ b/testing/mozbase/mozpower/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/testing/mozbase/mozpower/setup.py b/testing/mozbase/mozpower/setup.py new file mode 100644 index 0000000000..cf358ea262 --- /dev/null +++ b/testing/mozbase/mozpower/setup.py @@ -0,0 +1,36 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +from setuptools import setup + +PACKAGE_NAME = "mozpower" +PACKAGE_VERSION = "1.1.2" + +deps = ["mozlog >= 6.0", "mozdevice >= 4.0.0,<5"] + +setup( + name=PACKAGE_NAME, + version=PACKAGE_VERSION, + description="Mozilla-authored power usage measurement tools", + long_description="see https://firefox-source-docs.mozilla.org/mozbase/index.html", + classifiers=[ + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.5", + ], + # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers + keywords="", + author="Mozilla Performance Test Engineering Team", + author_email="tools@lists.mozilla.org", + url="https://wiki.mozilla.org/Auto-tools/Projects/Mozbase", + license="MPL", + packages=["mozpower"], + include_package_data=True, + zip_safe=False, + install_requires=deps, + entry_points=""" + # -*- Entry points: -*- + """, +) diff --git a/testing/mozbase/mozpower/tests/conftest.py b/testing/mozbase/mozpower/tests/conftest.py new file mode 100644 index 0000000000..5d8ecbbc9d --- /dev/null +++ b/testing/mozbase/mozpower/tests/conftest.py @@ -0,0 +1,105 @@ +from __future__ import absolute_import, print_function + +import mock +import os +import pytest +import tempfile +import time + +from mozpower import MozPower +from mozpower.macintelpower import MacIntelPower +from mozpower.powerbase import PowerBase +from mozpower.intel_power_gadget import IntelPowerGadget, IPGResultsHandler + + +def os_side_effect(*args, **kwargs): + """Used as a side effect to os.path.exists when + checking if the Intel Power Gadget executable exists. + """ + return True + + +def subprocess_side_effect(*args, **kwargs): + """Used as a side effect when running the Intel Power + Gadget tool. + """ + time.sleep(1) + + +@pytest.fixture(scope="function") +def powermeasurer(): + """Returns a testing subclass of the PowerBase class + for testing. + """ + + class PowerMeasurer(PowerBase): + pass + + return PowerMeasurer() + + +@pytest.fixture(scope="function") +def ipg_obj(): + """Returns an IntelPowerGadget object with the test + output file path. + """ + return IntelPowerGadget( + "ipg-path", + output_file_path=os.path.abspath(os.path.dirname(__file__)) + + "/files/raptor-tp6-amazon-firefox_powerlog", + ) + + +@pytest.fixture(scope="function") +def ipg_rh_obj(): + """Returns an IPGResultsHandler object set up with the + test files and cleans up the directory after the tests + are complete. + """ + base_path = os.path.abspath(os.path.dirname(__file__)) + "/files/" + tmpdir = tempfile.mkdtemp() + + # Return the results handler for the test + yield IPGResultsHandler( + [ + base_path + "raptor-tp6-amazon-firefox_powerlog_1_.txt", + base_path + "raptor-tp6-amazon-firefox_powerlog_2_.txt", + base_path + "raptor-tp6-amazon-firefox_powerlog_3_.txt", + ], + tmpdir, + ) + + +@pytest.fixture(scope="function") +def macintelpower_obj(): + """Returns a MacIntelPower object with subprocess.check_output + and os.path.exists calls patched with side effects. + """ + with mock.patch("subprocess.check_output") as subprocess_mock: + with mock.patch("os.path.exists") as os_mock: + subprocess_mock.side_effect = subprocess_side_effect + os_mock.side_effect = os_side_effect + + yield MacIntelPower(ipg_measure_duration=2) + + +@pytest.fixture(scope="function") +def mozpower_obj(): + """Returns a MozPower object with subprocess.check_output + and os.path.exists calls patched with side effects. + """ + with mock.patch.object( + MozPower, "_get_os", return_value="Darwin" + ) as _, mock.patch.object( + MozPower, "_get_processor_info", return_value="GenuineIntel" + ) as _, mock.patch.object( + MacIntelPower, "get_ipg_path", return_value="/" + ) as _, mock.patch( + "subprocess.check_output" + ) as subprocess_mock, mock.patch( + "os.path.exists" + ) as os_mock: + subprocess_mock.side_effect = subprocess_side_effect + os_mock.side_effect = os_side_effect + + yield MozPower(ipg_measure_duration=2) diff --git a/testing/mozbase/mozpower/tests/files/emptyfile.txt b/testing/mozbase/mozpower/tests/files/emptyfile.txt new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/testing/mozbase/mozpower/tests/files/emptyfile.txt diff --git a/testing/mozbase/mozpower/tests/files/raptor-tp6-amazon-firefox_powerlog_1_.txt b/testing/mozbase/mozpower/tests/files/raptor-tp6-amazon-firefox_powerlog_1_.txt new file mode 100644 index 0000000000..d5cc3f91a5 --- /dev/null +++ b/testing/mozbase/mozpower/tests/files/raptor-tp6-amazon-firefox_powerlog_1_.txt @@ -0,0 +1,30 @@ +"System Time","RDTSC","Elapsed Time (sec)","CPU Utilization(%)","CPU Frequency_0(MHz)","Processor Power_0(Watt)","Cumulative Processor Energy_0(Joules)","Cumulative Processor Energy_0(mWh)","IA Power_0(Watt)","Cumulative IA Energy_0(Joules)","Cumulative IA Energy_0(mWh)","Package Temperature_0(C)","Package Hot_0","DRAM Power_0(Watt)","Cumulative DRAM Energy_0(Joules)","Cumulative DRAM Energy_0(mWh)","GT Power_0(Watt)","Cumulative GT Energy_0(Joules)","Cumulative GT Energy_0(mWh)","Package Power Limit_0(Watt)","GT Frequency(MHz)" +"12:11:05:769","61075218263548"," 2.002"," 14.000"," 3400"," 23.427"," 23.460"," 6.517"," 21.019"," 21.048"," 5.847"," 75","0"," 0.877"," 0.878"," 0.244"," 0.020"," 0.020"," 0.006"," 45.000"," 0" +"12:11:06:774","61077822279584"," 3.007"," 20.000"," 3000"," 25.014"," 48.590"," 13.497"," 22.386"," 43.538"," 12.094"," 74","0"," 1.194"," 2.078"," 0.577"," 0.029"," 0.049"," 0.014"," 45.000"," 0" +"12:11:07:778","61080424421708"," 4.011"," 8.000"," 1300"," 9.512"," 58.140"," 16.150"," 6.904"," 50.469"," 14.019"," 65","0"," 0.836"," 2.917"," 0.810"," 0.007"," 0.057"," 0.016"," 45.000"," 0" +"12:11:08:781","61083023535972"," 5.013"," 1.000"," 1300"," 1.786"," 59.931"," 16.647"," 0.687"," 51.158"," 14.210"," 63","0"," 0.585"," 3.504"," 0.973"," 0.000"," 0.057"," 0.016"," 45.000"," 0" +"12:11:09:784","61085623402302"," 6.016"," 4.000"," 4000"," 5.249"," 65.195"," 18.110"," 3.743"," 54.912"," 15.253"," 75","0"," 0.660"," 4.166"," 1.157"," 0.003"," 0.059"," 0.017"," 45.000"," 0" +"12:11:10:787","61088224087008"," 7.020"," 20.000"," 3000"," 35.118"," 100.432"," 27.898"," 31.647"," 86.666"," 24.074"," 72","0"," 1.048"," 5.218"," 1.449"," 0.016"," 0.076"," 0.021"," 45.000"," 0" +"12:11:11:791","61090825821126"," 8.024"," 13.000"," 3000"," 25.436"," 125.965"," 34.990"," 22.228"," 108.979"," 30.272"," 71","0"," 0.868"," 6.089"," 1.691"," 0.004"," 0.080"," 0.022"," 45.000"," 0" +"12:11:12:795","61093429020836"," 9.028"," 5.000"," 1300"," 5.266"," 131.253"," 36.459"," 3.270"," 112.263"," 31.184"," 64","0"," 0.711"," 6.802"," 1.890"," 0.000"," 0.080"," 0.022"," 45.000"," 0" +"12:11:13:796","61096024160928"," 10.029"," 5.000"," 4000"," 11.070"," 142.338"," 39.538"," 9.613"," 121.888"," 33.858"," 78","0"," 0.650"," 7.453"," 2.070"," 0.000"," 0.080"," 0.022"," 45.000"," 0" +"12:11:13:796","61096024279264"," 10.029"," 57.000"," 4000"," 0.000"," 142.338"," 39.538"," 0.000"," 121.888"," 33.858"," 78","0"," 0.000"," 7.453"," 2.070"," 0.000"," 0.080"," 0.022"," 45.000"," 0" +
+"Total Elapsed Time (sec) = 10.029232" +"Measured RDTSC Frequency (GHz) = 2.592" +
+"Cumulative Processor Energy_0 (Joules) = 142.337524" +"Cumulative Processor Energy_0 (mWh) = 39.538201" +"Average Processor Power_0 (Watt) = 14.192265" +
+"Cumulative IA Energy_0 (Joules) = 121.888000" +"Cumulative IA Energy_0 (mWh) = 33.857778" +"Average IA Power_0 (Watt) = 12.153273" +
+"Cumulative DRAM Energy_0 (Joules) = 7.453308" +"Cumulative DRAM Energy_0 (mWh) = 2.070363" +"Average DRAM Power_0 (Watt) = 0.743158" +
+"Cumulative GT Energy_0 (Joules) = 0.079834" +"Cumulative GT Energy_0 (mWh) = 0.022176" +"Average GT Power_0 (Watt) = 0.007960" diff --git a/testing/mozbase/mozpower/tests/files/raptor-tp6-amazon-firefox_powerlog_2_.txt b/testing/mozbase/mozpower/tests/files/raptor-tp6-amazon-firefox_powerlog_2_.txt new file mode 100644 index 0000000000..9157ad1fad --- /dev/null +++ b/testing/mozbase/mozpower/tests/files/raptor-tp6-amazon-firefox_powerlog_2_.txt @@ -0,0 +1,30 @@ +"System Time","RDTSC","Elapsed Time (sec)","CPU Utilization(%)","CPU Frequency_0(MHz)","Processor Power_0(Watt)","Cumulative Processor Energy_0(Joules)","Cumulative Processor Energy_0(mWh)","IA Power_0(Watt)","Cumulative IA Energy_0(Joules)","Cumulative IA Energy_0(mWh)","Package Temperature_0(C)","Package Hot_0","DRAM Power_0(Watt)","Cumulative DRAM Energy_0(Joules)","Cumulative DRAM Energy_0(mWh)","GT Power_0(Watt)","Cumulative GT Energy_0(Joules)","Cumulative GT Energy_0(mWh)","Package Power Limit_0(Watt)","GT Frequency(MHz)" +"12:11:15:821","61101270635674"," 2.007"," 9.000"," 4200"," 10.341"," 10.382"," 2.884"," 7.710"," 7.740"," 2.150"," 80","0"," 0.828"," 0.831"," 0.231"," 0.000"," 0.000"," 0.000"," 45.000"," 0" +"12:11:16:823","61103869919264"," 3.010"," 4.000"," 1300"," 9.353"," 19.762"," 5.489"," 8.079"," 15.842"," 4.400"," 64","0"," 0.597"," 1.430"," 0.397"," 0.000"," 0.000"," 0.000"," 45.000"," 0" +"12:11:17:826","61106469262970"," 4.013"," 19.000"," 2500"," 21.127"," 40.950"," 11.375"," 17.753"," 33.645"," 9.346"," 69","0"," 1.091"," 2.524"," 0.701"," 0.011"," 0.011"," 0.003"," 45.000"," 0" +"12:11:18:827","61109063770516"," 5.014"," 11.000"," 2000"," 11.620"," 52.581"," 14.606"," 8.333"," 41.986"," 11.663"," 64","0"," 0.961"," 3.486"," 0.968"," 0.000"," 0.011"," 0.003"," 45.000"," 0" +"12:11:19:831","61111666134444"," 6.018"," 3.000"," 1300"," 2.722"," 55.314"," 15.365"," 1.497"," 43.489"," 12.080"," 64","0"," 0.620"," 4.108"," 1.141"," 0.000"," 0.011"," 0.003"," 45.000"," 0" +"12:11:20:835","61114266619236"," 7.021"," 14.000"," 3400"," 17.108"," 72.478"," 20.133"," 14.538"," 58.075"," 16.132"," 79","0"," 0.918"," 5.029"," 1.397"," 0.004"," 0.016"," 0.004"," 45.000"," 0" +"12:11:21:835","61116859007218"," 8.021"," 15.000"," 3000"," 16.651"," 89.132"," 24.759"," 13.055"," 71.132"," 19.759"," 70","0"," 1.103"," 6.132"," 1.703"," 0.005"," 0.020"," 0.006"," 45.000"," 0" +"12:11:22:836","61119453982614"," 9.023"," 4.000"," 2000"," 3.334"," 92.470"," 25.686"," 1.974"," 73.109"," 20.308"," 64","0"," 0.654"," 6.787"," 1.885"," 0.000"," 0.020"," 0.006"," 45.000"," 0" +"12:11:23:840","61122057636318"," 10.027"," 12.000"," 3400"," 13.925"," 106.458"," 29.572"," 11.603"," 84.764"," 23.546"," 79","0"," 0.856"," 7.647"," 2.124"," 0.006"," 0.027"," 0.007"," 45.000"," 0" +"12:11:23:840","61122057733984"," 10.027"," 66.000"," 3400"," 0.000"," 106.458"," 29.572"," 0.000"," 84.764"," 23.546"," 79","0"," 0.000"," 7.647"," 2.124"," 0.000"," 0.027"," 0.007"," 45.000"," 0" +
+"Total Elapsed Time (sec) = 10.027134" +"Measured RDTSC Frequency (GHz) = 2.592" +
+"Cumulative Processor Energy_0 (Joules) = 106.457581" +"Cumulative Processor Energy_0 (mWh) = 29.571550" +"Average Processor Power_0 (Watt) = 10.616950" +
+"Cumulative IA Energy_0 (Joules) = 84.764404" +"Cumulative IA Energy_0 (mWh) = 23.545668" +"Average IA Power_0 (Watt) = 8.453503" +
+"Cumulative DRAM Energy_0 (Joules) = 7.647095" +"Cumulative DRAM Energy_0 (mWh) = 2.124193" +"Average DRAM Power_0 (Watt) = 0.762640" +
+"Cumulative GT Energy_0 (Joules) = 0.026550" +"Cumulative GT Energy_0 (mWh) = 0.007375" +"Average GT Power_0 (Watt) = 0.002648" diff --git a/testing/mozbase/mozpower/tests/files/raptor-tp6-amazon-firefox_powerlog_3_.txt b/testing/mozbase/mozpower/tests/files/raptor-tp6-amazon-firefox_powerlog_3_.txt new file mode 100644 index 0000000000..a2779d93a1 --- /dev/null +++ b/testing/mozbase/mozpower/tests/files/raptor-tp6-amazon-firefox_powerlog_3_.txt @@ -0,0 +1,30 @@ +"System Time","RDTSC","Elapsed Time (sec)","CPU Utilization(%)","CPU Frequency_0(MHz)","Processor Power_0(Watt)","Cumulative Processor Energy_0(Joules)","Cumulative Processor Energy_0(mWh)","IA Power_0(Watt)","Cumulative IA Energy_0(Joules)","Cumulative IA Energy_0(mWh)","Package Temperature_0(C)","Package Hot_0","DRAM Power_0(Watt)","Cumulative DRAM Energy_0(Joules)","Cumulative DRAM Energy_0(mWh)","GT Power_0(Watt)","Cumulative GT Energy_0(Joules)","Cumulative GT Energy_0(mWh)","Package Power Limit_0(Watt)","GT Frequency(MHz)" +"12:11:25:867","61127310813944"," 2.006"," 6.000"," 2000"," 4.862"," 4.879"," 1.355"," 3.087"," 3.097"," 0.860"," 65","0"," 0.774"," 0.776"," 0.216"," 0.021"," 0.022"," 0.006"," 45.000"," 0" +"12:11:26:867","61129903677124"," 3.006"," 2.000"," 1300"," 2.883"," 7.763"," 2.156"," 1.253"," 4.350"," 1.208"," 63","0"," 0.644"," 1.421"," 0.395"," 0.018"," 0.039"," 0.011"," 45.000"," 0" +"12:11:27:873","61132509307964"," 4.011"," 2.000"," 1300"," 1.788"," 9.561"," 2.656"," 0.699"," 5.054"," 1.404"," 63","0"," 0.538"," 1.961"," 0.545"," 0.000"," 0.039"," 0.011"," 45.000"," 0" +"12:11:28:877","61135111021262"," 5.015"," 0.000"," 1300"," 1.177"," 10.743"," 2.984"," 0.321"," 5.375"," 1.493"," 63","0"," 0.535"," 2.499"," 0.694"," 0.000"," 0.039"," 0.011"," 45.000"," 0" +"12:11:29:881","61137714179976"," 6.020"," 0.000"," 1300"," 1.064"," 11.811"," 3.281"," 0.251"," 5.627"," 1.563"," 61","0"," 0.520"," 3.021"," 0.839"," 0.000"," 0.039"," 0.011"," 45.000"," 0" +"12:11:30:881","61140307250904"," 7.020"," 0.000"," 1300"," 1.053"," 12.864"," 3.573"," 0.256"," 5.884"," 1.634"," 61","0"," 0.528"," 3.550"," 0.986"," 0.000"," 0.039"," 0.011"," 45.000"," 0" +"12:11:31:885","61142907976768"," 8.023"," 0.000"," 1300"," 1.069"," 13.937"," 3.871"," 0.274"," 6.159"," 1.711"," 61","0"," 0.524"," 4.075"," 1.132"," 0.000"," 0.039"," 0.011"," 45.000"," 0" +"12:11:32:886","61145503940512"," 9.025"," 0.000"," 1300"," 1.008"," 14.947"," 4.152"," 0.228"," 6.387"," 1.774"," 60","0"," 0.522"," 4.598"," 1.277"," 0.000"," 0.039"," 0.011"," 45.000"," 0" +"12:11:33:891","61148109378824"," 10.030"," 0.000"," 1300"," 1.007"," 15.959"," 4.433"," 0.230"," 6.618"," 1.838"," 61","0"," 0.522"," 5.123"," 1.423"," 0.000"," 0.039"," 0.011"," 45.000"," 0" +"12:11:33:892","61148109766160"," 10.030"," 27.000"," 1300"," 0.000"," 15.959"," 4.433"," 0.000"," 6.618"," 1.838"," 61","0"," 2.448"," 5.123"," 1.423"," 0.000"," 0.039"," 0.011"," 45.000"," 0" +
+"Total Elapsed Time (sec) = 10.030310" +"Measured RDTSC Frequency (GHz) = 2.592" +
+"Cumulative Processor Energy_0 (Joules) = 15.958862" +"Cumulative Processor Energy_0 (mWh) = 4.433017" +"Average Processor Power_0 (Watt) = 1.591064" +
+"Cumulative IA Energy_0 (Joules) = 6.618042" +"Cumulative IA Energy_0 (mWh) = 1.838345" +"Average IA Power_0 (Watt) = 0.659804" +
+"Cumulative DRAM Energy_0 (Joules) = 5.122925" +"Cumulative DRAM Energy_0 (mWh) = 1.423035" +"Average DRAM Power_0 (Watt) = 0.510744" +
+"Cumulative GT Energy_0 (Joules) = 0.039490" +"Cumulative GT Energy_0 (mWh) = 0.010969" +"Average GT Power_0 (Watt) = 0.003937" diff --git a/testing/mozbase/mozpower/tests/files/valueerrorfile.txt b/testing/mozbase/mozpower/tests/files/valueerrorfile.txt new file mode 100644 index 0000000000..8f2d3485eb --- /dev/null +++ b/testing/mozbase/mozpower/tests/files/valueerrorfile.txt @@ -0,0 +1,30 @@ +"System Time","RDTSC","Elapsed Time (sec)","CPU Utilization(%)","CPU Frequency_0(MHz)","Processor Power_0(Watt)","Cumulative Processor Energy_0(Joules)","Cumulative Processor Energy_0(mWh)","IA Power_0(Watt)","Cumulative IA Energy_0(Joules)","Cumulative IA Energy_0(mWh)","Package Temperature_0(C)","Package Hot_0","DRAM Power_0(Watt)","Cumulative DRAM Energy_0(Joules)","Cumulative DRAM Energy_0(mWh)","GT Power_0(Watt)","Cumulative GT Energy_0(Joules)","Cumulative GT Energy_0(mWh)","Package Power Limit_0(Watt)","GT Frequency(MHz)" +"12:11:05:769","61075218263548"," 2.002"," 14.000"," 3400"," 23.427"," 23.460"," 6.517"," 21.019"," 21.048"," 5.847"," 75","0"," 0.877"," 0.878"," 0.244"," 0.020"," 0.020"," 0.006"," 45.000"," 0" +"12:11:06:774","61077822279584"," 3.007"," 20.000"," 3000"," 25.014"," 48.590"," 13.497"," 22.386"," 43.538"," 12.094"," 74","0"," 1.194"," 2.078"," 0.577"," 0.029"," 0.049"," 0.014"," 45.000"," 0" +"12:11:07:778","61080424421708"," 4.011"," 8.000"," 1300"," 9.512"," 58.140"," 16.150"," 6.904"," 50.469"," 14.019"," 65","0"," 0.836"," 2.917"," 0.810"," 0.007"," 0.057"," 0.016"," 45.000"," 0" +"12:11:08:781","61083023535972"," 5.013"," 1.000"," 1300"," 1.786"," 59.931"," 16.647"," 0.687"," 51.158"," 14.210"," 63","0"," 0.585"," 3.504"," 0.973"," 0.000"," 0.057"," 0.016"," 45.000"," 0" +"12:11:09:784","61085623402302"," none"," 4.000"," 4000"," 5.249"," 65.195"," 18.110"," 3.743"," 54.912"," 15.253"," 75","0"," 0.660"," 4.166"," 1.157"," 0.003"," 0.059"," 0.017"," 45.000"," 0" +"12:11:10:787","61088224087008"," 7.020"," 20.000"," 3000"," 35.118"," 100.432"," 27.898"," 31.647"," 86.666"," 24.074"," 72","0"," 1.048"," 5.218"," 1.449"," 0.016"," 0.076"," 0.021"," 45.000"," 0" +"12:11:11:791","61090825821126"," 8.024"," 13.000"," 3000"," 25.436"," 125.965"," 34.990"," 22.228"," 108.979"," 30.272"," 71","0"," 0.868"," 6.089"," 1.691"," 0.004"," 0.080"," 0.022"," 45.000"," 0" +"12:11:12:795","61093429020836"," 9.028"," 5.000"," 1300"," 5.266"," 131.253"," 36.459"," 3.270"," 112.263"," 31.184"," 64","0"," 0.711"," 6.802"," 1.890"," 0.000"," 0.080"," 0.022"," 45.000"," 0" +"12:11:13:796","61096024160928"," 10.029"," 5.000"," 4000"," 11.070"," 142.338"," 39.538"," 9.613"," 121.888"," 33.858"," 78","0"," 0.650"," 7.453"," 2.070"," 0.000"," 0.080"," 0.022"," 45.000"," 0" +"12:11:13:796","61096024279264"," 10.029"," 57.000"," 4000"," 0.000"," 142.338"," 39.538"," 0.000"," 121.888"," 33.858"," 78","0"," 0.000"," 7.453"," 2.070"," 0.000"," 0.080"," 0.022"," 45.000"," 0" + +"Total Elapsed Time (sec) = 10.029232" +"Measured RDTSC Frequency (GHz) = 2.592" + +"Cumulative Processor Energy_0 (Joules) = 142.337524" +"Cumulative Processor Energy_0 (mWh) = 39.538201" +"Average Processor Power_0 (Watt) = 14.192265" + +"Cumulative IA Energy_0 (Joules) = 121.888000" +"Cumulative IA Energy_0 (mWh) = 33.857778" +"Average IA Power_0 (Watt) = 12.153273" + +"Cumulative DRAM Energy_0 (Joules) = 7.453308" +"Cumulative DRAM Energy_0 (mWh) = 2.070363" +"Average DRAM Power_0 (Watt) = 0.743158" + +"Cumulative GT Energy_0 (Joules) = 0.079834" +"Cumulative GT Energy_0 (mWh) = 0.022176" +"Average GT Power_0 (Watt) = 0.007960" diff --git a/testing/mozbase/mozpower/tests/manifest.ini b/testing/mozbase/mozpower/tests/manifest.ini new file mode 100644 index 0000000000..2e825e2fa5 --- /dev/null +++ b/testing/mozbase/mozpower/tests/manifest.ini @@ -0,0 +1,6 @@ +[DEFAULT] +subsuite = mozbase +[test_powerbase.py] +[test_intelpowergadget.py] +[test_macintelpower.py] +[test_mozpower.py] diff --git a/testing/mozbase/mozpower/tests/test_intelpowergadget.py b/testing/mozbase/mozpower/tests/test_intelpowergadget.py new file mode 100644 index 0000000000..f508c83915 --- /dev/null +++ b/testing/mozbase/mozpower/tests/test_intelpowergadget.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python + +from __future__ import absolute_import + +import datetime +import mock +import mozunit +import os +import pytest +import time + +from mozpower.intel_power_gadget import ( + IPGEmptyFileError, + IPGMissingOutputFileError, + IPGTimeoutError, + IPGUnknownValueTypeError, +) + +import six + + +def thread_is_alive(thread): + if six.PY2: + return thread.isAlive() + return thread.is_alive() + + +def test_ipg_pathsplitting(ipg_obj): + """Tests that the output file path and prefix was properly split. + This test assumes that it is in the same directory as the conftest.py file. + """ + assert ( + ipg_obj.output_dir_path == os.path.abspath(os.path.dirname(__file__)) + "/files" + ) + assert ipg_obj.output_file_prefix == "raptor-tp6-amazon-firefox_powerlog" + + +def test_ipg_get_output_file_path(ipg_obj): + """Tests that the output file path is constantly changing + based on the file_counter value. + """ + test_path = "/test_path/" + test_ext = ".txt" + ipg_obj._file_counter = 1 + ipg_obj._output_dir_path = test_path + ipg_obj._output_file_ext = test_ext + + for i in range(1, 6): + fpath = ipg_obj._get_output_file_path() + + assert fpath.startswith(test_path) + assert fpath.endswith(test_ext) + assert str(i) in fpath + + +def test_ipg_start_and_stop(ipg_obj): + """Tests that the IPG thread can start and stop properly.""" + + def subprocess_side_effect(*args, **kwargs): + time.sleep(1) + + with mock.patch("subprocess.check_output") as m: + m.side_effect = subprocess_side_effect + + # Start recording IPG measurements + ipg_obj.start_ipg() + assert not ipg_obj._stop + + # Wait a bit for thread to start, then check it + timeout = 10 + start = time.time() + while time.time() - start < timeout and not ipg_obj._running: + time.sleep(1) + + assert ipg_obj._running + assert thread_is_alive(ipg_obj._thread) + + # Stop recording IPG measurements + ipg_obj.stop_ipg(wait_interval=1, timeout=30) + assert ipg_obj._stop + assert not ipg_obj._running + + +def test_ipg_stopping_timeout(ipg_obj): + """Tests that an IPGTimeoutError is raised when + the thread is still "running" and the wait in _wait_for_ipg + has exceeded the timeout value. + """ + with pytest.raises(IPGTimeoutError): + ipg_obj._running = True + ipg_obj._wait_for_ipg(wait_interval=1, timeout=2) + + +def test_ipg_rh_combine_cumulatives(ipg_rh_obj): + """Tests that cumulatives are correctly combined in + the _combine_cumulative_rows function. + """ + cumulatives_to_combine = [ + [0, 1, 2, 3, 4, 5], + [0, 1, 2, 3, 4, 5], + [0, 1, 2, 3, 4, 5], + [0, 1, 2, 3, 4, 5], + ] + + combined_cumulatives = ipg_rh_obj._combine_cumulative_rows(cumulatives_to_combine) + + # Check that accumulation worked, final value must be the maximum + assert combined_cumulatives[-1] == max(combined_cumulatives) + + # Check that the cumulative values are monotonically increasing + for count, val in enumerate(combined_cumulatives[:-1]): + assert combined_cumulatives[count + 1] - val >= 0 + + +def test_ipg_rh_clean_file(ipg_rh_obj): + """Tests that IPGResultsHandler correctly cleans the data + from one file. + """ + file = ipg_rh_obj._output_files[0] + linecount = 0 + with open(file, "r") as f: + for line in f: + linecount += 1 + + results, summary, clean_file = ipg_rh_obj._clean_ipg_file(file) + + # Check that each measure from the csv header + # is in the results dict and that the clean file output + # exists. + for measure in results: + assert measure in ipg_rh_obj._csv_header + assert os.path.exists(clean_file) + + clean_rows = [] + with open(clean_file, "r") as f: + for line in f: + if line.strip(): + clean_rows.append(line) + + # Make sure that the results and summary entries + # have the expected lengths. + for measure in results: + # Add 6 for new lines that were removed + assert len(results[measure]) + len(summary) + 6 == linecount + # Subtract 1 for the csv header + assert len(results[measure]) == len(clean_rows) - 1 + + +def test_ipg_rh_clean_ipg_data_no_files(ipg_rh_obj): + """Tests that IPGResultsHandler correctly handles the case + when no output files exist. + """ + ipg_rh_obj._output_files = [] + clean_data = ipg_rh_obj.clean_ipg_data() + assert clean_data is None + + +def test_ipg_rh_clean_ipg_data(ipg_rh_obj): + """Tests that IPGResultsHandler correctly handles cleaning + all known files and that the results and the merged output + are correct. + """ + clean_data = ipg_rh_obj.clean_ipg_data() + clean_files = ipg_rh_obj.cleaned_files + merged_output_path = ipg_rh_obj.merged_output_path + + # Check that the expected output exists + assert clean_data is not None + assert len(clean_files) == len(ipg_rh_obj._output_files) + assert os.path.exists(merged_output_path) + + # Check that the merged file length and results length + # is correct, and that no lines were lost and no extra lines + # were added. + expected_merged_line_count = 0 + for file in clean_files: + with open(file, "r") as f: + for count, line in enumerate(f): + if count == 0: + continue + if line.strip(): + expected_merged_line_count += 1 + + merged_line_count = 0 + with open(merged_output_path, "r") as f: + for count, line in enumerate(f): + if count == 0: + continue + if line.strip(): + merged_line_count += 1 + + assert merged_line_count == expected_merged_line_count + for measure in clean_data: + assert len(clean_data[measure]) == merged_line_count + + # Check that the clean data rows are ordered in increasing time + times_in_seconds = [] + for sys_time in clean_data["System Time"]: + split_sys_time = sys_time.split(":") + hour_min_sec = ":".join(split_sys_time[:-1]) + millis = float(split_sys_time[-1]) / 1000 + + timestruct = time.strptime(hour_min_sec, "%H:%M:%S") + times_in_seconds.append( + datetime.timedelta( + hours=timestruct.tm_hour, + minutes=timestruct.tm_min, + seconds=timestruct.tm_sec, + ).total_seconds() + + millis + ) + + for count, val in enumerate(times_in_seconds[:-1]): + assert times_in_seconds[count + 1] - val >= 0 + + +def test_ipg_rh_format_to_perfherder_with_no_results(ipg_rh_obj): + """Tests that formatting the data to a perfherder-like format + fails when clean_ipg_data was not called beforehand. + """ + formatted_data = ipg_rh_obj.format_ipg_data_to_partial_perfherder( + 1000, ipg_rh_obj._output_file_prefix + ) + assert formatted_data is None + + +def test_ipg_rh_format_to_perfherder_without_cutoff(ipg_rh_obj): + """Tests that formatting the data to a perfherder-like format + works as expected. + """ + ipg_rh_obj.clean_ipg_data() + formatted_data = ipg_rh_obj.format_ipg_data_to_partial_perfherder( + 1000, ipg_rh_obj._output_file_prefix + ) + + # Check that the expected entries exist + assert len(formatted_data.keys()) == 5 + assert "utilization" in formatted_data and "power-usage" in formatted_data + + assert ( + formatted_data["power-usage"]["test"] + == ipg_rh_obj._output_file_prefix + "-cumulative" + ) + assert ( + formatted_data["utilization"]["test"] + == ipg_rh_obj._output_file_prefix + "-utilization" + ) + assert ( + formatted_data["frequency-gpu"]["test"] + == ipg_rh_obj._output_file_prefix + "-frequency-gpu" + ) + assert ( + formatted_data["frequency-cpu"]["test"] + == ipg_rh_obj._output_file_prefix + "-frequency-cpu" + ) + assert ( + formatted_data["power-watts"]["test"] + == ipg_rh_obj._output_file_prefix + "-watts" + ) + + for measure in formatted_data: + # Make sure that the data exists + assert len(formatted_data[measure]["values"]) >= 1 + + for valkey in formatted_data[measure]["values"]: + # Make sure the names were simplified + assert "(" not in valkey + assert ")" not in valkey + + # Check that gpu utilization doesn't exist but cpu does + utilization_vals = formatted_data["utilization"]["values"] + assert "cpu" in utilization_vals + assert "gpu" not in utilization_vals + + expected_fields = ["processor-cores", "processor-package", "gpu", "dram"] + consumption_vals = formatted_data["power-usage"]["values"] + + consumption_vals_measures = list(consumption_vals.keys()) + + # This assertion ensures that the consumption values contain the expected + # fields and nothing more. + assert not list(set(consumption_vals_measures) - set(expected_fields)) + + +def test_ipg_rh_format_to_perfherder_with_cutoff(ipg_rh_obj): + """Tests that formatting the data to a perfherder-like format + works as expected. + """ + ipg_rh_obj.clean_ipg_data() + formatted_data = ipg_rh_obj.format_ipg_data_to_partial_perfherder( + 2.5, ipg_rh_obj._output_file_prefix + ) + + # Check that the formatted data was cutoff at the correct point, + # expecting that only the first row of merged will exist. + utilization_vals = formatted_data["utilization"]["values"] + assert utilization_vals["cpu"] == 14 + + # Expected vals are ordered in this way: [processor, cores, dram, gpu] + expected_vals = [6.517, 5.847, 0.244, 0.006] + consumption_vals = [ + formatted_data["power-usage"]["values"][measure] + for measure in formatted_data["power-usage"]["values"] + ] + assert not list(set(expected_vals) - set(consumption_vals)) + + +def test_ipg_rh_missingoutputfile(ipg_rh_obj): + """Tests that the IPGMissingOutputFileError is raised + when a bad file path is passed to _clean_ipg_file. + """ + bad_files = ["non-existent-file"] + with pytest.raises(IPGMissingOutputFileError): + ipg_rh_obj._clean_ipg_file(bad_files[0]) + + ipg_rh_obj._output_files = bad_files + with pytest.raises(IPGMissingOutputFileError): + ipg_rh_obj.clean_ipg_data() + + +def test_ipg_rh_emptyfile(ipg_rh_obj): + """Tests that the empty file error is raised when + a file exists, but does not contain any results in + it. + """ + base_path = os.path.abspath(os.path.dirname(__file__)) + "/files/" + bad_files = [base_path + "emptyfile.txt"] + with pytest.raises(IPGEmptyFileError): + ipg_rh_obj._clean_ipg_file(bad_files[0]) + + ipg_rh_obj._output_files = bad_files + with pytest.raises(IPGEmptyFileError): + ipg_rh_obj.clean_ipg_data() + + +def test_ipg_rh_valuetypeerrorfile(ipg_rh_obj): + """Tests that the IPGUnknownValueTypeError is raised + when a bad entry is encountered in a file that is cleaned. + """ + base_path = os.path.abspath(os.path.dirname(__file__)) + "/files/" + bad_files = [base_path + "valueerrorfile.txt"] + with pytest.raises(IPGUnknownValueTypeError): + ipg_rh_obj._clean_ipg_file(bad_files[0]) + + ipg_rh_obj._output_files = bad_files + with pytest.raises(IPGUnknownValueTypeError): + ipg_rh_obj.clean_ipg_data() + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/mozbase/mozpower/tests/test_macintelpower.py b/testing/mozbase/mozpower/tests/test_macintelpower.py new file mode 100644 index 0000000000..72fb6767f2 --- /dev/null +++ b/testing/mozbase/mozpower/tests/test_macintelpower.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python + +from __future__ import absolute_import + +import mock +import mozunit +import time + + +def test_macintelpower_init(macintelpower_obj): + """Tests that the MacIntelPower object is correctly initialized.""" + assert macintelpower_obj.ipg_path + assert macintelpower_obj.ipg + assert macintelpower_obj._os == "darwin" + assert macintelpower_obj._cpu == "intel" + + +def test_macintelpower_measuring(macintelpower_obj): + """Tests that measurement initialization and finalization works + for the MacIntelPower object. + """ + assert not macintelpower_obj.start_time + assert not macintelpower_obj.ipg._running + assert not macintelpower_obj.ipg._output_files + macintelpower_obj.initialize_power_measurements() + + # Check that initialization set start_time, and started the + # IPG measurer thread. + + # Wait a bit for thread to start, then check it + timeout = 10 + start = time.time() + while time.time() - start < timeout and not macintelpower_obj.ipg._running: + time.sleep(1) + + assert macintelpower_obj.start_time + assert macintelpower_obj.ipg._running + + test_data = {"power-usage": "data"} + + def formatter_side_effect(*args, **kwargs): + return test_data + + with mock.patch( + "mozpower.intel_power_gadget.IPGResultsHandler.clean_ipg_data" + ) as _: + with mock.patch( + "mozpower.intel_power_gadget.IPGResultsHandler." + "format_ipg_data_to_partial_perfherder" + ) as formatter: + formatter.side_effect = formatter_side_effect + + macintelpower_obj.finalize_power_measurements(wait_interval=2, timeout=30) + + # Check that finalization set the end_time, stopped the IPG measurement + # thread, added atleast one output file name, and initialized + # an IPGResultsHandler object + assert macintelpower_obj.end_time + assert not macintelpower_obj.ipg._running + assert macintelpower_obj.ipg._output_files + assert macintelpower_obj.ipg_results_handler + + # Check that the IPGResultHandler's methods were + # called + macintelpower_obj.ipg_results_handler.clean_ipg_data.assert_called() + macintelpower_obj.ipg_results_handler.format_ipg_data_to_partial_perfherder.assert_called_once_with( # NOQA: E501 + macintelpower_obj.end_time - macintelpower_obj.start_time, + "power-testing", + ) + + # Make sure we can get the expected perfherder data + # after formatting + assert macintelpower_obj.get_perfherder_data() == test_data + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/mozbase/mozpower/tests/test_mozpower.py b/testing/mozbase/mozpower/tests/test_mozpower.py new file mode 100644 index 0000000000..99a92abaef --- /dev/null +++ b/testing/mozbase/mozpower/tests/test_mozpower.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python + +from __future__ import absolute_import + +import mock +import mozunit +import pytest +import subprocess +import sys + +from mozpower import MozPower +from mozpower.mozpower import ( + OsCpuComboMissingError, + MissingProcessorInfoError, +) + + +def test_mozpower_android_init_failure(): + """Tests that the MozPower object fails when the android + flag is set. Remove this test once android is implemented. + """ + with pytest.raises(NotImplementedError): + MozPower(android=True) + + +def test_mozpower_oscpu_combo_missing_error(): + """Tests that the error OsCpuComboMissingError is raised + when we can't find a OS, and CPU combination (and, therefore, cannot + find a power measurer). + """ + with mock.patch.object( + MozPower, "_get_os", return_value="Not-An-OS" + ) as _, mock.patch.object( + MozPower, "_get_processor_info", return_value="Not-A-Processor" + ) as _: + with pytest.raises(OsCpuComboMissingError): + MozPower() + + +def test_mozpower_processor_info_missing_error(): + """Tests that the error MissingProcessorInfoError is raised + when failures occur during processor information parsing. + """ + # The builtins module name differs between python 2 and 3 + builtins_name = "__builtin__" + if sys.version_info[0] == 3: + builtins_name = "builtins" + + def os_side_effect_true(*args, **kwargs): + """Used as a passing side effect for os.path.exists calls.""" + return True + + def os_side_effect_false(*args, **kwargs): + """Used as a failing side effect for os.path.exists calls.""" + return False + + def subprocess_side_effect_fail(*args, **kwargs): + """Used to mock a failure in subprocess.check_output calls.""" + raise subprocess.CalledProcessError(1, "Testing failure") + + # Test failures in macos processor information parsing + with mock.patch.object(MozPower, "_get_os", return_value="Darwin") as _: + + with mock.patch("os.path.exists") as os_mock: + os_mock.side_effect = os_side_effect_false + + # Check that we fail properly if the processor + # information file doesn't exist. + with pytest.raises(MissingProcessorInfoError): + MozPower() + + # Check that we fail properly when an error occurs + # in the subprocess call. + os_mock.side_effect = os_side_effect_true + with mock.patch("subprocess.check_output") as subprocess_mock: + subprocess_mock.side_effect = subprocess_side_effect_fail + with pytest.raises(MissingProcessorInfoError): + MozPower() + + # Test failures in linux processor information parsing + with mock.patch.object(MozPower, "_get_os", return_value="Linux") as _: + + with mock.patch("os.path.exists") as os_mock: + os_mock.side_effect = os_side_effect_false + + # Check that we fail properly if the processor + # information file doesn't exist. + with pytest.raises(MissingProcessorInfoError): + MozPower() + + # Check that we fail properly when the model cannot be found + # with by searching for 'model name'. + os_mock.side_effect = os_side_effect_true + with mock.patch( + "%s.open" % builtins_name, mock.mock_open(read_data="") + ) as _: + with pytest.raises(MissingProcessorInfoError): + MozPower() + + +def test_mozpower_oscpu_combo(mozpower_obj): + """Tests that the correct class is instantiated for a given + OS and CPU combination (MacIntelPower in this case). + """ + assert mozpower_obj.measurer.__class__.__name__ == "MacIntelPower" + assert ( + mozpower_obj.measurer._os == "darwin" and mozpower_obj.measurer._cpu == "intel" + ) + + +def test_mozpower_measuring(mozpower_obj): + """Tests that measurers are properly called with each method.""" + with mock.patch( + "mozpower.macintelpower.MacIntelPower.initialize_power_measurements" + ) as _, mock.patch( + "mozpower.macintelpower.MacIntelPower.finalize_power_measurements" + ) as _, mock.patch( + "mozpower.macintelpower.MacIntelPower.get_perfherder_data" + ) as _: + mozpower_obj.initialize_power_measurements() + mozpower_obj.measurer.initialize_power_measurements.assert_called() + + mozpower_obj.finalize_power_measurements() + mozpower_obj.measurer.finalize_power_measurements.assert_called() + + mozpower_obj.get_perfherder_data() + mozpower_obj.measurer.get_perfherder_data.assert_called() + + +def test_mozpower_measuring_with_no_measurer(mozpower_obj): + """Tests that no errors occur when the measurer is None, and the + initialize, finalize, and get_perfherder_data functions are called. + """ + with mock.patch( + "mozpower.macintelpower.MacIntelPower.initialize_power_measurements" + ) as _, mock.patch( + "mozpower.macintelpower.MacIntelPower.finalize_power_measurements" + ) as _, mock.patch( + "mozpower.macintelpower.MacIntelPower.get_perfherder_data" + ) as _: + measurer = mozpower_obj.measurer + mozpower_obj.measurer = None + + mozpower_obj.initialize_power_measurements() + assert not measurer.initialize_power_measurements.called + + mozpower_obj.finalize_power_measurements() + assert not measurer.finalize_power_measurements.called + + mozpower_obj.get_perfherder_data() + assert not measurer.get_perfherder_data.called + + mozpower_obj.get_full_perfherder_data("mozpower") + assert not measurer.get_perfherder_data.called + + +def test_mozpower_get_full_perfherder_data(mozpower_obj): + """Tests that the full perfherder data blob is properly + produced given a partial perfherder data blob with correct + entries. + """ + partial_perfherder = { + "utilization": { + "type": "power", + "test": "mozpower", + "unit": "%", + "values": {"cpu": 50, "gpu": 0}, + }, + "power-usage": { + "type": "power", + "test": "mozpower", + "unit": "mWh", + "values": {"cpu": 2.0, "dram": 0.1, "gpu": 4.0}, + }, + "frequency-cpu": { + "type": "power", + "test": "mozpower", + "unit": "MHz", + "values": { + "cpu-favg": 2.0, + "cpu-fmax": 5.0, + "cpu-fmin": 0.0, + }, + }, + "frequency-gpu": { + "type": "power", + "test": "mozpower", + "unit": "MHz", + "values": {"gpu-favg": 3.0, "gpu-fmax": 6.0, "gpu-fmin": 0.0}, + }, + } + utilization_vals = [0, 50] + power_usage_vals = [2.0, 0.1, 4.0] + frequency_cpu_vals = [2.0, 5.0, 0.0] + frequency_gpu_vals = [3.0, 6.0, 0.0] + + with mock.patch("mozpower.macintelpower.MacIntelPower.get_perfherder_data") as gpd: + gpd.return_value = partial_perfherder + + full_perfherder = mozpower_obj.get_full_perfherder_data("mozpower") + assert full_perfherder["framework"]["name"] == "mozpower" + assert len(full_perfherder["suites"]) == 4 + + # Check that each of the two suites were created correctly. + suites = full_perfherder["suites"] + for suite in suites: + assert "subtests" in suite + + assert suite["type"] == "power" + assert suite["alertThreshold"] == 2.0 + assert suite["lowerIsBetter"] + + all_vals = [] + for subtest in suite["subtests"]: + assert "value" in subtest + + # Check that the subtest names were created correctly + if "utilization" in suite["name"]: + assert "utilization" in subtest["name"] + elif "power-usage" in suite["name"]: + assert "power-usage" in subtest["name"] + elif "frequency-cpu" in suite["name"]: + assert "frequency-cpu" in subtest["name"] + elif "frequency-gpu" in suite["name"]: + assert "frequency-gpu" in subtest["name"] + else: + assert False, "Unknown subtest name %s" % subtest["name"] + + all_vals.append(subtest["value"]) + + if "utilization" in suite["name"]: + assert len(all_vals) == 2 + assert suite["unit"] == "%" + assert suite["name"] == "mozpower-utilization" + assert not list(set(all_vals) - set(utilization_vals)) + assert suite["value"] == float(25) + elif "power-usage" in suite["name"]: + assert len(all_vals) == 3 + assert suite["unit"] == "mWh" + assert suite["name"] == "mozpower-power-usage" + assert not list(set(all_vals) - set(power_usage_vals)) + assert suite["value"] == float(6.1) + elif "frequency-cpu" in suite["name"]: + assert len(all_vals) == 3 + assert suite["unit"] == "MHz" + assert suite["name"] == "mozpower-frequency-cpu" + assert not list(set(all_vals) - set(frequency_cpu_vals)) + assert suite["value"] == float(2.0) + elif "frequency-gpu" in suite["name"]: + assert len(all_vals) == 3 + assert suite["unit"] == "MHz" + assert suite["name"] == "mozpower-frequency-gpu" + assert not list(set(all_vals) - set(frequency_gpu_vals)) + assert suite["value"] == float(3.0) + else: + assert False, "Unknown suite name %s" % suite["name"] + + +if __name__ == "__main__": + mozunit.main() diff --git a/testing/mozbase/mozpower/tests/test_powerbase.py b/testing/mozbase/mozpower/tests/test_powerbase.py new file mode 100644 index 0000000000..7fe14c97d4 --- /dev/null +++ b/testing/mozbase/mozpower/tests/test_powerbase.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python + +from __future__ import absolute_import + +import mock +import mozunit +import pytest + +from mozpower.powerbase import ( + PowerBase, + IPGExecutableMissingError, + PlatformUnsupportedError, +) + + +def test_powerbase_intialization(): + """Tests that the PowerBase class correctly raises + a NotImplementedError when attempting to instantiate + it directly. + """ + with pytest.raises(NotImplementedError): + PowerBase() + + +def test_powerbase_missing_methods(powermeasurer): + """Tests that trying to call PowerBase methods + without an implementation in the subclass raises + the NotImplementedError. + """ + with pytest.raises(NotImplementedError): + powermeasurer.initialize_power_measurements() + + with pytest.raises(NotImplementedError): + powermeasurer.finalize_power_measurements() + + with pytest.raises(NotImplementedError): + powermeasurer.get_perfherder_data() + + +def test_powerbase_get_ipg_path_mac(powermeasurer): + """Tests that getting the IPG path returns the expected results.""" + powermeasurer._os = "darwin" + powermeasurer._cpu = "not-intel" + + def os_side_effect(arg): + """Used to get around the os.path.exists check in + get_ipg_path which raises an IPGExecutableMissingError + otherwise. + """ + return True + + with mock.patch("os.path.exists", return_value=True) as m: + m.side_effect = os_side_effect + + # None is returned when a non-intel based machine is + # tested. + ipg_path = powermeasurer.get_ipg_path() + assert ipg_path is None + + # Check the path returned for mac intel-based machines. + powermeasurer._cpu = "intel" + ipg_path = powermeasurer.get_ipg_path() + assert ipg_path == "/Applications/Intel Power Gadget/PowerLog" + + +def test_powerbase_get_ipg_path_errors(powermeasurer): + """Tests that the appropriate error is output when calling + get_ipg_path with invalid/unsupported _os and _cpu entries. + """ + + powermeasurer._cpu = "intel" + powermeasurer._os = "Not-An-OS" + + def os_side_effect(arg): + """Used to force the error IPGExecutableMissingError to occur + (in case a machine running these tests is a mac, and has intel + power gadget installed). + """ + return False + + with pytest.raises(PlatformUnsupportedError): + powermeasurer.get_ipg_path() + + with mock.patch("os.path.exists", return_value=False) as m: + m.side_effect = os_side_effect + + powermeasurer._os = "darwin" + with pytest.raises(IPGExecutableMissingError): + powermeasurer.get_ipg_path() + + +if __name__ == "__main__": + mozunit.main() |