WIP Benchmark CPU to find clusters

source/SynthMark.h

@@ -67,6 +67,7 @@ constexpr int64_t SYNTHMARK_MICROS_PER_SECOND      = 1000 * 1000;
 constexpr int64_t SYNTHMARK_NANOS_PER_MICROSECOND  = 1000;
 constexpr int64_t SYNTHMARK_NANOS_PER_MILLISECOND  = 1000 * 1000;
 constexpr int64_t SYNTHMARK_NANOS_PER_SECOND       = 1000 * 1000 * 1000;
+constexpr double  SYNTHMARK_SECONDS_PER_NANO       = 1.0e-9;
 
 typedef float synth_float_t;
 

source/tools/AudioSinkBase.h

@@ -231,6 +231,19 @@ class AudioSinkBase
         mAdpfEnabled = enabled;
     }
 
+    bool isRealTime() const {
+        return mRealTime;
+    }
+
+    /**
+     * If true then the audio sink will run at the specified audio rate.
+     * If false then the audio sink will run as fast as possible.
+     * @param realTime
+     */
+    void setRealTime(bool realTime) {
+        mRealTime = realTime;
+    }
+
     static const char *schedulerToString(int scheduler) {
         scheduler = scheduler & 0xFFFF; // clear high flags like SCHED_RESET_ON_FORK
         switch(scheduler) {
@@ -287,6 +300,8 @@ class AudioSinkBase
     int32_t       mDefaultBufferSizeInBursts = kBufferSizeInBursts;
     int32_t       mBufferCapacityInFrames = 4 * 1024;
 
+    bool          mRealTime = false;
+
 private:
     IAudioSinkCallback *mCallback = NULL;
     int64_t        mFramesWritten = 0;

source/tools/AutomatedTestSuite.h

@@ -27,18 +27,18 @@
 #include "tools/VirtualAudioSink.h"
 #include "tools/VoiceMarkHarness.h"
 #include "tools/TestHarnessParameters.h"
+#include "BenchMarkHarness.h"
 
 /**
  * Run an automated test, analyze the results and print a report.
- * 1) Determine what CPUs are available
- * 2) Run VoiceMark on each CPU to determine big vs little CPUs
- * 3) Run LatencyMark with a light load, a heavy load, then an alternating load.
- * 4) Print report with analysis.
+ * 1) Determine what CPUs are available.
+ * 2) Run a quick BenchMark on each CPU to identify, little, medium, big and huge CPUs.
+ * 3) Use LatencyMark to detect various problems.
+ * 4) With a light load on a small core to detect preemption from bad drivers.
+ * 6) With a dynamic load on a small core to detect slow response frequency scaling CPU governor.
+ * 5) With moderate loads with no CPU affinity to detect problems with CPU migration.
+ * 6) With a dynamic load to detect slow response from CPU governor.
  *
- * The current code:
- *     assumes that the CPU numbering depends on their capacity,
- *     assumes that both lowCpu and highCpu are online,
- *     assume the existence of only two architectures, homogeneous or BIG.little.
  * TODO handle more architectures
  * TODO Measure latency without CPU affinity
  */
@@ -70,7 +70,7 @@ class AutomatedTestSuite : public TestHarnessParameters {
         mLogTool.log("\nSynthMark Version " SYNTHMARK_VERSION_TEXT "\n");
 
         mLogTool.log("\n-------- CPU Performance ------------\n");
-        int32_t err = measureLowHighCpuPerformance(sampleRate, framesPerBurst, 10);
+        int32_t err = measureCpuSpeeds(sampleRate, framesPerBurst, 10);
         if (err) return err;
 
         mLogTool.log("\n-------- LATENCY ------------\n");
@@ -147,87 +147,107 @@ struct LatencyResult {
         mResult->appendMessage(resultMessage.str());
     }
 
-    virtual int32_t measureLowHighCpuPerformance(int32_t sampleRate,
-                                                 int32_t framesPerBurst,
-                                                 int32_t numSeconds) {
-        std::stringstream resultMessage;
-        int numCPUs = HostTools::getCpuCount();
-        SynthMarkResult result1;
-        VoiceMarkHarness *harness = new VoiceMarkHarness(mAudioSink, &result1, mLogTool);
-        harness->setTargetCpuLoad(kMaxUtilization);
-        harness->setInitialVoiceCount(mNumVoices);
-        harness->setDelayNoteOnSeconds(mDelayNotesOn);
-        harness->setThreadType(mThreadType);
+    std::vector<std::vector<std::pair<int, int>>> findClustersWithIndices(const std::vector<int>& data, int threshold) {
+        std::vector<std::pair<int, int>> indexedData;
+        for (size_t i = 0; i < data.size(); ++i) {
+            indexedData.emplace_back(data[i], static_cast<int>(i)); // Store value and index
+        }
 
-        // TODO This is hack way to choose CPUs for BIG.little architectures.
-        // TODO Test each CPU or come up with something better.
-        mLogTool.log("Try to find BIG/LITTLE CPUs\n");
-        int lowCpu = (1 * numCPUs) / 4;
-        mAudioSink->setRequestedCpu(lowCpu);
-        mLogTool.log("Run low VoiceMark with CPU #%d\n", lowCpu);
-        int32_t err = harness->runTest(sampleRate, framesPerBurst, numSeconds);
-        if (err) {
-            delete harness;
-            return err;
+        std::sort(indexedData.begin(), indexedData.end()); // Sort based on values
+
+        std::vector<std::vector<std::pair<int, int>>> clusters;
+        std::vector<std::pair<int, int>> currentCluster;
+
+        if (!indexedData.empty()) {
+            currentCluster.push_back(indexedData[0]);
+            clusters.push_back(currentCluster);
+            currentCluster.clear();
+        }
+
+        for (size_t i = 1; i < indexedData.size(); ++i) {
+            if (std::abs(indexedData[i].first - indexedData[i - 1].first) > threshold) {
+                clusters.push_back(currentCluster);
+                currentCluster.clear();
+            }
+            if(currentCluster.empty()){
+                clusters.back().push_back(indexedData[i]);
+            }
+            else{
+                currentCluster.push_back(indexedData[i]);
+            }
+        }
+        if(!currentCluster.empty()){
+            clusters.push_back(currentCluster);
         }
+        if(clusters[0].empty()){
+            clusters.erase(clusters.begin());
+        }
+
+        return clusters;
+    }
 
-        double voiceMarkLowIndex = result1.getMeasurement();
-        mLogTool.log("low VoiceMark_%d = %5.1f\n", kMaxUtilizationPercent, voiceMarkLowIndex);
-        double voiceMarkHighIndex = voiceMarkLowIndex;
-
-        int highCpu = (3 * numCPUs) / 4;
-        // If there are more than 2 CPUs than measure one each from top and bottom half in
-        // case they are heterogeneous.
-        if (lowCpu != highCpu) {
-            mAudioSink->setRequestedCpu(highCpu);
-            mLogTool.log("Run high VoiceMark with CPU #%d\n", highCpu);
-            err = harness->runTest(sampleRate, framesPerBurst, numSeconds);
+    // Measure all of the CPU speeds.
+    virtual int32_t measureCpuSpeeds(int32_t sampleRate,
+                                     int32_t framesPerBurst,
+                                     int32_t numSeconds) {
+        std::stringstream resultMessage;
+        int numCPUs = HostTools::getCpuCount();
+        std::vector<int32_t> cpuMaxVoices;
+        int32_t maxVoicesPossible = 0;
+        SynthMarkResult result1;
+        for (int cpuIndex = 0; cpuIndex < numCPUs; cpuIndex++) {
+            BenchMarkHarness harness(mAudioSink, &result1, mLogTool);
+            harness.setNumVoices(getNumVoices());
+            harness.setDelayNoteOnSeconds(mDelayNotesOn);
+            harness.setThreadType(mThreadType);
+
+            mAudioSink->setRequestedCpu(cpuIndex);
+            mLogTool.log("Run BenchMark with CPU #%d\n", cpuIndex);
+            int32_t err = harness.runTest(sampleRate, framesPerBurst, numSeconds);
             if (err) {
-                delete harness;
                 return err;
             }
-            voiceMarkHighIndex = result1.getMeasurement();
-            mLogTool.log("high VoiceMark_%d = %5.1f\n", kMaxUtilizationPercent, voiceMarkHighIndex);
 
-            double averageVoiceMark = 0.5 * (voiceMarkHighIndex + voiceMarkLowIndex);
-            double threshold = kHighLowThreshold * averageVoiceMark;
+            int32_t maxVoicesPerCpu = harness.getMaxVoices();
+            mLogTool.log("max_voices_%d = %5.1f\n", cpuIndex, maxVoicesPerCpu);
+            cpuMaxVoices.push_back(maxVoicesPerCpu);
+            maxVoicesPossible = std::max(maxVoicesPossible, maxVoicesPerCpu);
+            // Give CPU time to settle down before the next benchmark.
+            HostTools::sleepForNanoseconds(500 * SYNTHMARK_NANOS_PER_MILLISECOND);
+        }
+
+        // Figure out CPU clusters from the benchmarks.
+        const int threshold = 30;
+        std::vector<std::vector<std::pair<int, int>>> clusters = findClustersWithIndices(cpuMaxVoices, threshold);
 
-            mHaveBigLittle = (abs(voiceMarkHighIndex - voiceMarkLowIndex) > threshold);
+        for (size_t i = 0; i < clusters.size(); ++i) {
+            mLogTool.log("Cluster[#%d]: ", i);
+            for (const auto& pair : clusters[i]) {
+                mLogTool.log("( %d, %d) " ,  pair.first, pair.second);
+            }
+            mLogTool.log("\n");
+            std::cout << std::endl;
         }
 
+        mHaveBigLittle = clusters.size() > 1;
+
+        const auto littleCluster = clusters[0];
+        const auto& pair = littleCluster[littleCluster.size() - 1];
+        mLittleCpu = pair.second;
+        mVoiceMarkLittle = pair.first;
         if (mHaveBigLittle) {
-            if (voiceMarkHighIndex > voiceMarkLowIndex) {
-                mLittleCpu = lowCpu;
-                mVoiceMarkLittle = voiceMarkLowIndex;
-                mBigCpu = highCpu;
-                mVoiceMarkBig = voiceMarkHighIndex;
-            } else {
-                mLittleCpu = highCpu;
-                mVoiceMarkLittle = voiceMarkHighIndex;
-                mBigCpu = lowCpu;
-                mVoiceMarkBig = voiceMarkLowIndex;
-            }
-            resultMessage << "# The CPU seems to be heterogeneous. Assume BIG-little.\n";
+            resultMessage << "# The CPU seems to be heterogeneous.\n";
             resultMessage << "cpu.little = " << mLittleCpu << std::endl;
-            resultMessage << "cpu.big = " << mBigCpu << std::endl;
         } else {
-            mLittleCpu = highCpu;
-            mVoiceMarkLittle = voiceMarkHighIndex;
-            mBigCpu = highCpu;
-            mVoiceMarkBig = voiceMarkHighIndex;
             resultMessage << "# The CPU seems to be homogeneous.\n";
             resultMessage << "cpu = " << mLittleCpu << std::endl;
         }
 
-        resultMessage << "voice.mark." << cpuToBigLittle(lowCpu) << " = " << voiceMarkLowIndex << std::endl;
-        if (lowCpu != highCpu) {
-            resultMessage << "voice.mark." << cpuToBigLittle(highCpu) << " = " << voiceMarkHighIndex << std::endl;
-        }
+        resultMessage << "max.voices.little  = " << mVoiceMarkLittle << std::endl;
 
-        // std::cout << result1.getResultMessage();
+        //std::cout << result1.getResultMessage();
 
         mResult->appendMessage(resultMessage.str());
-        delete harness;
         return SYNTHMARK_RESULT_SUCCESS;
     }
 

source/tools/BenchMarkHarness.h

@@ -0,0 +1,113 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef SYNTHMARK_BENCHMARK_HARNESS_H
+#define SYNTHMARK_BENCHMARK_HARNESS_H
+
+#include <cmath>
+#include <cstdint>
+#include <sstream>
+
+#include "AudioSinkBase.h"
+#include "SynthMark.h"
+#include "synth/Synthesizer.h"
+#include "tools/LogTool.h"
+#include "tools/TestHarnessBase.h"
+#include "tools/TimingAnalyzer.h"
+#include "TestHarnessParameters.h"
+
+
+/**
+ * Play a specified number of voices on a Synthesizer
+ * as fast as possible and measure the time it takes.
+ */
+class BenchMarkHarness : public TestHarnessBase {
+public:
+    BenchMarkHarness(AudioSinkBase *audioSink, SynthMarkResult *result, LogTool &logTool)
+            : TestHarnessBase(audioSink, result, logTool)
+    {
+        std::stringstream testName;
+        testName << "BenchMark";
+        mTestName = testName.str();
+        audioSink->setRealTime(false); // run as fast as possible
+    }
+
+    virtual ~BenchMarkHarness() {
+    }
+
+    virtual void onBeginMeasurement() override {
+        mResult->setTestName(mTestName);
+        mLogTool.log("---- Starting %s ----\n", mTestName.c_str());
+        mBeatCount = 0;
+        mTestStartTimeNanos = HostTools::getNanoTime();
+    }
+
+    double reportSpeed() {
+        double elapsedTimeSeconds = mTimer.getTotalTime() * SYNTHMARK_SECONDS_PER_NANO;
+        int32_t framesWritten = getFramesWritten();
+        double virtualTime = (framesWritten - mLastFramesWritten) / (double) getSampleRate();
+        double fractionRealTime = elapsedTimeSeconds / virtualTime;
+        mLogTool.log("%2d: %5.3f%% of real-time\n",
+                     mBeatCount, fractionRealTime * 100);
+        mLastFramesWritten = framesWritten;
+        return fractionRealTime;
+    }
+
+    virtual int32_t onBeforeNoteOn() override {
+        if (mBeatCount > 0) {
+            double fractionRealTime = reportSpeed();
+            mStableCount += (fractionRealTime > (mMinFractionRealTime * 0.99)) ? 1 : 0;
+            mMinFractionRealTime = std::min(mMinFractionRealTime, fractionRealTime);
+        }
+        mTimer.reset();
+        mBeatCount++;
+        int64_t now = HostTools::getNanoTime();
+        bool timeOut = (now - mTestStartTimeNanos) > (5 * SYNTHMARK_NANOS_PER_SECOND);
+        return ((mStableCount < 2) && !timeOut)  ? 0 : 1;
+    }
+
+    int32_t getMaxVoices() {
+        return (int32_t) (getNumVoices() / mMinFractionRealTime);
+    }
+
+    virtual void onEndMeasurement() override {
+        int8_t resultCode = SYNTHMARK_RESULT_SUCCESS;
+        std::stringstream resultMessage;
+        std::stringstream cpuPart;
+        if (mAudioSink->getActualCpu() >= 0) {
+            cpuPart << ".cpu." << mAudioSink->getActualCpu();
+        }
+        resultMessage << "percent.realtime" << cpuPart.str()
+                << ".voices." << getNumVoices()
+                << " = " <<  (mMinFractionRealTime * 100) << "%%" << std::endl;
+        resultMessage << "max.voices" << cpuPart.str()
+                << " = " << getMaxVoices() << std::endl;
+        mResult->setResultCode(resultCode);
+        resultMessage << mCpuAnalyzer.dump();
+
+        mResult->setMeasurement(mMinFractionRealTime);
+        mResult->appendMessage(resultMessage.str());
+    }
+
+private:
+    double  mMinFractionRealTime = 1.0e10; // start very high
+    int32_t mBeatCount = 0;
+    int32_t mLastFramesWritten = 0;
+    int32_t mStableCount = 0;
+    int64_t mTestStartTimeNanos = 0;
+};
+
+#endif //SYNTHMARK_BENCHMARK_HARNESS_H