Home | History | Annotate | Download | only in functional 
      1 /*
      2  * Copyright (C) 2017 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 #define LOG_TAG "neuralnetworks_hidl_hal_test"
     18 
     19 #include "VtsHalNeuralnetworksV1_0TargetTest.h"
     20 
     21 #include "Callbacks.h"
     22 #include "Models.h"
     23 #include "TestHarness.h"
     24 
     25 #include <android-base/logging.h>
     26 #include <android/hidl/memory/1.0/IMemory.h>
     27 #include <hidlmemory/mapping.h>
     28 
     29 namespace android {
     30 namespace hardware {
     31 namespace neuralnetworks {
     32 namespace V1_0 {
     33 namespace vts {
     34 namespace functional {
     35 
     36 using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;
     37 using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback;
     38 using ::generated_tests::MixedTypedExampleType;
     39 
     40 namespace generated_tests {
     41 extern void Execute(const sp<IDevice>&, std::function<Model(void)>, std::function<bool(int)>,
     42                     const std::vector<MixedTypedExampleType>&);
     43 }
     44 
     45 // A class for test environment setup
     46 NeuralnetworksHidlEnvironment::NeuralnetworksHidlEnvironment() {}
     47 
     48 NeuralnetworksHidlEnvironment::~NeuralnetworksHidlEnvironment() {}
     49 
     50 NeuralnetworksHidlEnvironment* NeuralnetworksHidlEnvironment::getInstance() {
     51     // This has to return a "new" object because it is freed inside
     52     // ::testing::AddGlobalTestEnvironment when the gtest is being torn down
     53     static NeuralnetworksHidlEnvironment* instance = new NeuralnetworksHidlEnvironment();
     54     return instance;
     55 }
     56 
     57 void NeuralnetworksHidlEnvironment::registerTestServices() {
     58     registerTestService<IDevice>();
     59 }
     60 
     61 // The main test class for NEURALNETWORK HIDL HAL.
     62 NeuralnetworksHidlTest::~NeuralnetworksHidlTest() {}
     63 
     64 void NeuralnetworksHidlTest::SetUp() {
     65     device = ::testing::VtsHalHidlTargetTestBase::getService<IDevice>(
     66         NeuralnetworksHidlEnvironment::getInstance());
     67     ASSERT_NE(nullptr, device.get());
     68 }
     69 
     70 void NeuralnetworksHidlTest::TearDown() {}
     71 
     72 sp<IPreparedModel> NeuralnetworksHidlTest::doPrepareModelShortcut() {
     73     Model model = createValidTestModel();
     74 
     75     sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
     76     if (preparedModelCallback == nullptr) {
     77         return nullptr;
     78     }
     79     Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);
     80     if (!prepareLaunchStatus.isOk() || prepareLaunchStatus != ErrorStatus::NONE) {
     81         return nullptr;
     82     }
     83 
     84     preparedModelCallback->wait();
     85     ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
     86     sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
     87     if (prepareReturnStatus != ErrorStatus::NONE || preparedModel == nullptr) {
     88         return nullptr;
     89     }
     90 
     91     return preparedModel;
     92 }
     93 
     94 // create device test
     95 TEST_F(NeuralnetworksHidlTest, CreateDevice) {}
     96 
     97 // status test
     98 TEST_F(NeuralnetworksHidlTest, StatusTest) {
     99     Return<DeviceStatus> status = device->getStatus();
    100     ASSERT_TRUE(status.isOk());
    101     EXPECT_EQ(DeviceStatus::AVAILABLE, static_cast<DeviceStatus>(status));
    102 }
    103 
    104 // initialization
    105 TEST_F(NeuralnetworksHidlTest, GetCapabilitiesTest) {
    106     Return<void> ret =
    107         device->getCapabilities([](ErrorStatus status, const Capabilities& capabilities) {
    108             EXPECT_EQ(ErrorStatus::NONE, status);
    109             EXPECT_LT(0.0f, capabilities.float32Performance.execTime);
    110             EXPECT_LT(0.0f, capabilities.float32Performance.powerUsage);
    111             EXPECT_LT(0.0f, capabilities.quantized8Performance.execTime);
    112             EXPECT_LT(0.0f, capabilities.quantized8Performance.powerUsage);
    113         });
    114     EXPECT_TRUE(ret.isOk());
    115 }
    116 
    117 // supported operations positive test
    118 TEST_F(NeuralnetworksHidlTest, SupportedOperationsPositiveTest) {
    119     Model model = createValidTestModel();
    120     Return<void> ret = device->getSupportedOperations(
    121         model, [&](ErrorStatus status, const hidl_vec<bool>& supported) {
    122             EXPECT_EQ(ErrorStatus::NONE, status);
    123             EXPECT_EQ(model.operations.size(), supported.size());
    124         });
    125     EXPECT_TRUE(ret.isOk());
    126 }
    127 
    128 // supported operations negative test 1
    129 TEST_F(NeuralnetworksHidlTest, SupportedOperationsNegativeTest1) {
    130     Model model = createInvalidTestModel1();
    131     Return<void> ret = device->getSupportedOperations(
    132         model, [&](ErrorStatus status, const hidl_vec<bool>& supported) {
    133             EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, status);
    134             (void)supported;
    135         });
    136     EXPECT_TRUE(ret.isOk());
    137 }
    138 
    139 // supported operations negative test 2
    140 TEST_F(NeuralnetworksHidlTest, SupportedOperationsNegativeTest2) {
    141     Model model = createInvalidTestModel2();
    142     Return<void> ret = device->getSupportedOperations(
    143         model, [&](ErrorStatus status, const hidl_vec<bool>& supported) {
    144             EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, status);
    145             (void)supported;
    146         });
    147     EXPECT_TRUE(ret.isOk());
    148 }
    149 
    150 // prepare simple model positive test
    151 TEST_F(NeuralnetworksHidlTest, SimplePrepareModelPositiveTest) {
    152     Model model = createValidTestModel();
    153     sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
    154     ASSERT_NE(nullptr, preparedModelCallback.get());
    155     Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);
    156     ASSERT_TRUE(prepareLaunchStatus.isOk());
    157     EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
    158 
    159     preparedModelCallback->wait();
    160     ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
    161     EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
    162     sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
    163     EXPECT_NE(nullptr, preparedModel.get());
    164 }
    165 
    166 // prepare simple model negative test 1
    167 TEST_F(NeuralnetworksHidlTest, SimplePrepareModelNegativeTest1) {
    168     Model model = createInvalidTestModel1();
    169     sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
    170     ASSERT_NE(nullptr, preparedModelCallback.get());
    171     Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);
    172     ASSERT_TRUE(prepareLaunchStatus.isOk());
    173     EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, static_cast<ErrorStatus>(prepareLaunchStatus));
    174 
    175     preparedModelCallback->wait();
    176     ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
    177     EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, prepareReturnStatus);
    178     sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
    179     EXPECT_EQ(nullptr, preparedModel.get());
    180 }
    181 
    182 // prepare simple model negative test 2
    183 TEST_F(NeuralnetworksHidlTest, SimplePrepareModelNegativeTest2) {
    184     Model model = createInvalidTestModel2();
    185     sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
    186     ASSERT_NE(nullptr, preparedModelCallback.get());
    187     Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);
    188     ASSERT_TRUE(prepareLaunchStatus.isOk());
    189     EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, static_cast<ErrorStatus>(prepareLaunchStatus));
    190 
    191     preparedModelCallback->wait();
    192     ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
    193     EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, prepareReturnStatus);
    194     sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
    195     EXPECT_EQ(nullptr, preparedModel.get());
    196 }
    197 
    198 // execute simple graph positive test
    199 TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphPositiveTest) {
    200     std::vector<float> outputData = {-1.0f, -1.0f, -1.0f, -1.0f};
    201     std::vector<float> expectedData = {6.0f, 8.0f, 10.0f, 12.0f};
    202     const uint32_t OUTPUT = 1;
    203 
    204     sp<IPreparedModel> preparedModel = doPrepareModelShortcut();
    205     ASSERT_NE(nullptr, preparedModel.get());
    206     Request request = createValidTestRequest();
    207 
    208     auto postWork = [&] {
    209         sp<IMemory> outputMemory = mapMemory(request.pools[OUTPUT]);
    210         if (outputMemory == nullptr) {
    211             return false;
    212         }
    213         float* outputPtr = reinterpret_cast<float*>(static_cast<void*>(outputMemory->getPointer()));
    214         if (outputPtr == nullptr) {
    215             return false;
    216         }
    217         outputMemory->read();
    218         std::copy(outputPtr, outputPtr + outputData.size(), outputData.begin());
    219         outputMemory->commit();
    220         return true;
    221     };
    222 
    223     sp<ExecutionCallback> executionCallback = new ExecutionCallback();
    224     ASSERT_NE(nullptr, executionCallback.get());
    225     executionCallback->on_finish(postWork);
    226     Return<ErrorStatus> executeLaunchStatus = preparedModel->execute(request, executionCallback);
    227     ASSERT_TRUE(executeLaunchStatus.isOk());
    228     EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executeLaunchStatus));
    229 
    230     executionCallback->wait();
    231     ErrorStatus executionReturnStatus = executionCallback->getStatus();
    232     EXPECT_EQ(ErrorStatus::NONE, executionReturnStatus);
    233     EXPECT_EQ(expectedData, outputData);
    234 }
    235 
    236 // execute simple graph negative test 1
    237 TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphNegativeTest1) {
    238     sp<IPreparedModel> preparedModel = doPrepareModelShortcut();
    239     ASSERT_NE(nullptr, preparedModel.get());
    240     Request request = createInvalidTestRequest1();
    241 
    242     sp<ExecutionCallback> executionCallback = new ExecutionCallback();
    243     ASSERT_NE(nullptr, executionCallback.get());
    244     Return<ErrorStatus> executeLaunchStatus = preparedModel->execute(request, executionCallback);
    245     ASSERT_TRUE(executeLaunchStatus.isOk());
    246     EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, static_cast<ErrorStatus>(executeLaunchStatus));
    247 
    248     executionCallback->wait();
    249     ErrorStatus executionReturnStatus = executionCallback->getStatus();
    250     EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, executionReturnStatus);
    251 }
    252 
    253 // execute simple graph negative test 2
    254 TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphNegativeTest2) {
    255     sp<IPreparedModel> preparedModel = doPrepareModelShortcut();
    256     ASSERT_NE(nullptr, preparedModel.get());
    257     Request request = createInvalidTestRequest2();
    258 
    259     sp<ExecutionCallback> executionCallback = new ExecutionCallback();
    260     ASSERT_NE(nullptr, executionCallback.get());
    261     Return<ErrorStatus> executeLaunchStatus = preparedModel->execute(request, executionCallback);
    262     ASSERT_TRUE(executeLaunchStatus.isOk());
    263     EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, static_cast<ErrorStatus>(executeLaunchStatus));
    264 
    265     executionCallback->wait();
    266     ErrorStatus executionReturnStatus = executionCallback->getStatus();
    267     EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, executionReturnStatus);
    268 }
    269 
    270 // Mixed-typed examples
    271 typedef MixedTypedExampleType MixedTypedExample;
    272 
    273 // in frameworks/ml/nn/runtime/tests/generated/
    274 #include "all_generated_vts_tests.cpp"
    275 
    276 // TODO: Add tests for execution failure, or wait_for/wait_until timeout.
    277 //       Discussion:
    278 //       https://googleplex-android-review.git.corp.google.com/#/c/platform/hardware/interfaces/+/2654636/5/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0TargetTest.cpp@222
    279 
    280 }  // namespace functional
    281 }  // namespace vts
    282 }  // namespace V1_0
    283 }  // namespace neuralnetworks
    284 }  // namespace hardware
    285 }  // namespace android
    286 
    287 using android::hardware::neuralnetworks::V1_0::vts::functional::NeuralnetworksHidlEnvironment;
    288 
    289 int main(int argc, char** argv) {
    290     ::testing::AddGlobalTestEnvironment(NeuralnetworksHidlEnvironment::getInstance());
    291     ::testing::InitGoogleTest(&argc, argv);
    292     NeuralnetworksHidlEnvironment::getInstance()->init(&argc, argv);
    293 
    294     int status = RUN_ALL_TESTS();
    295     return status;
    296 }
    297