Home | History | Annotate | Download | only in runtime 
      1 /*
      2  * Copyright (C) 2017 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 #ifndef ANDROID_ML_NN_RUNTIME_MANAGER_H
     18 #define ANDROID_ML_NN_RUNTIME_MANAGER_H
     19 
     20 #include "HalInterfaces.h"
     21 #include "Utils.h"
     22 #include "VersionedInterfaces.h"
     23 
     24 #include <android-base/macros.h>
     25 #include <map>
     26 #include <unordered_set>
     27 #include <vector>
     28 
     29 namespace android {
     30 namespace nn {
     31 
     32 // A unified interface for actual driver devices as well as the CPU
     33 class Device {
     34    public:
     35     virtual ~Device() {}
     36 
     37     // Get the handle of underlying VersionedIDevice, if any
     38     virtual VersionedIDevice* getInterface() = 0;
     39 
     40     // Introspection methods returning device information
     41     virtual const char* getName() const = 0;
     42     virtual const char* getVersionString() const = 0;
     43     virtual int64_t getFeatureLevel() = 0;
     44     virtual int32_t getType() const = 0;
     45     virtual hidl_vec<Extension> getSupportedExtensions() const = 0;
     46 
     47     // If hidlModel is not compliant with the HAL version of the driver device,
     48     // then the behavior depends on whether or not a non-nullptr slicer is
     49     // provided.
     50     //
     51     // If there is no slicer, then no operations are supported.
     52     //
     53     // If there is a slicer, and it successfully slices the model, then some
     54     // operations may be supported.
     55     //
     56     // See the IModelSlicer class in Utils.h for more details.
     57     virtual void getSupportedOperations(const Model& hidlModel, IModelSlicer* slicer,
     58                                         hidl_vec<bool>* supportedOperations) = 0;
     59     void getSupportedOperations(const Model& hidlModel, hidl_vec<bool>* supportedOperations) {
     60         return getSupportedOperations(hidlModel, nullptr, supportedOperations);
     61     }
     62 
     63     virtual PerformanceInfo getPerformance(OperandType type) const = 0;
     64     virtual PerformanceInfo getRelaxedFloat32toFloat16PerformanceScalar() const = 0;
     65     virtual PerformanceInfo getRelaxedFloat32toFloat16PerformanceTensor() const = 0;
     66     virtual std::pair<uint32_t, uint32_t> getNumberOfCacheFilesNeeded() const = 0;
     67     bool isCachingSupported() const;
     68 
     69     virtual int prepareModel(
     70             const Model& hidlModel, ExecutionPreference executionPreference,
     71             const hidl_vec<hidl_handle>& modelCache, const hidl_vec<hidl_handle>& dataCache,
     72             const hidl_array<uint8_t, ANEURALNETWORKS_BYTE_SIZE_OF_CACHE_TOKEN>& token,
     73             std::shared_ptr<VersionedIPreparedModel>* preparedModel) = 0;
     74     virtual int prepareModelFromCache(
     75             const hidl_vec<hidl_handle>& modelCache, const hidl_vec<hidl_handle>& dataCache,
     76             const hidl_array<uint8_t, ANEURALNETWORKS_BYTE_SIZE_OF_CACHE_TOKEN>& token,
     77             std::shared_ptr<VersionedIPreparedModel>* preparedModel) = 0;
     78 };
     79 
     80 // Manages the NN HAL devices.  Only one instance of this class will exist.
     81 // Use get() to retrieve it.
     82 class DeviceManager {
     83    public:
     84     const std::vector<std::shared_ptr<Device>>& getDrivers() const {
     85         if (mSetCpuOnly || mDebugNNCpuOnly) {
     86             return mDevicesCpuOnly;
     87         }
     88         return mDevices;
     89     }
     90 
     91     // For testing only:
     92     void setUseCpuOnly(bool useCpuOnly) { mSetCpuOnly = useCpuOnly; }
     93     bool getUseCpuOnly() const { return mSetCpuOnly; }
     94     void setSyncExecHal(bool val) {
     95         mSyncExecHal = val;
     96         mSyncExecHalSetter = true;
     97     }
     98 
     99     bool syncExecCpu() const { return mSyncExecCpu; }
    100     bool syncExecHal() const { return mSyncExecHal; }
    101     bool syncExecRuntime() const { return mSyncExecRuntime; }
    102 
    103     // How to handle graph partitioning?
    104     // 0 - Don't do graph partitioning.
    105     // 1 - Do graph partitioning; but fall back to non-partitioned
    106     //     execution if there is a partitioning failure.
    107     // 2 - Do graph partitioning, and rely on it; there is no fallback.
    108     enum {
    109         kPartitioningNo              = 0,
    110         kPartitioningWithFallback    = 1,
    111         kPartitioningWithoutFallback = 2
    112     };
    113     uint32_t getPartitioning() const { return mPartitioning; }
    114     static bool partitioningAllowsFallback(uint32_t partitioning) {
    115         return partitioning == kPartitioningWithFallback;
    116     }
    117 
    118     bool strictSlicing() const { return mStrictSlicing; }
    119 
    120     // Returns the singleton manager.
    121     static DeviceManager* get();
    122 
    123     // Returns the singleton Cpu device.
    124     static std::shared_ptr<Device> getCpuDevice();
    125 
    126     // The forTest_* functions below are solely intended for use by unit tests.
    127 
    128     // Returns all devices (ignores the cpu-only flags).
    129     std::vector<std::shared_ptr<Device>> forTest_getDevices() const { return mDevices; }
    130 
    131     // Sets the device list (does not affect cpu-only queries).
    132     void forTest_setDevices(std::vector<std::shared_ptr<Device>> devices) {
    133         mDevices = std::move(devices);
    134     }
    135 
    136     // Register a test device.
    137     void forTest_registerDevice(const char* name, const sp<V1_0::IDevice>& device) {
    138         registerDevice(name, device);
    139     }
    140 
    141     // Re-initialize the list of available devices.
    142     void forTest_reInitializeDeviceList() {
    143         mDevices.clear();
    144         mDevicesCpuOnly.clear();
    145         findAvailableDevices();
    146     }
    147 
    148     // Make a test device
    149     static std::shared_ptr<Device> forTest_makeDriverDevice(const std::string& name,
    150                                                             const sp<V1_0::IDevice>& device);
    151 
    152     bool forTest_isCpuDevice(const ANeuralNetworksDevice* device) const {
    153         return reinterpret_cast<const Device*>(device) == getCpuDevice().get();
    154     }
    155 
    156    private:
    157     // Builds the list of available drivers and queries their capabilities.
    158     DeviceManager();
    159 
    160     // Adds a device for the manager to use.
    161     void registerDevice(const char* name, const sp<V1_0::IDevice>& device);
    162 
    163     void findAvailableDevices();
    164 
    165     // List of all the devices we discovered (including CpuDevice).
    166     std::vector<std::shared_ptr<Device>> mDevices;
    167 
    168     // We set this one to have CpuDevice only. To be used when m*CpuOnly is true.
    169     std::vector<std::shared_ptr<Device>> mDevicesCpuOnly;
    170 
    171     // If either of these is true, we'll ignore the drivers that are
    172     // on the device and run everything on the CPU.
    173     bool mSetCpuOnly = false;      // set by setUseCpuOnly()
    174     bool mDebugNNCpuOnly = false;  // derived from system property debug.nn.cpuonly
    175 
    176     // synchronous execution
    177     bool mSyncExecCpu = true;
    178     bool mSyncExecHal = true;         // Call executeSynchronously() when available on device.
    179     bool mSyncExecHalSetter = false;  // Has mSyncExecHal been set by setSyncExecHal()?
    180                                       // If so, don't allow the setting to be overridden
    181                                       //     by system property debug.nn.syncexec-hal
    182     bool mSyncExecRuntime = false;
    183 
    184     static const uint32_t kPartitioningDefault = kPartitioningWithFallback;
    185     uint32_t mPartitioning = kPartitioningDefault;
    186 
    187     bool mStrictSlicing = false;
    188 };
    189 
    190 } // namespace nn
    191 } // namespace android
    192 
    193 #endif // ANDROID_ML_NN_RUNTIME_MANAGER_H
    194