android10/frameworks/ml/nn/common/operations/FullyConnected.cpp

/*
 * Copyright (C) 2017 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "CpuOperationUtils.h"
#include "OperationResolver.h"

#include "tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h"
#include "tensorflow/lite/kernels/internal/reference/reference_ops.h"

#include "Tracing.h"

namespace android {
namespace nn {
namespace fully_connected {

constexpr char kOperationName[] = "FULLY_CONNECTED";

constexpr uint32_t kNumInputs = 4;
constexpr uint32_t kInputTensor = 0;
constexpr uint32_t kWeightsTensor = 1;
constexpr uint32_t kBiasTensor = 2;
constexpr uint32_t kActivationScalar = 3;

constexpr uint32_t kNumOutputs = 1;
constexpr uint32_t kOutputTensor = 0;

namespace {

// executionMutex is used to protect concurrent access of non-threadsafe resources
// like gemmlowp::GemmContext.
// std::mutex is safe for pthreads on Android.
static std::mutex executionMutex;

bool fullyConnectedFloat32(const float* inputData, const Shape& inputShape,
                           const float* weightsData, const Shape& weightsShape,
                           const float* biasData, const Shape& biasShape, int32_t activation,
                           float* outputData, const Shape& outputShape) {
    NNTRACE_TRANS("fullyConnectedFloat32");
    float output_activation_min, output_activation_max;
    CalculateActivationRangeFloat(activation, &output_activation_min, &output_activation_max);

    // b/80425683, optimized implementation produces incorrect results when the
    // number of input elements is the squre of batch_size.
    uint32_t batch_size = getSizeOfDimension(outputShape, 0);
    uint32_t input_n_elements = getNumberOfElements(inputShape);
    if (batch_size * batch_size == input_n_elements) {
        NNTRACE_COMP_SWITCH("reference_ops::FullyConnected");
        tflite::reference_ops::FullyConnected(inputData, convertShapeToDims(inputShape),
                                              weightsData, convertShapeToDims(weightsShape),
                                              biasData, convertShapeToDims(biasShape),
                                              output_activation_min, output_activation_max,
                                              outputData, convertShapeToDims(outputShape));
    } else {
        NNTRACE_COMP_SWITCH("optimized_ops::FullyConnected");
        tflite::optimized_ops::FullyConnected(inputData, convertShapeToDims(inputShape),
                                              weightsData, convertShapeToDims(weightsShape),
                                              biasData, convertShapeToDims(biasShape),
                                              output_activation_min, output_activation_max,
                                              outputData, convertShapeToDims(outputShape));
    }
    return true;
}

bool fullyConnectedFloat16(const _Float16* inputData, const Shape& inputShape,
                           const _Float16* weightsData, const Shape& weightsShape,
                           const _Float16* biasData, const Shape& biasShape, int32_t activation,
                           _Float16* outputData, const Shape& outputShape) {
    NNTRACE_TRANS("fullyConnectedFloat16");
    std::vector<float> inputDataFloat32(getNumberOfElements(inputShape));
    convertFloat16ToFloat32(inputData, &inputDataFloat32);
    std::vector<float> weightsDataFloat32(getNumberOfElements(weightsShape));
    convertFloat16ToFloat32(weightsData, &weightsDataFloat32);
    std::vector<float> biasDataFloat32(getNumberOfElements(biasShape));
    convertFloat16ToFloat32(biasData, &biasDataFloat32);

    std::vector<float> outputDataFloat32(getNumberOfElements(outputShape));
    fullyConnectedFloat32(inputDataFloat32.data(), inputShape, weightsDataFloat32.data(),
                          weightsShape, biasDataFloat32.data(), biasShape, activation,
                          outputDataFloat32.data(), outputShape);
    convertFloat32ToFloat16(outputDataFloat32, outputData);

    return true;
}

bool fullyConnectedQuant8(const uint8_t* inputData, const Shape& inputShape,
                          const uint8_t* weightsData, const Shape& weightsShape,
                          const int32_t* biasData, const Shape& biasShape, int32_t activation,
                          uint8_t* outputData, const Shape& outputShape) {
    NNTRACE_TRANS("fullyConnectedQuant8");
    int32_t inputOffset = -inputShape.offset;
    int32_t weightsOffset = -weightsShape.offset;
    int32_t outputOffset = outputShape.offset;

    double realMultiplier = 0.0;
    int32_t outputMultiplier = 0;
    int32_t outputShift = 0;
    int32_t outputActivationMin = 0;
    int32_t outputActivationMax = 0;

    NN_RET_CHECK(GetQuantizedConvolutionMultipler(inputShape, weightsShape, biasShape, outputShape,
                                                  &realMultiplier));
    int exponent;
    NN_RET_CHECK(QuantizeMultiplier(realMultiplier, &outputMultiplier, &exponent));
    outputShift = -exponent;
    CalculateActivationRangeUint8(activation, outputShape, &outputActivationMin,
                                  &outputActivationMax);

    static gemmlowp::GemmContext gemmContext;

    // Prevent concurrent executions that access gemmContext.
    std::unique_lock<std::mutex> lock(executionMutex);
    // Alow gemmlowp automatically decide how many threads to use.
    gemmContext.set_max_num_threads(0);

    NNTRACE_COMP_SWITCH("optimized_ops::FullyConnected");
    tflite::optimized_ops::FullyConnected(inputData, convertShapeToDims(inputShape), inputOffset,
                                          weightsData, convertShapeToDims(weightsShape),
                                          weightsOffset, biasData, convertShapeToDims(biasShape),
                                          outputOffset, outputMultiplier, outputShift,
                                          outputActivationMin, outputActivationMax, outputData,
                                          convertShapeToDims(outputShape), &gemmContext);

    return true;
}

}  // namespace

bool validate(const IOperationValidationContext* context) {
    NN_RET_CHECK_EQ(context->getNumInputs(), kNumInputs);
    NN_RET_CHECK_EQ(context->getNumOutputs(), kNumOutputs);
    auto inputType = context->getInputType(kInputTensor);
    std::vector<OperandType> inExpectedTypes;
    std::vector<OperandType> outExpectedTypes;
    if (inputType == OperandType::TENSOR_FLOAT32) {
        NN_RET_CHECK(validateHalVersion(context, HalVersion::V1_0));
        inExpectedTypes = {
                OperandType::TENSOR_FLOAT32,
                OperandType::TENSOR_FLOAT32,
                OperandType::TENSOR_FLOAT32,
                OperandType::INT32,
        };
    } else if (inputType == OperandType::TENSOR_FLOAT16) {
        NN_RET_CHECK(validateHalVersion(context, HalVersion::V1_2));
        inExpectedTypes = {
                OperandType::TENSOR_FLOAT16,
                OperandType::TENSOR_FLOAT16,
                OperandType::TENSOR_FLOAT16,
                OperandType::INT32,
        };
    } else if (inputType == OperandType::TENSOR_QUANT8_ASYMM) {
        // NeuralNetworks.h specifies that ANEURALNETWORKS_FULLY_CONNECTED's output must
        // meet "outputScale > inputScale * weightsScale" for the operand type
        // ANEURALNETWORKS_TENSOR_QUANT8_ASYMM before API level 29.
        const float inputScale = context->getInputShape(kInputTensor).scale;
        const float weightsScale = context->getInputShape(kWeightsTensor).scale;
        const float outputScale = context->getOutputShape(kOutputTensor).scale;
        bool meetsQuantizedScaleConstraintBeforeV1_2 = (outputScale > inputScale * weightsScale);

        if (!meetsQuantizedScaleConstraintBeforeV1_2) {
            NN_RET_CHECK(validateHalVersion(context, HalVersion::V1_2));
        } else {
            NN_RET_CHECK(validateHalVersion(context, HalVersion::V1_0));
        }

        inExpectedTypes = {
                OperandType::TENSOR_QUANT8_ASYMM,
                OperandType::TENSOR_QUANT8_ASYMM,
                OperandType::TENSOR_INT32,
                OperandType::INT32,
        };
    } else {
        NN_RET_CHECK_FAIL() << "Unsupported input tensor type for operation " << kOperationName;
        return false;
    }
    NN_RET_CHECK(validateInputTypes(context, inExpectedTypes));
    NN_RET_CHECK(validateOutputTypes(context, {inputType}));
    return true;
}

bool prepare(IOperationExecutionContext* context) {
    Shape input = context->getInputShape(kInputTensor);
    Shape weights = context->getInputShape(kWeightsTensor);
    Shape bias = context->getInputShape(kBiasTensor);

    // Check all the parameters of tensor match within themselves and match the
    // input configuration.
    NN_RET_CHECK(input.type == weights.type);
    if (input.type == OperandType::TENSOR_QUANT8_ASYMM) {
        NN_RET_CHECK(bias.type == OperandType::TENSOR_INT32);
    } else {
        NN_RET_CHECK(input.type == bias.type);
    }
    // The Tensorflow fully connected layer specification says that input should
    // be of at least rank 2, so we check. Tflite doesn't check.
    NN_RET_CHECK_GE(getNumberOfDimensions(input), 2);
    NN_RET_CHECK_EQ(getNumberOfDimensions(weights), 2);
    uint32_t input_n_elements = getNumberOfElements(input);
    uint32_t num_units = getSizeOfDimension(weights, 0);
    uint32_t input_size = getSizeOfDimension(weights, 1);
    // Only batch_size can be 0.
    NN_RET_CHECK_GT(num_units, 0);
    NN_RET_CHECK_GT(input_size, 0);
    uint32_t batch_size = input_n_elements / input_size;
    NN_RET_CHECK_EQ(getSizeOfDimension(bias, 0), num_units);
    NN_RET_CHECK_EQ(input_size * batch_size, input_n_elements);

    Shape output = context->getOutputShape(kOutputTensor);
    output.type = input.type;
    output.dimensions = {batch_size, num_units};
    return context->setOutputShape(kOutputTensor, output);
}

bool execute(IOperationExecutionContext* context) {
    // Bypass execution in the case of zero-sized input.
    if (getNumberOfElements(context->getOutputShape(kOutputTensor)) == 0) return true;
    switch (context->getInputType(kInputTensor)) {
        case OperandType::TENSOR_FLOAT32:
            return fullyConnectedFloat32(context->getInputBuffer<float>(kInputTensor),
                                         context->getInputShape(kInputTensor),
                                         context->getInputBuffer<float>(kWeightsTensor),
                                         context->getInputShape(kWeightsTensor),
                                         context->getInputBuffer<float>(kBiasTensor),
                                         context->getInputShape(kBiasTensor),
                                         context->getInputValue<int32_t>(kActivationScalar),
                                         context->getOutputBuffer<float>(kOutputTensor),
                                         context->getOutputShape(kOutputTensor));
        case OperandType::TENSOR_FLOAT16:
            return fullyConnectedFloat16(context->getInputBuffer<_Float16>(kInputTensor),
                                         context->getInputShape(kInputTensor),
                                         context->getInputBuffer<_Float16>(kWeightsTensor),
                                         context->getInputShape(kWeightsTensor),
                                         context->getInputBuffer<_Float16>(kBiasTensor),
                                         context->getInputShape(kBiasTensor),
                                         context->getInputValue<int32_t>(kActivationScalar),
                                         context->getOutputBuffer<_Float16>(kOutputTensor),
                                         context->getOutputShape(kOutputTensor));
        case OperandType::TENSOR_QUANT8_ASYMM:
            return fullyConnectedQuant8(context->getInputBuffer<uint8_t>(kInputTensor),
                                        context->getInputShape(kInputTensor),
                                        context->getInputBuffer<uint8_t>(kWeightsTensor),
                                        context->getInputShape(kWeightsTensor),
                                        context->getInputBuffer<int32_t>(kBiasTensor),
                                        context->getInputShape(kBiasTensor),
                                        context->getInputValue<int32_t>(kActivationScalar),
                                        context->getOutputBuffer<uint8_t>(kOutputTensor),
                                        context->getOutputShape(kOutputTensor));
        default:
            NN_RET_CHECK_FAIL() << "Unsupported tensor type for operation " << kOperationName;
    }
}

}  // namespace fully_connected

NN_REGISTER_OPERATION(FULLY_CONNECTED, fully_connected::kOperationName, fully_connected::validate,
                      fully_connected::prepare, fully_connected::execute,
                      .allowZeroSizedInput = true);

}  // namespace nn
}  // namespace android