123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707 |
- #include "rsCpuScriptGroup2.h"
- #include <dlfcn.h>
- #include <stdio.h>
- #include <stdlib.h>
- #include <unistd.h>
- #include <set>
- #include <sstream>
- #include <string>
- #include <vector>
- #ifndef RS_COMPATIBILITY_LIB
- #include "bcc/Config.h"
- #endif
- #include "cpu_ref/rsCpuCore.h"
- #include "rsClosure.h"
- #include "rsContext.h"
- #include "rsCpuCore.h"
- #include "rsCpuExecutable.h"
- #include "rsCpuScript.h"
- #include "rsScript.h"
- #include "rsScriptGroup2.h"
- #include "rsScriptIntrinsic.h"
- using std::string;
- using std::vector;
- namespace android {
- namespace renderscript {
- namespace {
- const size_t DefaultKernelArgCount = 2;
- void groupRoot(const RsExpandKernelDriverInfo *kinfo, uint32_t xstart,
- uint32_t xend, uint32_t outstep) {
- const List<CPUClosure*>& closures = *(List<CPUClosure*>*)kinfo->usr;
- RsExpandKernelDriverInfo *mutable_kinfo = const_cast<RsExpandKernelDriverInfo *>(kinfo);
- const size_t oldInLen = mutable_kinfo->inLen;
- decltype(mutable_kinfo->inStride) oldInStride;
- memcpy(&oldInStride, &mutable_kinfo->inStride, sizeof(oldInStride));
- for (CPUClosure* cpuClosure : closures) {
- const Closure* closure = cpuClosure->mClosure;
-
- rsAssert(closure->mNumArg <= RS_KERNEL_INPUT_LIMIT);
- for (size_t i = 0; i < closure->mNumArg; i++) {
- const void* arg = closure->mArgs[i];
- const Allocation* a = (const Allocation*)arg;
- const uint32_t eStride = a->mHal.state.elementSizeBytes;
- const uint8_t* ptr = (uint8_t*)(a->mHal.drvState.lod[0].mallocPtr) +
- eStride * xstart;
- if (kinfo->dim.y > 1) {
- ptr += a->mHal.drvState.lod[0].stride * kinfo->current.y;
- }
- mutable_kinfo->inPtr[i] = ptr;
- mutable_kinfo->inStride[i] = eStride;
- }
- mutable_kinfo->inLen = closure->mNumArg;
- const Allocation* out = closure->mReturnValue;
- const uint32_t ostep = out->mHal.state.elementSizeBytes;
- const uint8_t* ptr = (uint8_t *)(out->mHal.drvState.lod[0].mallocPtr) +
- ostep * xstart;
- if (kinfo->dim.y > 1) {
- ptr += out->mHal.drvState.lod[0].stride * kinfo->current.y;
- }
- mutable_kinfo->outPtr[0] = const_cast<uint8_t*>(ptr);
-
-
- mutable_kinfo->usr = cpuClosure->mSi;
- cpuClosure->mFunc(kinfo, xstart, xend, ostep);
- }
- mutable_kinfo->inLen = oldInLen;
- mutable_kinfo->usr = &closures;
- memcpy(&mutable_kinfo->inStride, &oldInStride, sizeof(oldInStride));
- }
- }
- Batch::Batch(CpuScriptGroup2Impl* group, const char* name) :
- mGroup(group), mFunc(nullptr) {
- mName = strndup(name, strlen(name));
- }
- Batch::~Batch() {
- for (CPUClosure* c : mClosures) {
- delete c;
- }
- free(mName);
- }
- bool Batch::conflict(CPUClosure* cpuClosure) const {
- if (mClosures.empty()) {
- return false;
- }
- const Closure* closure = cpuClosure->mClosure;
- if (!closure->mIsKernel || !mClosures.front()->mClosure->mIsKernel) {
-
-
- return true;
- }
- const auto& globalDeps = closure->mGlobalDeps;
- const auto& argDeps = closure->mArgDeps;
- for (CPUClosure* c : mClosures) {
- const Closure* batched = c->mClosure;
- if (globalDeps.find(batched) != globalDeps.end()) {
- return true;
- }
- const auto& it = argDeps.find(batched);
- if (it != argDeps.end()) {
- const auto& args = (*it).second;
- for (const auto &p1 : *args) {
- if (p1.second.get() != nullptr) {
- return true;
- }
- }
- }
- }
-
-
- const Closure* lastBatched = mClosures.back()->mClosure;
- const auto& it = argDeps.find(lastBatched);
- if (it == argDeps.end()) {
- return true;
- }
- const auto& args = (*it).second;
- for (const auto &p1 : *args) {
- if (p1.first == 0 && p1.second.get() == nullptr) {
-
-
- return false;
- }
- }
- return true;
- }
- CpuScriptGroup2Impl::CpuScriptGroup2Impl(RsdCpuReferenceImpl *cpuRefImpl,
- const ScriptGroupBase *sg) :
- mCpuRefImpl(cpuRefImpl), mGroup((const ScriptGroup2*)(sg)),
- mExecutable(nullptr), mScriptObj(nullptr) {
- rsAssert(!mGroup->mClosures.empty());
- mCpuRefImpl->lockMutex();
- Batch* batch = new Batch(this, "Batch0");
- int i = 0;
- for (Closure* closure: mGroup->mClosures) {
- CPUClosure* cc;
- const IDBase* funcID = closure->mFunctionID.get();
- RsdCpuScriptImpl* si =
- (RsdCpuScriptImpl *)mCpuRefImpl->lookupScript(funcID->mScript);
- if (closure->mIsKernel) {
- MTLaunchStructForEach mtls;
- si->forEachKernelSetup(funcID->mSlot, &mtls);
- cc = new CPUClosure(closure, si, (ExpandFuncTy)mtls.kernel);
- } else {
- cc = new CPUClosure(closure, si);
- }
- if (batch->conflict(cc)) {
- mBatches.push_back(batch);
- std::stringstream ss;
- ss << "Batch" << ++i;
- std::string batchStr(ss.str());
- batch = new Batch(this, batchStr.c_str());
- }
- batch->mClosures.push_back(cc);
- }
- rsAssert(!batch->mClosures.empty());
- mBatches.push_back(batch);
- #ifndef RS_COMPATIBILITY_LIB
- compile(mGroup->mCacheDir);
- if (mScriptObj != nullptr && mExecutable != nullptr) {
- for (Batch* batch : mBatches) {
- batch->resolveFuncPtr(mScriptObj);
- }
- }
- #endif
- mCpuRefImpl->unlockMutex();
- }
- void Batch::resolveFuncPtr(void* sharedObj) {
- std::string funcName(mName);
- if (mClosures.front()->mClosure->mIsKernel) {
- funcName.append(".expand");
- }
- mFunc = dlsym(sharedObj, funcName.c_str());
- rsAssert (mFunc != nullptr);
- }
- CpuScriptGroup2Impl::~CpuScriptGroup2Impl() {
- for (Batch* batch : mBatches) {
- delete batch;
- }
- delete mExecutable;
-
- if (mScriptObj != nullptr) {
- dlclose(mScriptObj);
- }
- }
- namespace {
- #ifndef RS_COMPATIBILITY_LIB
- string getCoreLibPath(Context* context, string* coreLibRelaxedPath) {
- *coreLibRelaxedPath = "";
-
- if (context->getContextType() == RS_CONTEXT_TYPE_DEBUG) {
- return SYSLIBPATH_BC"/libclcore_debug.bc";
- }
-
- #if defined(ARCH_ARM_HAVE_NEON) && !defined(DISABLE_CLCORE_NEON)
-
-
-
-
- *coreLibRelaxedPath = SYSLIBPATH_BC"/libclcore_neon.bc";
- #endif
- #if defined(__i386__) || defined(__x86_64__)
-
- return SYSLIBPATH_BC"/libclcore_x86.bc";
- #else
- return SYSLIBPATH_BC"/libclcore.bc";
- #endif
- }
- void setupCompileArguments(
- const vector<const char*>& inputs, const vector<string>& kernelBatches,
- const vector<string>& invokeBatches,
- const char* outputDir, const char* outputFileName,
- const char* coreLibPath, const char* coreLibRelaxedPath,
- const bool emitGlobalInfo, const bool emitGlobalInfoSkipConstant,
- int optLevel, vector<const char*>* args) {
- args->push_back(RsdCpuScriptImpl::BCC_EXE_PATH);
- args->push_back("-fPIC");
- args->push_back("-embedRSInfo");
- if (emitGlobalInfo) {
- args->push_back("-rs-global-info");
- if (emitGlobalInfoSkipConstant) {
- args->push_back("-rs-global-info-skip-constant");
- }
- }
- args->push_back("-mtriple");
- args->push_back(DEFAULT_TARGET_TRIPLE_STRING);
- args->push_back("-bclib");
- args->push_back(coreLibPath);
- args->push_back("-bclib_relaxed");
- args->push_back(coreLibRelaxedPath);
- for (const char* input : inputs) {
- args->push_back(input);
- }
- for (const string& batch : kernelBatches) {
- args->push_back("-merge");
- args->push_back(batch.c_str());
- }
- for (const string& batch : invokeBatches) {
- args->push_back("-invoke");
- args->push_back(batch.c_str());
- }
- args->push_back("-output_path");
- args->push_back(outputDir);
- args->push_back("-O");
- switch (optLevel) {
- case 0:
- args->push_back("0");
- break;
- case 3:
- args->push_back("3");
- break;
- default:
- ALOGW("Expected optimization level of 0 or 3. Received %d", optLevel);
- args->push_back("3");
- break;
- }
-
-
- args->push_back("-o");
- args->push_back(outputFileName);
- }
- void generateSourceSlot(RsdCpuReferenceImpl* ctxt,
- const Closure& closure,
- const std::vector<const char*>& inputs,
- std::stringstream& ss) {
- const IDBase* funcID = (const IDBase*)closure.mFunctionID.get();
- const Script* script = funcID->mScript;
- rsAssert (!script->isIntrinsic());
- const RsdCpuScriptImpl *cpuScript =
- (const RsdCpuScriptImpl *)ctxt->lookupScript(script);
- const string& bitcodeFilename = cpuScript->getBitcodeFilePath();
- const int index = find(inputs.begin(), inputs.end(), bitcodeFilename) -
- inputs.begin();
- ss << index << "," << funcID->mSlot << ".";
- }
- #endif
- }
- extern __attribute__((noinline)) __attribute__((weak))
- void debugHintScriptGroup2(const char* groupName,
- const uint32_t groupNameSize,
- const ExpandFuncTy* kernel,
- const uint32_t kernelCount) {
- ALOGV("group name: %d:%s\n", groupNameSize, groupName);
- for (uint32_t i=0; i < kernelCount; ++i) {
- const char* f1 = (const char*)(kernel[i]);
- __asm__ __volatile__("");
- ALOGV(" closure: %p\n", (const void*)f1);
- }
-
- return;
- }
- void CpuScriptGroup2Impl::compile(const char* cacheDir) {
- #ifndef RS_COMPATIBILITY_LIB
- if (mGroup->mClosures.size() < 2) {
- return;
- }
- const int optLevel = getCpuRefImpl()->getContext()->getOptLevel();
- if (optLevel == 0) {
- std::vector<ExpandFuncTy> kernels;
- for (const Batch* b : mBatches)
- for (const CPUClosure* c : b->mClosures)
- kernels.push_back(c->mFunc);
- if (kernels.size()) {
-
- debugHintScriptGroup2(mGroup->mName,
- strlen(mGroup->mName),
- kernels.data(),
- kernels.size());
- }
-
-
- return;
- }
- auto comparator = [](const char* str1, const char* str2) -> bool {
- return strcmp(str1, str2) < 0;
- };
- std::set<const char*, decltype(comparator)> inputSet(comparator);
- for (Closure* closure : mGroup->mClosures) {
- const Script* script = closure->mFunctionID.get()->mScript;
-
- if (script->isIntrinsic()) {
- return;
- }
- const RsdCpuScriptImpl *cpuScript =
- (const RsdCpuScriptImpl *)mCpuRefImpl->lookupScript(script);
- const char* bitcodeFilename = cpuScript->getBitcodeFilePath();
- inputSet.insert(bitcodeFilename);
- }
- std::vector<const char*> inputs(inputSet.begin(), inputSet.end());
- std::vector<string> kernelBatches;
- std::vector<string> invokeBatches;
- int i = 0;
- for (const auto& batch : mBatches) {
- rsAssert(batch->size() > 0);
- std::stringstream ss;
- ss << batch->mName << ":";
- if (!batch->mClosures.front()->mClosure->mIsKernel) {
- rsAssert(batch->size() == 1);
- generateSourceSlot(mCpuRefImpl, *batch->mClosures.front()->mClosure, inputs, ss);
- invokeBatches.push_back(ss.str());
- } else {
- for (const auto& cpuClosure : batch->mClosures) {
- generateSourceSlot(mCpuRefImpl, *cpuClosure->mClosure, inputs, ss);
- }
- kernelBatches.push_back(ss.str());
- }
- }
- rsAssert(cacheDir != nullptr);
- string objFilePath(cacheDir);
- objFilePath.append("/");
- objFilePath.append(mGroup->mName);
- objFilePath.append(".o");
- const char* resName = mGroup->mName;
- string coreLibRelaxedPath;
- const string& coreLibPath = getCoreLibPath(getCpuRefImpl()->getContext(),
- &coreLibRelaxedPath);
- vector<const char*> arguments;
- bool emitGlobalInfo = getCpuRefImpl()->getEmbedGlobalInfo();
- bool emitGlobalInfoSkipConstant = getCpuRefImpl()->getEmbedGlobalInfoSkipConstant();
- setupCompileArguments(inputs, kernelBatches, invokeBatches, cacheDir,
- resName, coreLibPath.c_str(), coreLibRelaxedPath.c_str(),
- emitGlobalInfo, emitGlobalInfoSkipConstant,
- optLevel, &arguments);
- std::unique_ptr<const char> cmdLine(rsuJoinStrings(arguments.size() - 1,
- arguments.data()));
- inputs.push_back(coreLibPath.c_str());
- inputs.push_back(coreLibRelaxedPath.c_str());
- uint32_t checksum = constructBuildChecksum(nullptr, 0, cmdLine.get(),
- inputs.data(), inputs.size());
- if (checksum == 0) {
- return;
- }
- std::stringstream ss;
- ss << std::hex << checksum;
- std::string checksumStr(ss.str());
-
-
-
- bool alreadyLoaded = false;
- std::string cloneName;
- const bool useRSDebugContext =
- (mCpuRefImpl->getContext()->getContextType() == RS_CONTEXT_TYPE_DEBUG);
- const bool reuse = !is_force_recompile() && !useRSDebugContext;
- if (reuse) {
- mScriptObj = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName, nullptr,
- &alreadyLoaded);
- }
- if (mScriptObj != nullptr) {
-
-
- mExecutable = ScriptExecutable::createFromSharedObject(
- mScriptObj, checksum);
- if (mExecutable != nullptr) {
-
-
- return;
- }
- ALOGV("Failed to create an executable object from so file due to "
- "mismatching checksum");
- if (alreadyLoaded) {
-
-
-
- cloneName.append(resName);
- cloneName.append("#");
- cloneName.append(SharedLibraryUtils::getRandomString(6).c_str());
-
- arguments.pop_back();
- arguments.push_back(cloneName.c_str());
- }
- dlclose(mScriptObj);
- mScriptObj = nullptr;
- }
-
-
-
- arguments.push_back("-build-checksum");
- arguments.push_back(checksumStr.c_str());
- arguments.push_back(nullptr);
- bool compiled = rsuExecuteCommand(RsdCpuScriptImpl::BCC_EXE_PATH,
- arguments.size()-1,
- arguments.data());
- if (!compiled) {
- return;
- }
-
-
-
- std::string SOPath;
- if (!SharedLibraryUtils::createSharedLibrary(
- getCpuRefImpl()->getContext()->getDriverName(), cacheDir, resName,
- reuse, &SOPath)) {
- ALOGE("Failed to link object file '%s'", resName);
- unlink(objFilePath.c_str());
- return;
- }
- unlink(objFilePath.c_str());
- if (reuse) {
- mScriptObj = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName);
- } else {
- mScriptObj = SharedLibraryUtils::loadAndDeleteSharedLibrary(SOPath.c_str());
- }
- if (mScriptObj == nullptr) {
- ALOGE("Unable to load '%s'", resName);
- return;
- }
- if (alreadyLoaded) {
-
-
- string cloneFilePath(cacheDir);
- cloneFilePath.append("/");
- cloneFilePath.append(cloneName.c_str());
- cloneFilePath.append(".so");
- unlink(cloneFilePath.c_str());
- }
- mExecutable = ScriptExecutable::createFromSharedObject(mScriptObj);
- #endif
- }
- void CpuScriptGroup2Impl::execute() {
- for (auto batch : mBatches) {
- batch->setGlobalsForBatch();
- batch->run();
- }
- }
- void Batch::setGlobalsForBatch() {
- for (CPUClosure* cpuClosure : mClosures) {
- const Closure* closure = cpuClosure->mClosure;
- const IDBase* funcID = closure->mFunctionID.get();
- Script* s = funcID->mScript;;
- for (const auto& p : closure->mGlobals) {
- const int64_t value = p.second.first;
- int size = p.second.second;
- if (value == 0 && size == 0) {
-
-
-
-
- continue;
- }
- rsAssert(p.first != nullptr);
- Script* script = p.first->mScript;
- rsAssert(script == s);
- RsdCpuReferenceImpl* ctxt = mGroup->getCpuRefImpl();
- const RsdCpuScriptImpl *cpuScript =
- (const RsdCpuScriptImpl *)ctxt->lookupScript(script);
- int slot = p.first->mSlot;
- ScriptExecutable* exec = mGroup->getExecutable();
- if (exec != nullptr) {
- const char* varName = cpuScript->getFieldName(slot);
- void* addr = exec->getFieldAddress(varName);
- if (size < 0) {
- rsrSetObject(mGroup->getCpuRefImpl()->getContext(),
- (rs_object_base*)addr, (ObjectBase*)value);
- } else {
- memcpy(addr, (const void*)&value, size);
- }
- } else {
-
- if (size < 0) {
- s->setVarObj(slot, (ObjectBase*)value);
- } else {
- s->setVar(slot, (const void*)&value, size);
- }
- }
- }
- }
- }
- void Batch::run() {
- if (!mClosures.front()->mClosure->mIsKernel) {
- rsAssert(mClosures.size() == 1);
-
- CPUClosure* cc = mClosures.front();
- const Closure* c = cc->mClosure;
- if (mFunc != nullptr) {
-
-
- ((InvokeFuncTy)mFunc)(c->mParams, c->mParamLength);
- } else {
- const ScriptInvokeID* invokeID = (const ScriptInvokeID*)c->mFunctionID.get();
- rsAssert(invokeID != nullptr);
- cc->mSi->invokeFunction(invokeID->mSlot, c->mParams, c->mParamLength);
- }
- return;
- }
- if (mFunc != nullptr) {
- MTLaunchStructForEach mtls;
- const CPUClosure* firstCpuClosure = mClosures.front();
- const CPUClosure* lastCpuClosure = mClosures.back();
- firstCpuClosure->mSi->forEachMtlsSetup(
- (const Allocation**)firstCpuClosure->mClosure->mArgs,
- firstCpuClosure->mClosure->mNumArg,
- lastCpuClosure->mClosure->mReturnValue,
- nullptr, 0, nullptr, &mtls);
- mtls.script = nullptr;
- mtls.fep.usr = nullptr;
- mtls.kernel = (ForEachFunc_t)mFunc;
- mGroup->getCpuRefImpl()->launchForEach(
- (const Allocation**)firstCpuClosure->mClosure->mArgs,
- firstCpuClosure->mClosure->mNumArg,
- lastCpuClosure->mClosure->mReturnValue,
- nullptr, &mtls);
- return;
- }
- for (CPUClosure* cpuClosure : mClosures) {
- const Closure* closure = cpuClosure->mClosure;
- const ScriptKernelID* kernelID =
- (const ScriptKernelID*)closure->mFunctionID.get();
- cpuClosure->mSi->preLaunch(kernelID->mSlot,
- (const Allocation**)closure->mArgs,
- closure->mNumArg, closure->mReturnValue,
- nullptr, 0, nullptr);
- }
- const CPUClosure* cpuClosure = mClosures.front();
- const Closure* closure = cpuClosure->mClosure;
- MTLaunchStructForEach mtls;
- if (cpuClosure->mSi->forEachMtlsSetup((const Allocation**)closure->mArgs,
- closure->mNumArg,
- closure->mReturnValue,
- nullptr, 0, nullptr, &mtls)) {
- mtls.script = nullptr;
- mtls.kernel = &groupRoot;
- mtls.fep.usr = &mClosures;
- mGroup->getCpuRefImpl()->launchForEach(nullptr, 0, nullptr, nullptr, &mtls);
- }
- for (CPUClosure* cpuClosure : mClosures) {
- const Closure* closure = cpuClosure->mClosure;
- const ScriptKernelID* kernelID =
- (const ScriptKernelID*)closure->mFunctionID.get();
- cpuClosure->mSi->postLaunch(kernelID->mSlot,
- (const Allocation**)closure->mArgs,
- closure->mNumArg, closure->mReturnValue,
- nullptr, 0, nullptr);
- }
- }
- }
- }
|