-
Codec創建流程
Android提供給應用編解碼的介面為MediaCodec，我們這里從NuPlayerDecoder開始分析，一是為了銜接之前將的MediaPlayer-NuPlayer流程，二是可以從中參考到MediaCodec是怎么用的，然后去分析內部流程會更好，
之前說到NuPlayer.cpp創建Decoder流程（Video部分）：instantiateDecoder
1965 sp<AMessage> notify = new AMessage(kWhatVideoNotify, this);
1966 ++mVideoDecoderGeneration;
1967 notify->setInt32("generation", mVideoDecoderGeneration);
1968
1969 *decoder = new Decoder(
1970 notify, mSource, mPID, mUID, mRenderer, mSurface, mCCDecoder);
1971 mVideoDecoderError = false;NuPlayer::Decoder是繼承于DecoderBase的，不過DecoderBase主要就是用于處理一些基本的流程（setParameter、start、stop等），如果Decoder重寫了就更不需要看了，Decoder建構式中主要是創建了一個looper，每一個Decoder都需要有一個looper，因為MediaCodec是阻塞式的呼叫，但是NuPlayer需要的是異步操作:
Decoder::Decoder
94 mCodecLooper = new ALooper;
95 mCodecLooper->setName("NPDecoder-CL");
96 mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
97 mVideoTemporalLayerAggregateFps[0] = mFrameRateTotal;再回來init：NuPlayer.cpp::instantiateDecoder
2032 (*decoder)->init();
2042 (*decoder)->configure(format);Init()其實主要就是注冊handler，主要看下configure，DecoderBase的未重寫函式，最終發送kWhatConfigure，呼叫NuPlayer::Decoder::onConfigure：
293 AString mime;
294 CHECK(format->findString("mime", &mime));
295
296 mIsAudio = !strncasecmp("audio/", mime.c_str(), 6);
297 mIsVideoAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
298
299 mComponentName = mime;
300 mComponentName.append(" decoder");
301 ALOGV("[%s] onConfigure (surface=%p)", mComponentName.c_str(), mSurface.get());
302
303 mCodec = MediaCodec::CreateByType(
304 mCodecLooper, mime.c_str(), false /* encoder */, NULL /* err */, mPid, mUid);
305 int32_t secure = 0;
306 if (format->findInt32("secure", &secure) && secure != 0) {
307 if (mCodec != NULL) {
308 mCodec->getName(&mComponentName);
309 mComponentName.append(".secure");
310 mCodec->release();
311 ALOGI("[%s] creating", mComponentName.c_str());
312 mCodec = MediaCodec::CreateByComponentName(
313 mCodecLooper, mComponentName.c_str(), NULL /* err */, mPid, mUid);
314 }
315 }可以看到MediaCodec創建有兩種方式
- CreateByType
- CreateByComponentName
CreateByType是通過mime創建，而CreateByComponentName是通過組件名創建，當format的secure是1的時候會獲取之前mime創建mCodec的組件名，并釋放，然后在組件名后拼接secure，然后再通過CreateByComponentName創建codec，Secure型別比較典型的就是youtube視頻，（組件名一般為OMX.qcom.video.decoder.avc這種，對應media_codec.xml中），
再看一下后面幾個重要的呼叫：
NuPlayer::Decoder::onConfigure：
360 err = mCodec->configure(
361 format, mSurface, crypto, 0 /* flags */);
//新建一個AMessage傳給mediacodec，當mediacodec做好一些處理（如編碼一幀）就把buffer index、flag、size等傳入msg，然后post回來，相當于一個callback，
388 sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
389 mCodec->setCallback(reply);
390
391 err = mCodec->start();然后看下MediaCodec對應的處理吧：
CreateByType
439 sp<MediaCodec> MediaCodec::CreateByType(
440 const sp<ALooper> &looper, const AString &mime, bool encoder, status_t *err, pid_t pid,
441 uid_t uid) {
442 Vector<AString> matchingCodecs;
443
444 MediaCodecList::findMatchingCodecs(
445 mime.c_str(),
446 encoder,
447 0,
448 &matchingCodecs);
449
450 if (err != NULL) {
451 *err = NAME_NOT_FOUND;
452 }
453 for (size_t i = 0; i < matchingCodecs.size(); ++i) {
454 sp<MediaCodec> codec = new MediaCodec(looper, pid, uid);
455 AString componentName = matchingCodecs[i];
456 status_t ret = codec->init(componentName);
457 if (err != NULL) {
458 *err = ret;
459 }
460 if (ret == OK) {
461 return codec;
462 }
463 ALOGD("Allocating component '%s' failed (%d), try next one.",
464 componentName.c_str(), ret);
465 }
466 return NULL;
467 }CreateByComponentName
470 sp<MediaCodec> MediaCodec::CreateByComponentName(
471 const sp<ALooper> &looper, const AString &name, status_t *err, pid_t pid, uid_t uid) {
472 sp<MediaCodec> codec = new MediaCodec(looper, pid, uid);
473
474 const status_t ret = codec->init(name);
475 if (err != NULL) {
476 *err = ret;
477 }
478 return ret == OK ? codec : NULL; // NULL deallocates codec.
479 }findMatchingCodecs其實就是從所有的mediaCodec陣列中選取匹配mime，mediaCodec陣列是加載MediaCodecList的時候從media_codecs.xml中讀取到的，具體流程放在Extend中跟蹤，
CreateByComponentName和CreateByType最終都是new了MediaCodec，呼叫init傳入組件名
887 status_t MediaCodec::init(const AString &name) {
906 const sp<IMediaCodecList> mcl = MediaCodecList::getInstance();
...
911 for (const AString &codecName : { name, tmp }) {
912 ssize_t codecIdx = mcl->findCodecByName(codecName.c_str());
...
916 mCodecInfo = mcl->getCodecInfo(codecIdx);
917 Vector<AString> mediaTypes;
918 mCodecInfo->getSupportedMediaTypes(&mediaTypes);
...
//根據組件名的開頭選擇CodecBase，如omx.則為ACodec，C2則為CCodec，
931 mCodec = GetCodecBase(name, mCodecInfo->getOwnerName());
//傳給ACodec兩個callback，引數都為AMessage，what為kWhatCodecNotify，用于ACodec訊息回呼，
//CodecCallback主要作用為狀態回呼，BufferChannel主要作用為buffer狀態的回呼，
951 mCodec->setCallback(
952 std::unique_ptr<CodecBase::CodecCallback>(
953 new CodecCallback(new AMessage(kWhatCodecNotify, this))));
954 mBufferChannel = mCodec->getBufferChannel();
955 mBufferChannel->setCallback(
956 std::unique_ptr<CodecBase::BufferCallback>(
957 new BufferCallback(new AMessage(kWhatCodecNotify,
this))));
...
959 sp<AMessage> msg = new AMessage(kWhatInit, this);
986 err = PostAndAwaitResponse(msg, &response);看一下kWhatInit的訊息處理：
2355 case kWhatInit:
2377 mCodec->initiateAllocateComponent(format);-
OMX組件創建
上面提到了ACodec/CCodec/MediaFilter都是GetCodecBase通過組件名來創建的，這里拿ACodec當例子，如下為ACodec的建構式，可以看到ACodec創建了九種狀態機，ACodec在不同階段會通過changeState切換不同的狀態，用于管理底層各個Component的各種狀態，理解狀態機也能幫忙我們理解ACodec作業流程，這里可以看到初始化后狀態為mUninitializedState，具體每一個狀態機的相互切換留在寫資料流的時候再寫，
581
582 mUninitializedState = new UninitializedState(this);
583 mLoadedState = new LoadedState(this);
584 mLoadedToIdleState = new LoadedToIdleState(this);
585 mIdleToExecutingState = new IdleToExecutingState(this);
586 mExecutingState = new ExecutingState(this);
587
588 mOutputPortSettingsChangedState =
589 new OutputPortSettingsChangedState(this);
590
591 mExecutingToIdleState = new ExecutingToIdleState(this);
592 mIdleToLoadedState = new IdleToLoadedState(this);
593 mFlushingState = new FlushingState(this);
594
595 mPortEOS[kPortIndexInput] = mPortEOS[kPortIndexOutput] = false;
596 mInputEOSResult = OK;
597
598 mPortMode[kPortIndexInput] = IOMX::kPortModePresetByteBuffer;
599 mPortMode[kPortIndexOutput] = IOMX::kPortModePresetByteBuffer;
600
601 memset(&mLastNativeWindowCrop, 0, sizeof(mLastNativeWindowCrop));
602
603 changeState(mUninitializedState);之后呼叫initiateAllocateComponent，看下代碼：
634 void ACodec::initiateAllocateComponent(const sp<AMessage> &msg) {
635 msg->setWhat(kWhatAllocateComponent);
636 msg->setTarget(this);
637 msg->post();
6653 case ACodec::kWhatAllocateComponent:
6654 {
6655 onAllocateComponent(msg);
6656 handled = true;
6657 break;
6705 bool ACodec::UninitializedState::onAllocateComponent(const sp<AMessage> &msg){
6710 sp<AMessage> notify = new AMessage(kWhatOMXMessageList, mCodec);
6724 CHECK(msg->findString("componentName", &componentName));
6726 sp<CodecObserver> observer = new CodecObserver(notify);
6731 OMXClient client;
6732 if (client.connect(owner.c_str()) != OK) {
6733 mCodec->signalError(OMX_ErrorUndefined, NO_INIT);
6734 return false;
6735 }
6736 omx = client.interface();
//提高執行緒調度優先級
6739 int prevPriority = androidGetThreadPriority(tid);
6740 androidSetThreadPriority(tid, ANDROID_PRIORITY_FOREGROUND);
6741 err = omx->allocateNode(componentName.c_str(), observer, &omxNode);
6742 androidSetThreadPriority(tid, prevPriority);
//通過CodecCallback呼叫MediaCodec的onComponentAllocated來通知MediaCodec當前狀態
6769 mCodec->mOMX = omx;
6770 mCodec->mOMXNode = omxNode;
6771 mCodec->mCallback->onComponentAllocated(mCodec->mComponentName.c_str());接下來主要看下OMX的allocateNode，OMX物件是通過client.interface()獲取到的，首先看下OMX是怎么起來的，如下代碼可以看到OMX服務是跑在MediaCodec行程中，很標準的HIDL實作，hal介面定義在hardware/hardware/interfaces/media/omx/1.0中，
main_codecservice.cpp
51 using namespace ::android::hardware::media::omx::V1_0;
52 sp<IOmx> omx = new implementation::Omx();
55 } else if (omx->registerAsService() != OK) {
60 sp<IOmxStore> omxStore = new implementation::OmxStore(omx);
63 } else if (omxStore->registerAsService() != OK) {OmxStore和Omx建構式除了加載mediacodec xml，主要還new了OMXMaster，OMXMaster比較重要的就是添加了各種Plugin，Plugin如其名，就是加載OMX具體組件的插件，一般android原生都是加載soft plugin，各大平臺廠商在實作自己硬編解碼的時候也會添加自己的plugin，
32 OMXMaster::OMXMaster() {
...
53 addVendorPlugin();
54 addPlatformPlugin();
55 }
61 void OMXMaster::addVendorPlugin() {
62 addPlugin("libstagefrighthw.so");
63 }
64
65 void OMXMaster::addPlatformPlugin() {
66 addPlugin("libstagefright_softomx_plugin.so");
67 }
69 void OMXMaster::addPlugin(const char *libname) {
70 void *libHandle = android_load_sphal_library(libname, RTLD_NOW);
71
72 if (libHandle == NULL) {
73 return;
74 }
75
76 typedef OMXPluginBase *(*CreateOMXPluginFunc)();
77 CreateOMXPluginFunc createOMXPlugin =
78 (CreateOMXPluginFunc)dlsym(
79 libHandle, "createOMXPlugin");
80 if (!createOMXPlugin)
81 createOMXPlugin = (CreateOMXPluginFunc)dlsym(
82 libHandle, "_ZN7android15createOMXPluginEv");
83
84 OMXPluginBase *plugin = nullptr;
85 if (createOMXPlugin) {
86 plugin = (*createOMXPlugin)();
87 }
88
89 if (plugin) {
90 mPlugins.push_back({ plugin, libHandle });
91 addPlugin(plugin);
92 } else {
93 android_unload_sphal_library(libHandle);
94 }
95 }
97 void OMXMaster::addPlugin(OMXPluginBase *plugin) {
98 Mutex::Autolock autoLock(mLock);
104 while ((err = plugin->enumerateComponents(
105 name, sizeof(name), index++)) == OMX_ErrorNone) {
108 if (mPluginByComponentName.indexOfKey(name8) >= 0) {
112 continue;
113 }
115 mPluginByComponentName.add(name8, plugin);
116 }回到ACodec，之前呼叫了omx->allocateNode(componentName.c_str(), observer, &omxNode);
84 Return<void> Omx::allocateNode(
85 const hidl_string& name,
86 const sp<IOmxObserver>& observer,
87 allocateNode_cb _hidl_cb) {
//OMXNodeInstance相當于組件的句柄，物件需要傳回給ACodec，
//component就是通過Instance回呼一些Codec函式，傳回一些狀態以及buffer資訊，
100 instance = new OMXNodeInstance(
101 this, new LWOmxObserver(observer), name.c_str());
//呼叫OmxMaster::makeComponentInstance
104 OMX_ERRORTYPE err = mMaster->makeComponentInstance(
105 name.c_str(), &OMXNodeInstance::kCallbacks,
106 instance.get(), &handle);
116 instance->setHandle(handle);
//傳給ACodec instance
144 _hidl_cb(toStatus(OK), new TWOmxNode(instance));
}
146 OMX_ERRORTYPE OMXMaster::makeComponentInstance(
147 const char *name,
148 const OMX_CALLBACKTYPE *callbacks,
149 OMX_PTR appData,
150 OMX_COMPONENTTYPE **component) {
//mPluginByComponentName是之前加載plugin存入的
156 ssize_t index = mPluginByComponentName.indexOfKey(String8(name));
162 OMXPluginBase *plugin = mPluginByComponentName.valueAt(index);
//呼叫plugin中makeComponentInstance
163 OMX_ERRORTYPE err =
164 plugin->makeComponentInstance(name, callbacks, appData, component);
170 mPluginByInstance.add(*component, plugin);很多平臺廠商的plugin檔案的實際函式邏輯都在so檔案中，無法查看，這里以Soft Plugin作為例子：
87 OMX_ERRORTYPE SoftOMXPlugin::makeComponentInstance(
88 const char *name,
89 const OMX_CALLBACKTYPE *callbacks,
90 OMX_PTR appData,
91 OMX_COMPONENTTYPE **component) {
94 for (size_t i = 0; i < kNumComponents; ++i) {
95 if (strcmp(name, kComponents[i].mName)) {
96 continue;
97 }
//libstagefright_soft_拼接上組件名
99 AString libName = "libstagefright_soft_";
100 libName.append(kComponents[i].mLibNameSuffix);
101 libName.append(".so");
//dlopen打開軟編解碼器so
117 void *libHandle = dlopen(libName.c_str(), RTLD_NOW|RTLD_NODELETE);
129 CreateSoftOMXComponentFunc createSoftOMXComponent =
130 (CreateSoftOMXComponentFunc)dlsym(
131 libHandle,
132 "_Z22createSoftOMXComponentPKcPK16OMX_CALLBACKTYPE"
133 "PvPP17OMX_COMPONENTTYPE");
142 sp<SoftOMXComponent> codec =
143 (*createSoftOMXComponent)(name, callbacks, appData, component);
160 codec->incStrong(this);
161 codec->setLibHandle(libHandle);以libstagefright_soft_m4vh263dec.so為例
415 android::SoftOMXComponent *createSoftOMXComponent(
416 const char *name, const OMX_CALLBACKTYPE *callbacks,
417 OMX_PTR appData, OMX_COMPONENTTYPE **component) {
418 using namespace android;
419 if (!strcmp(name, "OMX.google.h263.decoder")) {
420 return new android::SoftMPEG4(
421 name, "video_decoder.h263", OMX_VIDEO_CodingH263,
422 kH263ProfileLevels, ARRAY_SIZE(kH263ProfileLevels),
423 callbacks, appData, component);
424 } else if (!strcmp(name, "OMX.google.mpeg4.decoder")) {
425 return new android::SoftMPEG4(
426 name, "video_decoder.mpeg4", OMX_VIDEO_CodingMPEG4,
427 kM4VProfileLevels, ARRAY_SIZE(kM4VProfileLevels),
428 callbacks, appData, component);
429 } else {
430 CHECK(!"Unknown component");
431 }
432 return NULL;
433 }最終根據傳入組件名創建相應解碼器SoftMPEG4,看了下SoftMPEG4繼承于SoftVideoDecoderOMXComponent，而SoftVideoDecoderOMXComponent又繼承于SimpleSoftOMXComponent，所以emptyThisBuffer、fillThisBuffer等omx基本方法，都在SimpleSoftOMXComponent中定義，而SoftMPEG4沒有重寫，至此omx組件創建完成，后續章節通過OMX各個狀態的切換以及ACodec對應狀態機的切換來繼續分析資料流走向，同時會整理omx框架每個部分的關連，
至于MediaFilter(濾波器)、Codec2，大體流程跟ACodec都差不多，不過多分析，Codec2在Q版本及之前基本都是用于管理軟編解碼器，R版本應該會兼容硬編解碼器，大概率會完全替換OMX框架，補一張整體流程圖：
Extend
MediaCodecList加載xml
void MediaCodecList::findMatchingCodecs(
346 const char *mime, bool encoder, uint32_t flags,
347 Vector<AString> *matches) {
348 matches->clear();
349
350 const sp<IMediaCodecList> list = getInstance();
351 if (list == nullptr) {
352 return;
353 }
354
355 size_t index = 0;
356 for (;;) {
357 ssize_t matchIndex =
358 list->findCodecByType(mime, encoder, index);
359
360 if (matchIndex < 0) {
361 break;
362 }
363
364 index = matchIndex + 1;
365
366 const sp<MediaCodecInfo> info = list->getCodecInfo(matchIndex);
367 CHECK(info != nullptr);
368 AString componentName = info->getCodecName();
369
370 if ((flags & kHardwareCodecsOnly) && isSoftwareCodec(componentName)) {
371 ALOGV("skipping SW codec '%s'", componentName.c_str());
372 } else {
373 matches->push(componentName);
374 ALOGV("matching '%s'", componentName.c_str());
375 }
376 }
377 //所以更改軟硬編解碼優先可以在這修改
378 if (flags & kPreferSoftwareCodecs ||
379 property_get_bool("debug.stagefright.swcodec", false)) {
380 matches->sort(compareSoftwareCodecsFirst);
381 }
382 }
383 getInstance的流程跟起來最后都是到了getLocalInstance:
143 sp<IMediaCodecList> MediaCodecList::getLocalInstance() {
MediaCodecList *codecList = new MediaCodecList(GetBuilders());主要就是new了MediaCodecList，關鍵是引數GetBuilders（），先看omx的話，MediaCodecList中的builder就是OmxInfoBuilder ，
80 OmxInfoBuilder sOmxInfoBuilder{true /* allowSurfaceEncoders */};
95 std::vector<MediaCodecListBuilderBase *> GetBuilders() {
96 std::vector<MediaCodecListBuilderBase *> builders;
97 // if plugin provides the input surface, we cannot use OMX video encoders.
98 // In this case, rely on plugin to provide list of OMX codecs that are usable.
99 sp<PersistentSurface> surfaceTest =
100 StagefrightPluginLoader::GetCCodecInstance()->createInputSurface();
101 if (surfaceTest == nullptr) {
102 ALOGD("Allowing all OMX codecs");
103 builders.push_back(&sOmxInfoBuilder);
104 } else {
105 ALOGD("Allowing only non-surface-encoder OMX codecs");
106 builders.push_back(&sOmxNoSurfaceEncoderInfoBuilder);
107 }
108 builders.push_back(GetCodec2InfoBuilder());
109 return builders;
110 }看一下MediaCodecList的建構式：
203 MediaCodecList::MediaCodecList(std::vector<MediaCodecListBuilderBase*> builders) {
204 mGlobalSettings = new AMessage();
205 mCodecInfos.clear();
206 MediaCodecListWriter writer;
207 for (MediaCodecListBuilderBase *builder : builders) {
208 if (builder == nullptr) {
209 ALOGD("ignored a null builder");
210 continue;
211 }
212 mInitCheck = builder->buildMediaCodecList(&writer);
213 if (mInitCheck != OK) {
214 break;
215 }
216 }
217 writer.writeGlobalSettings(mGlobalSettings);
218 writer.writeCodecInfos(&mCodecInfos);
219 std::stable_sort(
220 mCodecInfos.begin(),
221 mCodecInfos.end(),
222 [](const sp<MediaCodecInfo> &info1, const sp<MediaCodecInfo> &info2) {
223 // null is lowest
224 return info1 == nullptr
225 || (info2 != nullptr && info1->getRank() < info2->getRank());
226 });
227
228 // remove duplicate entries
229 bool dedupe = property_get_bool("debug.stagefright.dedupe-codecs", true);
230 if (dedupe) {
231 std::set<std::string> codecsSeen;
232 for (auto it = mCodecInfos.begin(); it != mCodecInfos.end(); ) {
233 std::string codecName = (*it)->getCodecName();
234 if (codecsSeen.count(codecName) == 0) {
235 codecsSeen.emplace(codecName);
236 it++;
237 } else {
238 it = mCodecInfos.erase(it);
239 }
240 }
241 }
242
================================ buildMediaCodecList ================================
94 status_t OmxInfoBuilder::buildMediaCodecList(MediaCodecListWriter* writer) {
95 // Obtain IOmxStore
96 sp<IOmxStore> omxStore = IOmxStore::getService();
97 if (omxStore == nullptr) {
98 ALOGE("Cannot find an IOmxStore service.");
99 return NO_INIT;
100 }
101
102 // List service attributes (global settings)
103 Status status;
104 hidl_vec<IOmxStore::RoleInfo> roles;
105 auto transStatus = omxStore->listRoles(
106 [&roles] (
107 const hidl_vec<IOmxStore::RoleInfo>& inRoleList) {
108 roles = inRoleList;
109 });
================================ OmxStore::listRoles ================================
136 Return<void> OmxStore::listRoles(listRoles_cb _hidl_cb) {
137 _hidl_cb(mRoleList);
138 return Void();
139 }
================================ mRoleList： ================================
38 OmxStore::OmxStore(
39 const sp<IOmx> &omx,
40 const char* owner,
41 const std::vector<std::string> &searchDirs,
42 const std::vector<std::string> &xmlNames,
43 const char* profilingResultsXmlPath) {
44 // retrieve list of omx nodes 57 MediaCodecsXmlParser parser;
58 parser.parseXmlFilesInSearchDirs(xmlNames, searchDirs);
59 if (profilingResultsXmlPath != nullptr) {
60 parser.parseXmlPath(profilingResultsXmlPath);
61 }
75 const auto& roleMap = parser.getRoleMap(); for (const auto& rolePair : roleMap) {
79 RoleInfo role;
80 role.role = rolePair.first;
81 role.type = rolePair.second.type;
82 role.isEncoder = rolePair.second.isEncoder;
112 mRoleList[i] = std::move(role);
OmxStore解構式引數在頭檔案里寫死的：
================================ OmxStore.h ================================
46 OmxStore(
47 const sp<IOmx> &omx = nullptr,
48 const char* owner = "default",
49 const std::vector<std::string> &searchDirs =
50 MediaCodecsXmlParser::getDefaultSearchDirs(),
51 const std::vector<std::string> &xmlFiles =
52 MediaCodecsXmlParser::getDefaultXmlNames(),
53 const char *xmlProfilingResultsPath =
54 MediaCodecsXmlParser::defaultProfilingResultsXmlPath);
================================ MediaCodecsXmlParser ================================
37 static std::vector<std::string> getDefaultSearchDirs() {
38 return { "/odm/etc", "/vendor/etc", "/etc" };
39 }
40 static std::vector<std::string> getDefaultXmlNames() {
41 return { "media_codecs.xml", "media_codecs_performance.xml" };
42 }
43 static constexpr char const* defaultProfilingResultsXmlPath =
44 "/data/misc/media/media_codecs_profiling_results.xml";
45 轉載請註明出處，本文鏈接：https://www.uj5u.com/qianduan/50895.html
標籤：其他
上一篇：JAVA復習之java語言基礎