需求：在unity中解析4K视频/mpeg4/mp4/rtsp/rtmp/hls视频
场景：需要在unity中实现较为复杂的ui和空间交互的同时，解析播放各种复杂的视频流
问题分析及思路：unity 的MovieTexture和比较常见的插件支持的视频解析功能太弱，个人觉得比较好用的就是 bilibili的 ijkplayer，基于ffmpeg，功能强大性能稳定且开源，是我辈最爱。 如果讲ijkplayer解析出的纹理装载到unity中，即可完美解决问题，解析直播流或是4k视频毫无压力，可以贴到球体/弧面上面轻松实现VR全景播放器等复杂功能和交互。
参考资料：EasyMovieTexture ijkplayer
方案：这个思路中关键点在于把ijkplayer解析出来的纹理传递给unity，所以我们需要一个 TextureId 来传递纹理（这也可以用 Texture.LoadRawTextureData(byte[]传递图像数据），但是这样效率太低），仔细阅读EasyMovieTexture.java这部分工作事实上已经做好了，那么最简单粗暴的方法则是直接找到其中的android.media.MediaPlayer 替换为 tv.danmaku.ijk.media.player.IjkMediaPlayer 即可。
贴上代码：
public class EasyMovieTexture implements IMediaPlayer.OnPreparedListener, IMediaPlayer.OnBufferingUpdateListener, IMediaPlayer.OnCompletionListener, IMediaPlayer.OnErrorListener, OnFrameAvailableListener {
private Activity m_UnityActivity = null;
private IjkMediaPlayer m_MediaPlayer = null;
private int m_iUnityTextureID = -1;
private int m_iSurfaceTextureID = -1;
private SurfaceTexture m_SurfaceTexture = null;
private Surface m_Surface = null;
private int m_iCurrentSeekPercent = 0;
private int m_iCurrentSeekPosition = 0;
public int m_iNativeMgrID;
private String m_strFileName;
private int m_iErrorCode;
private int m_iErrorCodeExtra;
private boolean m_bRockchip = true;
private boolean m_bSplitOBB = false;
private String m_strOBBName;
public boolean m_bUpdate= false;
public static ArrayList<EasyMovieTexture> m_objCtrl = new ArrayList<EasyMovieTexture>();
public static EasyMovieTexture GetObject(int iID)
{
for(int i = 0; i < m_objCtrl.size(); i++)
{
if(m_objCtrl.get(i).m_iNativeMgrID == iID)
{
return m_objCtrl.get(i);
}
}
return null;
}
private static final int GL_TEXTURE_EXTERNAL_OES = 0x8D65;
public native int InitNDK(Object obj);
public native void SetAssetManager(AssetManager assetManager);
public native int InitApplication();
public native void QuitApplication();
public native void SetWindowSize(int iWidth,int iHeight,int iUnityTextureID,boolean bRockchip);
public native void RenderScene(float [] fValue, int iTextureID,int iUnityTextureID);
public native void SetManagerID(int iID);
public native int GetManagerID();
public native int InitExtTexture();
public native void SetUnityTextureID(int iTextureID);
static
{
System.loadLibrary("BlueDoveMediaRender");
}
MEDIAPLAYER_STATE m_iCurrentState = MEDIAPLAYER_STATE.NOT_READY;
public void Destroy()
{
if(m_iSurfaceTextureID != -1)
{
int [] textures = new int[1];
textures[0] = m_iSurfaceTextureID;
GLES20.glDeleteTextures(1, textures, 0);
m_iSurfaceTextureID = -1;
}
SetManagerID(m_iNativeMgrID);
QuitApplication();
m_objCtrl.remove(this);
}
public void UnLoad()
{
if(m_MediaPlayer!=null)
{
if(m_iCurrentState != MEDIAPLAYER_STATE.NOT_READY )
{
try {
m_MediaPlayer.stop();
m_MediaPlayer.release();
} catch (SecurityException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IllegalStateException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
m_MediaPlayer = null;
}
else
{
try {
m_MediaPlayer.release();
} catch (SecurityException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IllegalStateException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
m_MediaPlayer = null;
}
if(m_Surface != null)
{
m_Surface.release();
m_Surface = null;
}
if(m_SurfaceTexture != null)
{
m_SurfaceTexture.release();
m_SurfaceTexture = null;
}
if(m_iSurfaceTextureID != -1)
{
int [] textures = new int[1];
textures[0] = m_iSurfaceTextureID;
GLES20.glDeleteTextures(1, textures, 0);
m_iSurfaceTextureID = -1;
}
}
}
public boolean Load() throws SecurityException, IllegalStateException, IOException
{
UnLoad();
m_iCurrentState = MEDIAPLAYER_STATE.NOT_READY;
m_MediaPlayer = new IjkMediaPlayer();
m_MediaPlayer.setOption(OPT_CATEGORY_PLAYER, "mediacodec", 1);
m_MediaPlayer.setOption(OPT_CATEGORY_PLAYER, "opensles", 1);
m_MediaPlayer.setOption(OPT_CATEGORY_PLAYER, "mediacodec-auto-rotate", 1);
//int mediaHardDecoder = isDeviceSupportHardDecorder()? 1 : 0;
m_MediaPlayer.setOption(IjkMediaPlayer.OPT_CATEGORY_CODEC, "mediacodec", 1);
//player.setOption(IjkMediaPlayer.OPT_CATEGORY_CODEC, "mediacodec-avc", mediaHardDecoder);
m_MediaPlayer.setOption(OPT_CATEGORY_PLAYER, "mediacodec-auto-rotate", 1);
m_MediaPlayer.setOption(OPT_CATEGORY_PLAYER, "overlay-format", IjkMediaPlayer.SDL_FCC_RV32);
m_MediaPlayer.setOption(OPT_CATEGORY_PLAYER, "framedrop", 1);
//mediaPlayer.setOption(IjkMediaPlayer.OPT_CATEGORY_PLAYER, "framedrop", 30);
m_MediaPlayer.setOption(OPT_CATEGORY_PLAYER, "max-fps", 0);
m_MediaPlayer.setOption(IjkMediaPlayer.OPT_CATEGORY_CODEC, "skip_loop_filter", 48);
m_MediaPlayer.setAudioStreamType(AudioManager.STREAM_MUSIC);
m_bUpdate = false;
if(m_strFileName.contains("file://") == true)
{
File sourceFile = new File(m_strFileName.replace("file://", ""));
if ( sourceFile.exists() )
{
FileInputStream fs = new FileInputStream(sourceFile);
//m_MediaPlayer.setDataSource(fs.getFD());
m_MediaPlayer.setDataSource(m_strFileName);
fs.close();
}
}
else if(m_strFileName.contains("://") == true)
{
try {
Map<String, String> headers = new HashMap<String, String>();
headers.put("rtsp_transport", "tcp") ;
headers.put("max_analyze_duration", "1000") ;
//m_MediaPlayer.setDataSource(m_UnityActivity, Uri.parse(m_strFileName), headers);
m_MediaPlayer.setDataSource(m_strFileName);
} catch (IOException e) {
// TODO Auto-generated catch block
Log.e("Unity","Error m_MediaPlayer.setDataSource() : " + m_strFileName);
e.printStackTrace();
m_iCurrentState = MEDIAPLAYER_STATE.ERROR;
return false;
}
}
else
{
if(m_bSplitOBB)
{
try {
ZipResourceFile expansionFile = new ZipResourceFile(m_strOBBName);
Log.e("unity", m_strOBBName + " " + m_strFileName);
AssetFileDescriptor afd = expansionFile.getAssetFileDescriptor("assets/" + m_strFileName);
ZipEntryRO[] data =expansionFile.getAllEntries();
for(int i = 0; i <data.length; i++)
{
Log.e("unity", data[i].mFileName);
}
Log.e("unity", afd + " " );
//m_MediaPlayer.setDataSource(afd.getFileDescriptor(),afd.getStartOffset(),afd.getLength());
//m_MediaPlayer.setDataSource(afd.getFileDescriptor());
m_MediaPlayer.setDataSource(m_strFileName);
} catch (IOException e) {
m_iCurrentState = MEDIAPLAYER_STATE.ERROR;
e.printStackTrace();
return false;
}
}
else
{
AssetFileDescriptor afd;
try {
afd = m_UnityActivity.getAssets().openFd(m_strFileName);
//m_MediaPlayer.setDataSource(afd.getFileDescriptor(),afd.getStartOffset(),afd.getLength());
//m_MediaPlayer.setDataSource(afd.getFileDescriptor());
m_MediaPlayer.setDataSource(m_strFileName);
afd.close();
} catch (IOException e) {
// TODO Auto-generated catch block
Log.e("Unity","Error m_MediaPlayer.setDataSource() : " + m_strFileName);
e.printStackTrace();
m_iCurrentState = MEDIAPLAYER_STATE.ERROR;
return false;
}
}
}
if(m_iSurfaceTextureID == -1)
{
m_iSurfaceTextureID = InitExtTexture();
}
m_SurfaceTexture = new SurfaceTexture(m_iSurfaceTextureID);
m_SurfaceTexture.setOnFrameAvailableListener(this);
m_Surface = new Surface( m_SurfaceTexture);
m_MediaPlayer.setSurface(m_Surface);
m_MediaPlayer.setOnPreparedListener(this);
m_MediaPlayer.setOnCompletionListener(this);
m_MediaPlayer.setOnErrorListener(this);
m_MediaPlayer.prepareAsync();
return true;
}
synchronized public void onFrameAvailable(SurfaceTexture surface) {
m_bUpdate = true;
}
public void UpdateVideoTexture()
{
if(m_bUpdate == false)
return;
if(m_MediaPlayer != null)
{
if(m_iCurrentState == MEDIAPLAYER_STATE.PLAYING || m_iCurrentState == MEDIAPLAYER_STATE.PAUSED)
{
SetManagerID(m_iNativeMgrID);
boolean [] abValue = new boolean[1];
GLES20.glGetBooleanv(GLES20.GL_DEPTH_TEST, abValue,0);
GLES20.glDisable(GLES20.GL_DEPTH_TEST);
m_SurfaceTexture.updateTexImage();
float [] mMat = new float[16];
m_SurfaceTexture.getTransformMatrix(mMat);
RenderScene(mMat,m_iSurfaceTextureID,m_iUnityTextureID);
if(abValue[0])
{
GLES20.glEnable(GLES20.GL_DEPTH_TEST);
}
else
{
}
abValue = null;
}
}
}
public void SetRockchip(boolean bValue)
{
m_bRockchip = bValue;
}
public void SetLooping(boolean bLoop)
{
// if(m_MediaPlayer != null)
// m_MediaPlayer.setOption();
// int loopCount = bLoop ? 0 : 1;
// m_MediaPlayer.setOption(OPT_CATEGORY_PLAYER, "loop", loopCount);
// _setLoopCount(loopCount);
}
//
// private native void _setLoopCount(int loopCount);
public void SetVolume(float fVolume)
{
if(m_MediaPlayer != null)
{
m_MediaPlayer.setVolume(fVolume, fVolume);
}
}
public void SetSeekPosition(int iSeek)
{
if(m_MediaPlayer != null)
{
if(m_iCurrentState == MEDIAPLAYER_STATE.READY || m_iCurrentState == MEDIAPLAYER_STATE.PLAYING || m_iCurrentState == MEDIAPLAYER_STATE.PAUSED)
{
m_MediaPlayer.seekTo(iSeek);
}
}
}
public int GetSeekPosition()
{
if(m_MediaPlayer != null)
{
if(m_iCurrentState == MEDIAPLAYER_STATE.READY || m_iCurrentState == MEDIAPLAYER_STATE.PLAYING || m_iCurrentState == MEDIAPLAYER_STATE.PAUSED)
{
try {
m_iCurrentSeekPosition = (int) m_MediaPlayer.getCurrentPosition();
} catch (SecurityException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IllegalStateException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
return m_iCurrentSeekPosition;
}
public int GetCurrentSeekPercent()
{
return m_iCurrentSeekPercent;
}
public void Play(int iSeek)
{
if(m_MediaPlayer != null)
{
if(m_iCurrentState == MEDIAPLAYER_STATE.READY || m_iCurrentState == MEDIAPLAYER_STATE.PAUSED || m_iCurrentState == MEDIAPLAYER_STATE.END )
{
//m_MediaPlayer.seekTo(iSeek);
m_MediaPlayer.start();
m_iCurrentState = MEDIAPLAYER_STATE.PLAYING;
}
}
}
public void Reset()
{
if(m_MediaPlayer != null)
{
if(m_iCurrentState == MEDIAPLAYER_STATE.PLAYING)
{
m_MediaPlayer.reset();
}
}
m_iCurrentState = MEDIAPLAYER_STATE.NOT_READY;
}
public void Stop()
{
if(m_MediaPlayer != null)
{
if(m_iCurrentState == MEDIAPLAYER_STATE.PLAYING)
{
m_MediaPlayer.stop();
}
}
m_iCurrentState = MEDIAPLAYER_STATE.NOT_READY;
}
public void RePlay()
{
if(m_MediaPlayer != null)
{
if(m_iCurrentState == MEDIAPLAYER_STATE.PAUSED)
{
m_MediaPlayer.start();
m_iCurrentState = MEDIAPLAYER_STATE.PLAYING;
}
}
}
public void Pause()
{
if(m_MediaPlayer != null)
{
if(m_iCurrentState == MEDIAPLAYER_STATE.PLAYING)
{
m_MediaPlayer.pause();
m_iCurrentState = MEDIAPLAYER_STATE.PAUSED;
}
}
}
public int GetVideoWidth()
{
if(m_MediaPlayer != null)
{
return m_MediaPlayer.getVideoWidth();
}
return 0;
}
public int GetVideoHeight()
{
if(m_MediaPlayer != null)
{
return m_MediaPlayer.getVideoHeight();
}
return 0;
}
public boolean IsUpdateFrame()
{
if(m_bUpdate == true)
{
return true;
}
else
{
return false;
}
}
public void SetUnityTexture(int iTextureID)
{
m_iUnityTextureID = iTextureID;
SetManagerID(m_iNativeMgrID);
SetUnityTextureID(m_iUnityTextureID);
}
public void SetUnityTextureID(Object texturePtr)
{
}
public void SetSplitOBB( boolean bValue,String strOBBName)
{
m_bSplitOBB = bValue;
m_strOBBName = strOBBName;
}
public int GetDuration()
{
if(m_MediaPlayer != null)
{
return (int)m_MediaPlayer.getDuration();
}
return -1;
}
public int InitNative(EasyMovieTexture obj)
{
m_iNativeMgrID = InitNDK(obj);
m_objCtrl.add(this);
return m_iNativeMgrID;
}
public void SetUnityActivity(Activity unityActivity)
{
SetManagerID(m_iNativeMgrID);
m_UnityActivity = unityActivity;
SetAssetManager(m_UnityActivity.getAssets());
}
public void NDK_SetFileName(String strFileName)
{
m_strFileName = strFileName;
}
public void InitJniManager()
{
SetManagerID(m_iNativeMgrID);
InitApplication();
}
public int GetStatus()
{
return m_iCurrentState.GetValue();
}
public void SetNotReady()
{
m_iCurrentState = MEDIAPLAYER_STATE.NOT_READY;
}
public void SetWindowSize()
{
SetManagerID(m_iNativeMgrID);
SetWindowSize(GetVideoWidth(),GetVideoHeight(),m_iUnityTextureID ,m_bRockchip);
}
public int GetError()
{
return m_iErrorCode;
}
public int GetErrorExtra()
{
return m_iErrorCodeExtra;
}
@Override
public boolean onError(IMediaPlayer iMediaPlayer, int i, int i1) {
//public boolean onError(MediaPlayer arg0, int arg1, int arg2) {
// TODO Auto-generated method stub
if (iMediaPlayer == m_MediaPlayer)
{
String strError;
switch (i)
{
case IMediaPlayer.MEDIA_ERROR_NOT_VALID_FOR_PROGRESSIVE_PLAYBACK:
strError = "MEDIA_ERROR_NOT_VALID_FOR_PROGRESSIVE_PLAYBACK";
break;
case MediaPlayer.MEDIA_ERROR_SERVER_DIED:
strError = "MEDIA_ERROR_SERVER_DIED";
break;
case MediaPlayer.MEDIA_ERROR_UNKNOWN:
strError = "MEDIA_ERROR_UNKNOWN";
break;
default:
strError = "Unknown error " + i;
}
m_iErrorCode = i;
m_iErrorCodeExtra = i1;
m_iCurrentState = MEDIAPLAYER_STATE.ERROR;
return true;
}
return false;
}
@Override
public void onCompletion(IMediaPlayer arg0) {
// TODO Auto-generated method stub
if (arg0 == m_MediaPlayer)
m_iCurrentState = MEDIAPLAYER_STATE.END;
}
@Override
public void onBufferingUpdate(IMediaPlayer arg0, int arg1) {
// TODO Auto-generated method stub
if (arg0 == m_MediaPlayer)
m_iCurrentSeekPercent = arg1;
}
@Override
public void onPrepared(IMediaPlayer arg0) {
// TODO Auto-generated method stub
if (arg0 == m_MediaPlayer)
{
m_iCurrentState = MEDIAPLAYER_STATE.READY;
SetManagerID(m_iNativeMgrID);
m_iCurrentSeekPercent = 0;
m_MediaPlayer.setOnBufferingUpdateListener(this);
}
}
public enum MEDIAPLAYER_STATE
{
NOT_READY (0),
READY (1),
END (2),
PLAYING (3),
PAUSED (4),
STOPPED (5),
ERROR (6);
private int iValue;
MEDIAPLAYER_STATE (int i)
{
iValue = i;
}
public int GetValue()
{
return iValue;
}
}
}补充: 以上仅仅针对有基础的同学们，如果看到这还是云里雾里的同学可以邮箱留言我直接发demo吧。