JNI for x264 encoder

I decided to implement JNI for x264 encoder as one of most popular H264 codec implementations. My short guideline covers x264, JNI related compilation steps and sample Java class.

So let start.

The first step that you need to perform is to compile proper library from x264 C code. For this activity I used following Alex's guide. I used several steps from this manual. One difference with my way was 64bit mingw installed on Ubuntu 14 64bit LTS, installed newest JDK 1.8. The JNI code compilation requires C headers copied from the Windows JDK located under C:\Program Files\Java\jdk1.8.0\include\win32. Copy this directory into your Ubuntu JDK folder: /usr/lib/jvm/java-8-oracle/include. Finally I configured x264:

./configure --cross-prefix=x86_64-w64-mingw32- --host=x86_64-pc-mingw32 --disable-avs --enable-shared --disable-cli --enable-win32thread

make

After a time you have to see the libx264-142.dll.

Next step I compile the JNI related code:

x86_64-w64-mingw32-gcc -Wl,--add-stdcall-alias -shared -m64 -O3 -ffast-math -I/home/vadym/win32-cross/src/x264 -I/home/vadym/win32-cross/include -I $JAVA_HOME/include -I $JAVA_HOME/include/win32 -mfpmath=sse -msse -std=gnu99 -fomit-frame-pointer -fno-tree-vectorize -fno-zero-initialized-in-bss -o h264-jni.dll h264-jni.c -L ~/win32-cross/src/x264 -l x264.dll

This command gets h264-jni.dll

These two dll's will be required for your Java encoder:

H264Encoder.java

package x264;

public class H264Encoder {

    static {
            System.loadLibrary("libx264-142");
            System.loadLibrary("h264-jni");
        }

    private long encoder_pointer;

    public H264Encoder(int width, int height) {
        this.encoder_pointer = openEncoder(width, height);
    }
        
    public long openEncoder(int width, int height){
        encoder_pointer = this.open(width, height);
        return encoder_pointer;
    }

    public int encode(int type, byte[] in, int insize, byte[] out, boolean[] keyframe){
        return this.encode(encoder_pointer, type, in, insize, out, keyframe);
    }
    
    public int finish(){
        return this.finish(encoder_pointer);
    }
    
    private final native long open(int width, int height);
    
    private final native int encode(long encoder, int type, byte[] in, int insize, byte[] out, boolean[] keyframe);
    
    private final native int finish(long encoder);

    public static void main(String[] args){
        H264Encoder encoder = new H264Encoder(300, 200);
        encoder.finish();    
    }
}


h264-jni.c



#include <stdint.h>
#include <string.h>
#include <jni.h>
#include <stdio.h>
#include <stdlib.h>
#include <x264.h>

#define DATA_MAX 3000000
#define H264_MTU 1024

typedef struct

{
    x264_param_t * param;
    x264_t *handle;
    x264_picture_t * picture;
    x264_nal_t  *nal;
} Encoder;

JNIEXPORT jlong JNICALL Java_x264_H264Encoder_open(JNIEnv* env, jobject thiz, jint width, jint height){

    Encoder * en = (Encoder *) malloc(sizeof(Encoder));
    en->param = (x264_param_t *) malloc(sizeof(x264_param_t));
    en->picture = (x264_picture_t *) malloc(sizeof(x264_picture_t));
    x264_param_default(en->param); //set default param

 //ultrafast,superfast,veryfast,faster,fast,medium,slow,slower,veryslow,placebo
 //film,animation,grain,stillimage,psnr,ssim,fastdecode,zerolatency,touhou
    x264_param_default_preset(en->param, "fast", "animation");
    //en->param->rc.i_rc_method = X264_RC_CQP;
    //en->param->i_log_level = X264_LOG_NONE;
    en->param->i_width = width; //set frame width
    en->param->i_height = height; //set frame height
    //en->param->rc.i_lookahead = 0;
    en->param->i_bframe = 1;
    //en->param->i_fps_num = 5;
    //en->param->i_fps_den = 1;
    //baseline,main,high,high10,high422,high444
    x264_param_apply_profile(en->param, "main");
    if ((en->handle = x264_encoder_open(en->param)) == 0) {
        return 0;
    }
    /* Create a new pic */
    x264_picture_alloc(en->picture, X264_CSP_I420, en->param->i_width, en->param->i_height);
    return (jlong) en;
}

JNIEXPORT jint JNICALL Java_x264_H264Encoder_finish(JNIEnv* env, jobject thiz, jlong handle){
    Encoder * en = (Encoder *) handle;
    if(en->picture)
    {
        x264_picture_clean(en->picture);
        free(en->picture);
        en->picture = 0;
    }
    if(en->param)
    {
        free(en->param);
        en->param = 0;
    }

    if(en->handle)
    {
        x264_encoder_close(en->handle);
    }
    free(en);
    return 0;
}

JNIEXPORT jint JNICALL Java_x264_H264Encoder_encode(JNIEnv* env, jobject thiz, jlong handle, jint type, jbyteArray in, jint insize, jbyteArray out, jbooleanArray keyframe)

{
    Encoder * en = (Encoder *) handle;
    jboolean key_frame[1];

    x264_picture_t pic_out;
    int i_data=0;
    int nNal =-1;
    int result=0;
    int i=0;
    unsigned char in_buf[insize];
    unsigned char * buf = in_buf;
    (*env)->GetByteArrayRegion (env, in, 0, insize, buf);
    jbyte h264Buf[insize];
    jbyte * pTmpOut = h264Buf;
    int nPicSize=en->param->i_width*en->param->i_height;
    int size = nPicSize*sizeof(unsigned char)/4;
    memcpy(en->picture->img.plane[0], in_buf, 4*size);
    memcpy(en->picture->img.plane[1],&buf[nPicSize],  size);
    memcpy(en->picture->img.plane[2],&buf[nPicSize+size],size);
  /*#define X264_TYPE_AUTO          0x0000
    #define X264_TYPE_IDR           0x0001
    #define X264_TYPE_I             0x0002
    #define X264_TYPE_P             0x0003
    #define X264_TYPE_BREF          0x0004
    #define X264_TYPE_B             0x0005
    #define X264_TYPE_KEYFRAME      0x0006*/
    en->picture->i_type = type;

    if(x264_encoder_encode(en->handle, &(en->nal), &nNal, en->picture, &pic_out) < 0)
    {
        return -1;
    }
    for (i = 0; i < nNal; i++){
        memcpy(pTmpOut, en->nal[i].p_payload, en->nal[i].i_payload);
        pTmpOut += en->nal[i].i_payload;
        result += en->nal[i].i_payload;
    }
    
    key_frame[0] = (jboolean) pic_out.b_keyframe;
    (*env)->SetBooleanArrayRegion(env, keyframe, 0, 1, key_frame);
    (*env)->SetByteArrayRegion(env, out, 0, result, h264Buf);
    return result;

}

H264 Specification Documents

1.	Advanced video coding for generic audiovisual services. TELECOMMUNICATION STANDARDIZATION SECTOR OF ITU. 676p. 03/2010.	PDF	4741k	Download
2.	Information technology — Coding of audio-visual objects — Part 10: Advanced Video Coding. INTERNATIONAL STANDARD ISO/IEC 14496-10. 268p. 10/2004.	PDF	1964k	Download
3.	Conformance specification for H.264 advanced video coding. TELECOMMUNICATION STANDARDIZATION SECTOR OF ITU. 47p. 03/2005.	PDF	368k	Download

H264 AVCVIDEOPACKET

Let see how to parse the AVCVIDEOPACKET accordingly with FLV video tag specfication. As an example I took one of Youtube videos to show where is the record. On the picture below you can see the first video packet inside red rectangle (1).

FLV Video tag

1. FLV Tag. First red rectangle.

09 - TagType(UB[5]): Video Packet. Previous fields Reserved(UB[2]) = 0, Filter(UB[1]) = 0. Sum of these fields length = 1Byte = UB[5]+UB[2]+UB[1].

00 00 2F - DataSize(UI[24]): Length of the message. 00 00 2F = 47Bytes. Number of bytes after StreamID to end of tag (Equal to length of the tag – 11).

00 00 00 - Timestamp(UI[24]): Time in milliseconds at which the data in this tag applies. This value is relative to the first tag in the FLV file, which always has a timestamp of 0.

00 - TimestampExtended(UI[8]): Extension of the Timestamp field to form a SI32 value.

00 00 00 - StreamID (UI[24]): Always 0.

2. VideoTagHeader. Yellow rectangle.

17 - Composition of two fields. Frame Type(UI[4]) = 1 means It's key frame. And CodecID(UI[4]) = 7 means It's AVC encoded frame.

00 - AVCPacketType(UI[8]). 00 = 0 = AVC sequence header.

00 00 00 - CompositionTime (UI[24]). Defined only for AVCPacketType == 1. Otherwise always = 0.

3. VideoTagBody. White rectangle. If AVCPacketType == 0 AVCDecoderConfigurationRecord.

01 - configurationVersion (UI[8]) = 1

4D - AVCProfileIndication (UI[8]) = 77 contains the profile code as defined in ISO/IEC 14496-10.

40 - profile_compatibility (UI[8]) = 64 is a byte defined exactly the same as the byte which occurs between the profile_IDC and level_IDC in a sequence parameter set (SPS), as defined in ISO/IEC 14496-10.

1E - AVCLevelIndication (UI[8]) = 30 contains the level code as defined in ISO/IEC 14496-10.

FF - Composition of two fields. reserved = ‘111111’b. And lengthSizeMinusOne (UI[2]) = 11b = 3, indicates the length in bytes of the NALUnitLength field in an AVC video sample or AVC parameter set sample.

E1 - Composition of two fields. reserved = ‘111’b. And numOfSequenceParameterSets (UI[5]) = 000001b = 1 indicates the number of SPSs that are used as the initial set of SPSs for decoding the AVC elementary stream.

Next fields are included into a cycle.

for (i=0; i< numOfSequenceParameterSets; i++) {
unsigned int(16) sequenceParameterSetLength ;
bit(8*sequenceParameterSetLength) sequenceParameterSetNALUnit; }

 ------ Start Cycle ------

4. Green rectangle. 00 1B - sequenceParameterSetLength (UI[16]) = 27 indicates the length in bytes of the SPS NAL unit as defined in ISO/IEC 14496-10.

5. Green rectangle. 67 4D 40 ... B1 72 40 - pictureParameterSetNALUnit (bit(8*pictureParameterSetLength)=8*27b=27Bytes). contains a SPS NAL unit, as specified in ISO/IEC 14496-10. SPSs shall occur in order of ascending parameter set identifier with gaps being allowed.

 ------ End Cycle ------

01 - numOfPictureParameterSets (UI[8]). indicates the number of picture parameter sets (PPSs) that are used as the initial set of PPSs for decoding the AVC elementary stream.

 Next fields are included into a cycle.

for (i=0; i< numOfPictureParameterSets; i++)
{ unsigned int(16) pictureParameterSetLength;
bit(8*pictureParameterSetLength) pictureParameterSetNALUnit; }

 ------ Start Cycle ------

6. Green rectangle. 00 04 - pictureParameterSetLength (UI[16]) = 4 indicates the length in bytes of the PPS NAL unit as defined in ISO/IEC 14496-10.

7. White rectangle. 68 EE 32 C8 - pictureParameterSetNALUnit (bit(8*pictureParameterSetLength) = 8*4b = 4Bytes) contains a PPS NAL unit, as specified in ISO/IEC 14496-10. PPSs shall occur in order of ascending parameter set identifier with gaps being allowed.

 ------ End Cycle ------

 8. Red rectangle. 00 00 00 3A - PreviousTagSize1 (UI[32]) = 58Bytes. Size of previous tag, including its header, in bytes. Size of previous tag, including its header, in bytes. For FLV version 1, this value is 11 plus the DataSize of the previous tag You may check the length of the tag pictured sections from 1 to 7. It's exactly 58Bytes.

Summary

H264 profile should give the following data:

AVCProfileIndication
profile_compatibility
AVCLevelIndication
lengthSizeMinusOne (length in bytes of the NALUnitLength)
numOfSequenceParameterSets
sequenceParameterSetLength
sequenceParameterSetNALUnit
numOfPictureParameterSets
pictureParameterSetLength
pictureParameterSetNALUnit


Download
1. Adobe Flash Video File Format Specification Version 10.1
2. Advanced Video Coding (AVC) file format

Ремонт компьютеров

How to compile Red5 sources

Some times I need to recompile the latest sources from google code svn repository. I needed to compile the sources from Maven. So there is the pom.xml that I got. But first you should enable Maven dependencies management from the maven2eclipse plugin options.

How to publish H264 frame

One more task is to publish H264 frame. In my case it is byte[] encoded array. I found one Java library that can deal with this ShoutcastH264.jar. I do not copy the code but some technical info saved my time:

public class VideoFramer
{
  protected static Logger log = LoggerFactory.getLogger(VideoFramer.class);
  private IICYEventSink output;
  private int _codecSetupLength;
  private boolean codecSent;
  private int _SPSLength;
  private int _PPSLength;
  private int[] _pCodecSetup;
  private int[] _pSPS;
  private int[] _pPPS;
  private int[] _pSEI;
  public boolean keepNALU;
  private IRTMPEvent lastKey;
  private List<IRTMPEvent> slices = new ArrayList<IRTMPEvent>();
 
  private final int CodedSlice = 1;

  private final int IDR = 5;
  private final int SEI = 6;
  private final int SPS = 7;
  private final int PPS = 8;

...

  public void pushAVCFrame(int[] frame, int timecode)
  {
    if (frame.length == 0) {
      return;
    }
    if (this.keepNALU)
    {
      IoBuffer buffV = IoBuffer.allocate(frame.length + 6);
      buffV.setAutoExpand(true);
      buffV.put((byte)23);
      buffV.put((byte)1);

      buffV.put((byte)0);
      buffV.put((byte)0);
      buffV.put((byte)0);

      buffV.put((byte)0);
      buffV.put((byte)0);

      for (int r = 0; r < frame.length; ++r)
      {
        buffV.put((byte)frame[r]);
      }

      buffV.put((byte)0);
      buffV.flip();
      buffV.position(0);

      IRTMPEvent videoNAL = new VideoData(buffV);
      videoNAL.setHeader(new Header());

//      this.output.dispatchEvent(videoNAL);

      return;
    }

    for (int i = 0; i < frame.length; ++i) {
      if ((frame[i] != 0) ||
        (frame[(i + 1)] != 0) ||
        (frame[(i + 2)] != 0) ||
        (frame[(i + 3)] != 1))
        continue;
      i += 4;

      int size = findFrameEnd(frame, i);

      if (size == -1) {
        size = frame.length - i;
      }
      else {
        size -= i;
      }
      processNal(frame, i, size);

      i += size - 1;
    }
  }

How to open YUV pictures (sequences)

I created a small YUV viewer to check YUV video frames and draw it as a picture. You may find it here:
YuvViewer in the Softpedia

Converter BufferedImage to YUV (4:2:0)

I decided to implement h264 encoding. There is an issue to put input data in YUV (4:2:0) format. Looking for the solution I found the project:

http://code.google.com/p/java-yuv/source/browse/trunk/WriteYUV.java

I hope you can find It useful also:

public void writeImage(BufferedImage bi)
{
        int w = bi.getWidth();
        int h = bi.getHeight();

        LinkedList<Byte> uBuffer = new LinkedList<Byte>();
        LinkedList<Byte> vBuffer = new LinkedList<Byte>();
        try
        {
                boolean s = false;

                for (int j = 0; j < h; j++)
                {
                        for (int i = 0; i < w; i++)
                        {
                                int color = bi.getRGB(i, j);

                                int alpha = color >> 24 & 0xff;
                                int R = color >> 16 & 0xff;
                                int G = color >> 8 & 0xff;
                                int B = color & 0xff;

                                //~ int y = (int) ((0.257 * red) + (0.504 * green) + (0.098 * blue) + 16);
                                //~ int v = (int) ((0.439 * red) - (0.368 * green) - (0.071 * blue) + 128);
                                //~ int u = (int) (-(0.148 * red) - (0.291 * green) + (0.439 * blue) + 128);
                               
                                int Y = (int)(R *  .299000 + G *  .587000 + B *  0.114000);
                                int U = (int)(R * -.168736 + G * -.331264 + B *  0.500000 + 128);
                                int V = (int)(R *  .500000 + G * -.418688 + B * -0.081312 + 128);

                               
                                int arraySize = height * width;
                                int yLoc = j * width + i;
                                int uLoc = (j/2) * (width/2) + i/2 + arraySize;
                                int vLoc = (j/2) * (width/2) + i/2 + arraySize + arraySize/4;
                               
                                oneFrame[yLoc] = (byte)Y;
                                oneFrame[uLoc] = (byte)U;
                                oneFrame[vLoc] = (byte)V;
                               
                                s = !s;
                        }
                }
               
                for(int i=0;i<oneFrameLength;i++)
                {
                        dos.write(oneFrame[i]);
                }
        }
        catch (Exception e)
        {
                e.printStackTrace();
        }
}

Thanks for the code. Author saved my time.