BPI-CM9-Linux/aml-comp/multimedia/gst-plugin-aml-subtitlesink/src/gstamlsubtsink.c

/* GStreamer
 * Some extracts from GStreamer Plugins Base, which is:
 * Copyright (C) 1999,2000 Erik Walthinsen <omega@cse.ogi.edu>
 * Copyright 2005 Wim Taymans <wim@fluendo.com>
 * Copyright 2020 Amlogic, Inc.
 * Here licensed under the GNU Lesser General Public License, version 2.1
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation;either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the
 * Free SoftwareFoundation, Inc., 59 Temple Place, Suite 330, Boston,
 * MA 02111-1307 USA
 */
/**
 * SECTION:element-gstamlsubtitlesink
 *
 * The amlsubtitlesink element connects to tv subtitle service for subtitle
 * decoding and rendering.
 *
 */

#include <inttypes.h>
#include <fcntl.h>
#include <unistd.h>
#include <string.h>
#include <stdlib.h>
#include <pthread.h>
#include <stdlib.h>
#include <sys/prctl.h>

#include "gstamlsubtsink.h"
#include "stdio.h"

#include "aml_version.h"


GST_DEBUG_CATEGORY (gst_aml_subt_sink_debug_category);
#define GST_CAT_DEFAULT gst_aml_subt_sink_debug_category

#define GST_AML_SUBT_SINK_GET_PRIVATE(obj)  \
   (G_TYPE_INSTANCE_GET_PRIVATE ((obj), GST_TYPE_AML_SUBT_SINK, GstAmlSubtSinkPrivate))

#define PTS_90K (90000)
#define QUEUE_MAX (1024)
#define REND_THRESHOLD_L (90 * 50) //50ms
#define REND_THRESHOLD_H (90 * 30) //30ms

#define ENABLE_SUB_SERVER 1
#define DUMP_ES_DATA 0

#if ENABLE_SUB_SERVER
#include "subtitlecc.h"
#endif


struct _GstAmlSubtSinkPrivate
{
  GstPadEventFunction parentEventFunc;

  GstElement *video_sink;

  /* render thread */
  gboolean quit_render_thread;
  GThread *render_thread;
  gboolean render_start;

  /* eos thread */
  gboolean quit_eos_thread;
  GThread *eos_thread;
  gboolean eos_start;

  /* buf info */
  GstQueueArray* buffer_queue;

  guint32 in_buf_cnt;
  guint32 out_buf_cnt;
  guint32 drop_buf_cnt;

  /* caps info */
  guint8 *codec_data;
  int codec_data_len;
  guint composition_page_id;
  guint ancillary_page_id;

  gboolean paused;
  gboolean eos;
  guint32 seqnum;

#if ENABLE_SUB_SERVER
  /*  subtitle client */
  SubSourceHandle subt_src;
  AmlSubtitleHnd subt_hnd;
  enum CodecID sub_format;
  gboolean sub_format_changed;
#endif

  /* cc channel */
  int cc_channel;
  gboolean show;
  gboolean session_initialized;
  gboolean is_flushing;
  GMutex data_lock;
};

enum
{
  PROP_0,
  PROP_SHOW_SUBTITLE,
  PROP_IS_CC,
  PROP_CC_CUR_TRACK,
  PROP_CC_TRACK_NUM,
  PROP_LAST
};

/* pad templates */
static GstStaticPadTemplate aml_subt_sink_template =
GST_STATIC_PAD_TEMPLATE ("sink",
    GST_PAD_SINK,
    GST_PAD_ALWAYS,
    GST_STATIC_CAPS (
        "application/x-ssa; "
        "subpicture/x-dvd; "
        "text/x-raw, format = (string) pango-markup; "
        "subpicture/x-dvb; "
        "subpicture/x-pgs; "
        "text/x-raw(meta:GstSubtitleMeta); "
        "text/x-cc; "
        )
);

/* class initialization */
#define gst_aml_subt_sink_parent_class parent_class
#if GLIB_CHECK_VERSION(2,58,0)
G_DEFINE_TYPE_WITH_CODE (GstAmlSubtSink, gst_aml_subt_sink, GST_TYPE_BASE_SINK,
  GST_DEBUG_CATEGORY_INIT (gst_aml_subt_sink_debug_category, "amlsubtsink", 0,
  "debug category for amlsubtsink element");G_ADD_PRIVATE(GstAmlSubtSink));
#else
G_DEFINE_TYPE_WITH_CODE (GstAmlSubtSink, gst_aml_subt_sink, GST_TYPE_BASE_SINK,
  GST_DEBUG_CATEGORY_INIT (gst_aml_subt_sink_debug_category, "amlsubtsink", 0,
  "debug category for amlsubtsink element");
  );
#endif

static void gst_aml_subt_sink_dispose(GObject * object);
static void gst_aml_subt_sink_set_property(GObject * object, guint prop_id,
    const GValue * value, GParamSpec * pspec);
static void gst_aml_subt_sink_get_property(GObject * object, guint prop_id,
    GValue * value, GParamSpec * pspec);
static GstStateChangeReturn gst_aml_subt_sink_change_state(GstElement *
    element, GstStateChange transition);
static gboolean gst_aml_subt_sink_query(GstElement * element, GstQuery * query);
static GstFlowReturn gst_aml_subt_sink_chain (GstPad * pad, GstObject * parent, GstBuffer * buf);
static gboolean gst_aml_subt_sink_event (GstPad * pad, GstObject * parent, GstEvent * event);

static int start_render_thread (GstAmlSubtSink * sink);
static void stop_render_thread(GstAmlSubtSink * sink);
static int start_eos_thread (GstAmlSubtSink * sink);
static void stop_eos_thread (GstAmlSubtSink * sink);

static gboolean subt_set_caps(GstAmlSubtSink *sink, GstCaps *caps);

static void
gst_aml_subt_sink_class_init (GstAmlSubtSinkClass * klass)
{
  GObjectClass *gobject_class;
  GstElementClass *gstelement_class;

  gobject_class = (GObjectClass *) klass;
  gstelement_class = (GstElementClass *) klass;

#if GLIB_CHECK_VERSION(2,58,0)
#else
  g_type_class_add_private (klass, sizeof (GstAmlSubtSinkPrivate));
#endif

  /* Setting up pads and setting metadata should be moved to
     base_class_init if you intend to subclass this class. */
  gst_element_class_add_static_pad_template (GST_ELEMENT_CLASS(klass),
      &aml_subt_sink_template);

  gst_element_class_set_static_metadata (GST_ELEMENT_CLASS(klass),
      "Amlogic subtitle sink", "Sink/Subtitle", "gstream plugin to connect subtitle server",
      "bo.xiao@amlogic.com");

  gobject_class->set_property = gst_aml_subt_sink_set_property;
  gobject_class->get_property = gst_aml_subt_sink_get_property;
  gobject_class->dispose = gst_aml_subt_sink_dispose;


  gstelement_class->change_state = gst_aml_subt_sink_change_state;
  gstelement_class->query = gst_aml_subt_sink_query;

  g_object_class_install_property(
        gobject_class, PROP_SHOW_SUBTITLE,
        g_param_spec_boolean("show-subtitle",
                             "show subtitle or not",
                             "0: hide; 1: show",
                             TRUE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
  g_object_class_install_property(
        gobject_class, PROP_IS_CC,
        g_param_spec_boolean("is-cc",
                             "subtitle is cc or not",
                             "subtitle is cc or not",
                             FALSE, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));
  g_object_class_install_property(
        gobject_class, PROP_CC_CUR_TRACK,
        g_param_spec_int("cc-cur-track",
                             "current cc track",
                             "set/get current cc track",
                             G_MININT, G_MAXINT, 0, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
  g_object_class_install_property(
        gobject_class, PROP_CC_TRACK_NUM,
        g_param_spec_int("cc-track-num",
                             "cc track number",
                             "get cc track number",
                             G_MININT, G_MAXINT, 0, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));

  gst_element_class_set_details_simple (gstelement_class, "AmlSubtSink",
      "Decoder/Sink/Subtitle",
      "Gstreamer sink plugin for subtitle decoding and rendering",
      "Amlogic");

  GST_DEBUG_CATEGORY_INIT (gst_aml_subt_sink_debug_category,
                           "amlsubtsink",
                           0,
                           "amlsubtsink element"
                          );

}

static void
gst_aml_subt_sink_init (GstAmlSubtSink* sink)
{
  GstBaseSink *basesink;

  GstAmlSubtSinkPrivate *priv = GST_AML_SUBT_SINK_GET_PRIVATE (sink);

  sink->priv = priv;
  basesink = GST_BASE_SINK_CAST (sink);

  priv->parentEventFunc = GST_PAD_EVENTFUNC(basesink->sinkpad);

  /* bypass sync control of basesink */
  gst_base_sink_set_sync(basesink, FALSE);
  gst_pad_set_event_function (basesink->sinkpad, gst_aml_subt_sink_event);
  gst_pad_set_chain_function (basesink->sinkpad, gst_aml_subt_sink_chain);

  memset(priv, 0, sizeof(GstAmlSubtSinkPrivate));
#if ENABLE_SUB_SERVER
  priv->sub_format = CODEC_ID_SSA;
  priv->sub_format_changed = FALSE;
#endif
  priv->composition_page_id = 0;
  priv->ancillary_page_id = 0;
  priv->show = TRUE;
  priv->session_initialized = FALSE;
  priv->buffer_queue = gst_queue_array_new (QUEUE_MAX);
  gst_queue_array_set_clear_func(priv->buffer_queue, (GDestroyNotify)gst_buffer_unref);
  priv->cc_channel = 0;
  priv->quit_eos_thread = FALSE;
  priv->is_flushing = FALSE;
  g_mutex_init (&priv->data_lock);
}

static void
gst_aml_subt_sink_dispose (GObject * object)
{
  GstAmlSubtSink * sink = GST_AML_SUBT_SINK(object);
  GstAmlSubtSinkPrivate *priv = sink->priv;

  GST_DEBUG_OBJECT (sink, "dispose");

  if (priv->buffer_queue)
  {
    GST_WARNING_OBJECT (sink, "free array, in(%d), drop(%d), out(%d)", priv->in_buf_cnt, priv->drop_buf_cnt, priv->out_buf_cnt);
    gst_queue_array_free (priv->buffer_queue);
    priv->buffer_queue = NULL;
  }
  g_mutex_clear (&priv->data_lock);
  G_OBJECT_CLASS (parent_class)->dispose (object);
}

static gboolean
gst_aml_subt_sink_query (GstElement * element, GstQuery * query)
{
  gboolean res = FALSE;
  GstAmlSubtSink *sink = GST_AML_SUBT_SINK (element);

  switch (GST_QUERY_TYPE (query)) {
    case GST_QUERY_CAPS:
    {
      GST_LOG_OBJECT (sink, "query CAPS");
      break;
    }
    case GST_QUERY_ACCEPT_CAPS:
    {
      GST_LOG_OBJECT (sink, "query ACCEPT_CAPS");
      break;
    }
    default:
      break;
  }

  res = GST_ELEMENT_CLASS (parent_class)->query (element, query);

  return res;
}

static void
gst_aml_subt_sink_set_property (GObject * object, guint property_id,
    const GValue * value, GParamSpec * pspec)
{
  GstAmlSubtSink *sink = GST_AML_SUBT_SINK (object);
  GstAmlSubtSinkPrivate *priv = sink->priv;

  GST_DEBUG_OBJECT (sink, "id = %d", property_id);

  switch (property_id) {
    case PROP_SHOW_SUBTITLE:
      {
        gboolean show = g_value_get_boolean(value);
#if ENABLE_SUB_SERVER
        if (priv->subt_hnd && show != priv->show)
        {
          sub_show(priv->subt_hnd, show);
        }
#endif
        priv->show = show;
      }
      break;
    case PROP_CC_CUR_TRACK:
      priv->cc_channel = g_value_get_int(value);
      if (priv->subt_hnd) {
        sub_set_cc_channel(priv->subt_hnd, priv->cc_channel);
      }
      break;
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
      break;
  }
}

static void gst_aml_subt_sink_get_property (GObject * object, guint property_id,
    GValue * value, GParamSpec * pspec)
{
  GstAmlSubtSink *sink = GST_AML_SUBT_SINK (object);
  GstAmlSubtSinkPrivate *priv = sink->priv;

  GST_DEBUG_OBJECT (sink, "id = %d", property_id);

  switch (property_id) {
    case PROP_SHOW_SUBTITLE:
      g_value_set_boolean(value, priv->show);
      break;
    case PROP_CC_CUR_TRACK:
      g_value_set_int(value, priv->cc_channel);
      break;
    case PROP_CC_TRACK_NUM:
      {
        int num = sub_get_cc_channel_number();
        g_value_set_int(value, num);
      }
      break;
    case PROP_IS_CC:
      {
        gboolean is_cc = !priv->sub_format;
        g_value_set_boolean(value, is_cc);
      }
      break;
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
      break;
  }
}

static gboolean
gst_aml_subt_sink_event (GstPad * pad, GstObject * parent, GstEvent * event)
{
  gboolean result = TRUE;
  GstAmlSubtSink *sink = GST_AML_SUBT_SINK (parent);
  GstAmlSubtSinkPrivate *priv = sink->priv;

  //GST_DEBUG_OBJECT (sink, "received event %p %" GST_PTR_FORMAT, event, event);

  switch (GST_EVENT_TYPE (event)) {
    case GST_EVENT_EOS:
    {
      GST_DEBUG_OBJECT (sink, "eos, Got seqnum #%" G_GUINT32_FORMAT,
          gst_event_get_seqnum (event));

      priv->seqnum = gst_event_get_seqnum (event);

      priv->eos = TRUE;
#if ENABLE_SUB_SERVER
      start_eos_thread (sink);
#else
      //not enable server, post eos immediately.
      GstMessage *message;
      GST_WARNING_OBJECT (sink, "Posting EOS, in(%d), drop(%d), out(%d)",
        priv->in_buf_cnt, priv->drop_buf_cnt, priv->out_buf_cnt);
      message = gst_message_new_eos (GST_OBJECT_CAST (sink));
      gst_message_set_seqnum (message, priv->seqnum);
      gst_element_post_message (GST_ELEMENT_CAST (sink), message);
#endif
      break;
    }
    case GST_EVENT_FLUSH_START:
    {
      GST_DEBUG_OBJECT (sink, "flush start");
      priv->is_flushing = TRUE;
      g_mutex_lock(&priv->data_lock);
      if (priv->buffer_queue)
      {
        GST_WARNING_OBJECT (sink, "clear array, in(%d), drop(%d), out(%d)", priv->in_buf_cnt, priv->drop_buf_cnt, priv->out_buf_cnt);
        gst_queue_array_clear (priv->buffer_queue);
      }
      g_mutex_unlock(&priv->data_lock);
      break;
    }
    case GST_EVENT_FLUSH_STOP:
    {
      GST_DEBUG_OBJECT (sink, "flush stop");
      priv->out_buf_cnt = 0;
      priv->in_buf_cnt = 0;
      priv->drop_buf_cnt = 0;
      priv->is_flushing = FALSE;
      break;
    }
    case GST_EVENT_SEGMENT:
    {
      GstSegment segment;

      GST_DEBUG_OBJECT (sink, "segment");
      if (GST_STATE(sink) <= GST_STATE_READY) {
        GST_WARNING_OBJECT (sink, "segment event in wrong state %d", GST_STATE(sink));
        break;
      }

      gst_event_copy_segment (event, &segment);
      GST_DEBUG_OBJECT (sink, "configured segment %" GST_SEGMENT_FORMAT,
              &segment);
      break;
    }
    case GST_EVENT_STREAM_START:
    {
      guint group_id;
      gst_event_parse_group_id (event, &group_id);

      GST_DEBUG_OBJECT (sink, "stream start, gid %d", group_id);
      break;
    }
    case GST_EVENT_STREAM_GROUP_DONE:
    {
      guint group_id;
      gst_event_parse_stream_group_done (event, &group_id);
      GST_DEBUG_OBJECT (sink, "stream group done, gid %d", group_id);
      break;
    }
    case GST_EVENT_CAPS:
    {
      GstCaps *caps = NULL;

      gst_event_parse_caps (event, &caps);
      if (caps) {
        gchar *str= gst_caps_to_string(caps);
        GST_DEBUG_OBJECT (sink, "caps :%s ", str);
        g_free( str );

        subt_set_caps(sink, caps);
      }
      break;
    }
    default:
    {
      //GST_LOG_OBJECT (sink, "not handle");
      break;
    }
  }

  if (priv->parentEventFunc)
  {
     result= priv->parentEventFunc(pad, parent, event);
  }

  return result;
}

static GstFlowReturn
gst_aml_subt_sink_chain (GstPad * pad, GstObject * parent, GstBuffer * buf)
{
  GstAmlSubtSink *sink = GST_AML_SUBT_SINK (parent);
  GstAmlSubtSinkPrivate *priv = sink->priv;

#if ENABLE_SUB_SERVER
  if (priv->is_flushing) {
    GST_INFO ("is flushing,dropped");
    goto fail_return;
  }

  // push queue
  g_mutex_lock(&priv->data_lock);
  if (priv->buffer_queue)
  {
    gst_queue_array_push_tail (priv->buffer_queue, buf);
  }
  priv->in_buf_cnt++;
  g_mutex_unlock(&priv->data_lock);
#endif

done:
  return GST_FLOW_OK;

fail_return:
  gst_buffer_unref (buf);
  goto done;
}

static int restart_subtitle_handle(GstAmlSubtSink * sink)
{
  GstAmlSubtSinkPrivate *priv = sink->priv;

  GST_ERROR("restart enter:%d", priv->sub_format);
  sub_stop(priv->subt_hnd);
  sub_start(priv->subt_hnd, 0, priv->sub_format, priv->composition_page_id, priv->ancillary_page_id);
  sub_show(priv->subt_hnd, priv->show);
  if (priv->sub_format != 0)
  {
    SubSource_ReportType(priv->subt_src, priv->sub_format);
  }
  else
  {
    sub_set_cc_channel(priv->subt_hnd, priv->cc_channel);
    GST_INFO_OBJECT(sink, "set cc channel %d %p", priv->cc_channel, priv->subt_hnd);
  }
  GST_ERROR("restart ok:%d", priv->sub_format);
}

static int start_subtitle_handle(GstAmlSubtSink * sink)
{
  GstAmlSubtSinkPrivate *priv = sink->priv;
#if ENABLE_SUB_SERVER
  if (priv->session_initialized)
  {
    return 0;
  }

  GST_INFO_OBJECT(sink, "sub_format %d, handle:%p", priv->sub_format, priv->subt_src);
  if (priv->sub_format != 0) {
    int sessionId = -1;
    amlsub_GetSessionId(priv->subt_hnd, &sessionId);
    GST_INFO_OBJECT(sink, "SubSource_Create");
    priv->subt_src = SubSource_Create(sessionId < 0 ? /*SUBTITLE_SOURCE_CREATE_ID*/0 : sessionId);
    GST_INFO_OBJECT(sink, "SubSource_Create over %p, sessionId:%d", priv->subt_src, sessionId);
    if (!priv->subt_src) {
      GST_ERROR ("Create subtitle client fail.");
      return -1;
    }
  }
  priv->session_initialized = TRUE;
  sub_start(priv->subt_hnd, 0, priv->sub_format, priv->composition_page_id, priv->ancillary_page_id);
  sub_show(priv->subt_hnd, priv->show);
  if (priv->sub_format != 0)
  {
    SubSource_ReportType(priv->subt_src, priv->sub_format);
  }
  else
  {
    sub_set_cc_channel(priv->subt_hnd, priv->cc_channel);
    GST_INFO_OBJECT(sink, "set cc channel %d %p", priv->cc_channel, priv->subt_hnd);
  }
#endif
  return 0;
}

static int stop_subtitle_handle(GstAmlSubtSink * sink)
{
  GstAmlSubtSinkPrivate *priv = sink->priv;
#if ENABLE_SUB_SERVER
  SubSource_Stop(priv->subt_src);
  sub_stop(priv->subt_hnd);
  if (priv->subt_src) {
    GST_LOG_OBJECT (sink, "free subtitle handle");
    SubSource_Destroy(priv->subt_src);
  }
  priv->subt_src = NULL;
  priv->session_initialized = FALSE;
#endif
  return 0;
}

static int create_subtitle_handle(GstAmlSubtSink * sink)
{
  GstAmlSubtSinkPrivate *priv = sink->priv;
#if ENABLE_SUB_SERVER
  priv->subt_hnd = sub_create();
  GST_INFO_OBJECT(sink, "sub_create: %p", priv->subt_hnd);
  if (!priv->subt_hnd) {
    GST_ERROR ("sub_create fail.");
    return -1;
  }
#endif
  return 0;
}

static int release_subtitle_handle(GstAmlSubtSink * sink)
{
  GstAmlSubtSinkPrivate *priv = sink->priv;
#if ENABLE_SUB_SERVER
  if (priv->subt_hnd)
  {
    GST_LOG_OBJECT (sink, "free cc subtitle handle");
    sub_release(priv->subt_hnd);
  }
  priv->subt_hnd = NULL;
#endif
  return 0;
}

static GstStateChangeReturn
gst_aml_subt_sink_change_state (GstElement * element,
    GstStateChange transition)
{
  GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS;
  GstAmlSubtSink *sink = GST_AML_SUBT_SINK (element);
  GstAmlSubtSinkPrivate *priv = sink->priv;

  switch (transition) {
    case GST_STATE_CHANGE_NULL_TO_READY:
    {
      GST_INFO_OBJECT(sink, "null to ready");
      if (-1 == create_subtitle_handle(sink)) {
        GST_ERROR ("sub_create fail.");
        goto fail_return;
      }
      GST_INFO_OBJECT(sink, "null to ready done");
      break;
    }
    case GST_STATE_CHANGE_READY_TO_PAUSED:
    {
      GST_INFO_OBJECT(sink, "ready to paused");
      gst_base_sink_set_async_enabled (GST_BASE_SINK_CAST(sink), FALSE);
      priv->paused = TRUE;
      start_render_thread(sink);
    }
      break;
    case GST_STATE_CHANGE_PAUSED_TO_PLAYING:
    {
      start_subtitle_handle(sink);
      priv->paused = FALSE;
      GST_INFO_OBJECT(sink, "paused to playing");
      break;
    }
    case GST_STATE_CHANGE_PLAYING_TO_PAUSED:
    {
      priv->paused = TRUE;
      GST_INFO_OBJECT(sink, "playing to paused");
      break;
    }
    case GST_STATE_CHANGE_PAUSED_TO_READY:
    {
      GST_INFO_OBJECT(sink, "paused to ready");
      stop_render_thread(sink);
      stop_eos_thread(sink);
      stop_subtitle_handle(sink);
    }
      break;
    default:
      break;
  }

  ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);

  switch (transition) {
    case GST_STATE_CHANGE_READY_TO_NULL:
    {
      GST_INFO_OBJECT(sink, "ready to null");
      release_subtitle_handle(sink);
    }
      break;
    default:
      break;
  }

  return ret;

fail_return:
  return GST_STATE_CHANGE_FAILURE;

}

gboolean detect_video_sink (GstAmlSubtSink * sink)
{
  GstElement *pipeline = NULL;
  GstElement *element, *elementPrev = NULL;
  GstAmlSubtSinkPrivate *priv = sink->priv;
  gboolean found = FALSE;

  element = GST_ELEMENT_CAST (sink);
  do {
    if (elementPrev)
      gst_object_unref(elementPrev);

    element = GST_ELEMENT_CAST (gst_element_get_parent (element));
    if (element) {
      elementPrev = pipeline;
      pipeline = element;
    }
  } while (element);

  if (pipeline) {
    GstIterator *iter = gst_bin_iterate_recurse (GST_BIN(pipeline));

    if (iter) {
      GValue val = G_VALUE_INIT;

      while (gst_iterator_next (iter, &val) == GST_ITERATOR_OK) {
        element = (GstElement*)g_value_get_object (&val);
        if (element && !GST_IS_BIN(element)) {
          const gchar *name = g_type_name(G_OBJECT_TYPE(element));

          if (name && (!strcmp(name, "GstWesterosSink") || !strcmp(name, "GstAmlVideoSink"))) {
            priv->video_sink = element;
            g_value_reset (&val);
            found = TRUE;
            GST_LOG_OBJECT (sink, "find vsink %s", name);
            break;
          }
        }
        g_value_reset (&val);
      }
      gst_iterator_free(iter);
    }
    gst_object_unref(pipeline);
  }

  return found;
}

static void subt_dump_data(unsigned char* data, int size)
{
#if DUMP_ES_DATA
  if (!data || size < 0)
  {
    //printf (sink, "subt_dump_data: dump data error %d", size);
    return;
  }

  char name[50] = "/data/subtitle_es";
  FILE* fd = fopen(name, "ab");
  if (fd) {
      fwrite(data, 1, size, fd);
      fclose(fd);
  }
#endif
}

static gint64 peek_pts (GstAmlSubtSink * sink)
{
  gint64 pts = 0;
  GstAmlSubtSinkPrivate *priv = sink->priv;
  g_mutex_lock(&priv->data_lock);
  GstBuffer *buf = (GstBuffer *)gst_queue_array_peek_head (priv->buffer_queue);
  if (buf) {
    pts = GST_BUFFER_TIMESTAMP (buf);
  }
  g_mutex_unlock(&priv->data_lock);

  return gst_util_uint64_scale_int (pts, PTS_90K, GST_SECOND);
}

static gint64 get_video_pts (GstAmlSubtSink * sink)
{
  gint64 vpts = 0;
  GstAmlSubtSinkPrivate *priv = sink->priv;
  if (priv->video_sink)
  {
    g_object_get (priv->video_sink, "video-pts", &vpts, NULL); // 90k
  }
  return vpts;
}

static int set_pts_to_subt (GstAmlSubtSink * sink, int64_t pts)
{
  GstAmlSubtSinkPrivate *priv = sink->priv;
  if (priv->sub_format == 0)
  {
    sub_hnd_set_pts(priv->subt_hnd, pts);
  }
  else
  {
    sub_src_set_pts(priv->subt_src, pts);
  }
  return 0;
}

static gpointer subt_render_thread (gpointer data)
{
  GstAmlSubtSink * sink = data;
  GstAmlSubtSinkPrivate *priv = sink->priv;

  struct sched_param schedParam;

  prctl (PR_SET_NAME, "subt_render");
  GST_INFO_OBJECT (sink, "enter");

  if (!detect_video_sink (sink)) {
    GST_ERROR ("find video sink error");
    goto exit;
  }

  schedParam.sched_priority = sched_get_priority_max(SCHED_FIFO) / 2;
  if (pthread_setschedparam (pthread_self(), SCHED_FIFO, &schedParam))
    GST_WARNING ("fail to set subt_render_thread priority");

  while (!priv->quit_render_thread)
  {
    if (priv->paused)
    {
      usleep(50000);
      continue;
    }

    if (priv->sub_format_changed)
    {
      priv->sub_format_changed = FALSE;
      restart_subtitle_handle(sink);
    }

    gint64 vpts = get_video_pts(sink);
#if ENABLE_SUB_SERVER
    //GST_LOG_OBJECT (sink, "set_pts_to_subt:%lld", vpts);
    set_pts_to_subt(sink, vpts);
#endif
    // queue is empty or cc type, continue
    if (priv->sub_format == 0 || !priv->buffer_queue || gst_queue_array_is_empty(priv->buffer_queue))
    {
      // cc has no buffer
      usleep(50000);
      continue;
    }

    // peek pts
    gint64 pts = peek_pts(sink);
    if (pts > vpts + REND_THRESHOLD_H)
    {
      //hold frame
      GST_TRACE_OBJECT (sink, "hold buf pts=%lld, vpts=%lld", pts, vpts);
      usleep(50000);
      continue;
    }

    // pop frame
    g_mutex_lock(&priv->data_lock);
    GstBuffer *buf = (GstBuffer*)gst_queue_array_pop_head(priv->buffer_queue);
    g_mutex_unlock(&priv->data_lock);
    if (pts < vpts - REND_THRESHOLD_L)
    {
      //drop frame
      priv->drop_buf_cnt++;
      GST_LOG_OBJECT (sink, "drop buf pts=%lld, vpts=%lld", pts, vpts);
    }
    else
    {
      // got frame
      priv->out_buf_cnt++;
      GstClockTime time = GST_BUFFER_TIMESTAMP (buf);
      GST_LOG_OBJECT (sink, "got buf pts=%lld, vpts=%lld", pts, vpts);
      GstMapInfo info;
      gst_buffer_map (buf, &info, GST_MAP_READ);
      GST_LOG_OBJECT (sink, "show subtitle: time %" GST_TIME_FORMAT ", size %u", GST_TIME_ARGS (time), info.size);

      subt_dump_data(info.data, info.size);
#if ENABLE_SUB_SERVER
      if (priv->sub_format != 0) {
        if (SUB_STAT_OK != SubSource_SendData(priv->subt_src, (const char *)info.data, info.size, pts, priv->sub_format)) {
          GST_ERROR_OBJECT (sink, "send data failed");
        }
      }
#endif
      gst_buffer_unmap (buf, &info);
    }
    gst_buffer_unref (buf);
  }

exit:
  GST_INFO ("quit");
  return NULL;
}

static int start_render_thread (GstAmlSubtSink * sink)
{
  GstAmlSubtSinkPrivate *priv = sink->priv;

  priv->quit_render_thread = FALSE;
  if (!priv->render_thread) {
    GST_DEBUG_OBJECT (sink, "starting sub render thread");
    priv->render_thread = g_thread_new ("sub_render_thread", subt_render_thread, sink);
    if (!priv->render_thread) {
      GST_ERROR_OBJECT (sink, "fail to create thread");
      return -1;
    }
  }

  priv->render_start = TRUE;
  return 0;
}

static void stop_render_thread (GstAmlSubtSink *sink)
{
  GstAmlSubtSinkPrivate *priv = sink->priv;

  priv->quit_render_thread = TRUE;

  if (priv->render_thread) {
    g_thread_join (priv->render_thread);
    priv->render_thread = NULL;
  }
}

static gpointer subt_eos_thread (gpointer data)
{
  GstAmlSubtSink * sink = data;
  GstAmlSubtSinkPrivate *priv = sink->priv;
  guint32 timeout_cnt = 0;

  prctl (PR_SET_NAME, "subt_eos");
  GST_INFO ("enter");

  while (!priv->quit_eos_thread) {
    usleep (100000);
    if (priv->eos
          && ((priv->in_buf_cnt == priv->drop_buf_cnt + priv->out_buf_cnt) || (timeout_cnt > 30)))
      {
        GstMessage *message;
        GST_WARNING_OBJECT (sink, "Posting EOS, in(%d), drop(%d), out(%d)",
          priv->in_buf_cnt, priv->drop_buf_cnt, priv->out_buf_cnt);
        message = gst_message_new_eos (GST_OBJECT_CAST (sink));
        gst_message_set_seqnum (message, priv->seqnum);
        gst_element_post_message (GST_ELEMENT_CAST (sink), message);

      break;
    }
    timeout_cnt++;
  }

  GST_INFO ("quit");

  return NULL;
}

static int start_eos_thread (GstAmlSubtSink *sink)
{
  GstAmlSubtSinkPrivate *priv = sink->priv;
  priv->quit_eos_thread = FALSE;
  priv->eos_thread = g_thread_new ("subt eos thread", subt_eos_thread, sink);
  if (!priv->eos_thread) {
      GST_ERROR_OBJECT (sink, "fail to create thread");
      return -1;
  }
  return 0;
}

static void stop_eos_thread (GstAmlSubtSink *sink)
{
  GstAmlSubtSinkPrivate *priv = sink->priv;

  priv->quit_eos_thread = TRUE;

  if (priv->eos_thread) {
    g_thread_join (priv->eos_thread);
    priv->eos_thread = NULL;
  }
}

static gboolean subt_set_caps (GstAmlSubtSink *sink, GstCaps *caps)
{
   gboolean result= FALSE;
   GstStructure *structure;
   const gchar *mime;
   int len;
   GstAmlSubtSinkPrivate *priv = sink->priv;

   structure= gst_caps_get_structure(caps, 0);
   if (structure)
   {
      mime= gst_structure_get_name(structure);
      if (mime)
      {
         len= strlen(mime);
         GST_INFO ("len%d, caps:%s", len, mime);
#if ENABLE_SUB_SERVER
         enum CodecID old_format = priv->sub_format;
         if ((len == 17) && !strncmp("application/x-ssa", mime, len))
         {
            priv->sub_format = CODEC_ID_SSA;
            result= TRUE;
         }
         else if ((len == 16) && !strncmp("subpicture/x-dvb", mime, len))
         {
            priv->sub_format = CODEC_ID_DVB_SUBTITLE;
            if (!gst_structure_get_uint (structure, "composition-page-id", &priv->composition_page_id)) {
              GST_WARNING_OBJECT(sink, "can't get composition-page-id from caps");
            }
            if (!gst_structure_get_uint (structure, "ancillary-page-id", &priv->ancillary_page_id)) {
              GST_WARNING_OBJECT(sink, "can't get ancillary-page-id from caps");
            }
            result= TRUE;
         }
         else if ((len == 16) && !strncmp("subpicture/x-dvd", mime, len))
         {
            priv->sub_format = CODEC_ID_DVD_SUBTITLE;
            result= TRUE;
         }
         else if ((len == 16) && !strncmp("subpicture/x-pgs", mime, len))
         {
            priv->sub_format = CODEC_ID_HDMV_PGS_SUBTITLE;
            result= TRUE;
         }
         else if ((len == 10) && !strncmp("text/x-raw", mime, len))
         {
            priv->sub_format = CODEC_ID_SRT;
            result= TRUE;
         }
         else if ((len == 9) && !strncmp("text/x-cc", mime, len))
         {
           // cc no need set type, 0 is unused
            priv->sub_format = 0;
            result= TRUE;
         }
         else
         {
            GST_ERROR("not accepting caps (%s)", mime );
         }

        if (old_format != priv->sub_format && priv->session_initialized)
        {
          // no need flush queue data here, because gst called flush event
          GST_ERROR("old_format:%d, sub_format:%d", old_format, priv->sub_format);
          priv->sub_format_changed = TRUE;
        }
#else
         result = TRUE;
#endif
      }

      if (result == TRUE)
      {
         if (gst_structure_has_field(structure, "codec_data"))
         {
            const GValue *value= gst_structure_get_value(structure, "codec_data");
            if (value)
            {
               GstBuffer *buf;
               GstMapInfo map;
               buf= gst_value_get_buffer(value);
               if (buf)
               {
                  if (gst_buffer_map(buf, &map, GST_MAP_READ))
                  {
                     priv->codec_data= (guint8*)malloc( map.size );
                     if ( priv->codec_data )
                     {
                        memcpy( priv->codec_data, map.data, map.size);
                        priv->codec_data_len= map.size;
                     }
                     else
                     {
                        GST_ERROR("no memory for codec data size %d", map.size);
                     }

                     gst_buffer_unmap(buf, &map);
                  }
                  else
                  {
                     GST_ERROR("gst_buffer_map failed for codec data");
                  }
               }
            }
         }
      }
   }

   return result;
}

static gboolean
plugin_init (GstPlugin * plugin)
{
  return gst_element_register (plugin, "amlsubtsink", GST_RANK_PRIMARY + 1,
      GST_TYPE_AML_SUBT_SINK);
}

#ifndef VERSION
#define VERSION "0.1.0"
#endif
#ifndef PACKAGE
#define PACKAGE "aml_package"
#endif
#ifndef PACKAGE_NAME
#define PACKAGE_NAME "aml_media"
#endif
#ifndef GST_PACKAGE_ORIGIN
#define GST_PACKAGE_ORIGIN "http://amlogic.com"
#endif

GST_PLUGIN_DEFINE (GST_VERSION_MAJOR,
    GST_VERSION_MINOR,
    amlsubtsink,
    "Amlogic plugin for subtitle decoding and rendering",
    plugin_init, VERSION, "LGPL", PACKAGE_NAME, GST_PACKAGE_ORIGIN)