diff --git a/README.md b/README.md index eca9c816..9423f115 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Java Webcam Capture -This library allows you to use your build-in or external webcam directly from Java. +This library allows you to use your build-in or external webcam directly from Java. This fork adds streaming audio in addition to video. [![Build Status](https://secure.travis-ci.org/sarxos/webcam-capture.png?branch=master)](http://travis-ci.org/sarxos/webcam-capture) diff --git a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/agent/StreamClient.java b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/agent/StreamClient.java index 41f32e7c..00826bc3 100644 --- a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/agent/StreamClient.java +++ b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/agent/StreamClient.java @@ -4,39 +4,98 @@ import java.awt.image.BufferedImage; import java.net.InetSocketAddress; +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioSystem; +import javax.sound.sampled.DataLine; +import javax.sound.sampled.LineUnavailableException; +import javax.sound.sampled.SourceDataLine; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.xuggle.xuggler.IAudioSamples; + import us.sosia.video.stream.agent.ui.SingleVideoDisplayWindow; import us.sosia.video.stream.handler.StreamFrameListener; public class StreamClient { - /** - * @author kerr - * */ - private final static Dimension dimension = new Dimension(320,240); - private final static SingleVideoDisplayWindow displayWindow = new SingleVideoDisplayWindow("Stream example",dimension); - protected final static Logger logger = LoggerFactory.getLogger(StreamClient.class); - public static void main(String[] args) { - //setup the videoWindow - displayWindow.setVisible(true); - - //setup the connection - logger.info("setup dimension :{}",dimension); - StreamClientAgent clientAgent = new StreamClientAgent(new StreamFrameListenerIMPL(),dimension); - clientAgent.connect(new InetSocketAddress("localhost", 20000)); - } - - - protected static class StreamFrameListenerIMPL implements StreamFrameListener{ - private volatile long count = 0; - @Override - public void onFrameReceived(BufferedImage image) { - logger.info("frame received :{}",count++); - displayWindow.updateImage(image); - } - - } - + /** + * @author kerr + * */ + private static SourceDataLine mLine; + private static boolean isFirst = true; + private final static Dimension dimension = new Dimension(320,240); + private final static SingleVideoDisplayWindow displayWindow = new SingleVideoDisplayWindow("Stream example",dimension); + protected final static Logger logger = LoggerFactory.getLogger(StreamClient.class); + public static void main(String[] args) { + //setup the videoWindow + displayWindow.setVisible(true); + + //setup the connection + logger.info("setup dimension :{}",dimension); + StreamClientAgent clientAgent = new StreamClientAgent(new StreamFrameListenerIMPL(),dimension); + clientAgent.connect(new InetSocketAddress("localhost", 20000)); + try { + openJavaSound(); + } catch (LineUnavailableException e) { + e.printStackTrace(); + } + } + + + protected static class StreamFrameListenerIMPL implements StreamFrameListener{ + private volatile long count = 0; + @Override + public void onFrameReceived(BufferedImage image) { + logger.info("frame received :{}",count++); + displayWindow.updateImage(image); + } + @Override + public void onAudioRecieved(IAudioSamples samples) { + playJavaSound(samples); + } + } + + private static void openJavaSound() throws LineUnavailableException + { + AudioFormat audioFormat = new AudioFormat( + AudioFormat.Encoding.PCM_SIGNED, + 44100.0F, 16, 2, 4, 44100, false); + + AudioFormat format = new AudioFormat(44100, + 16, + 2, + true, /* xuggler defaults to signed 16 bit samples */ + false); + DataLine.Info info = new DataLine.Info(SourceDataLine.class, format); + mLine = (SourceDataLine) AudioSystem.getLine(info); + /** + * if that succeeded, try opening the line. + */ + mLine.open(audioFormat); + /** + * And if that succeed, start the line. + */ + mLine.start(); + + + } + + private static synchronized void playJavaSound(IAudioSamples aSamples) + { + if (isFirst) { + isFirst = false; + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + /** + * We're just going to dump all the samples into the line. + */ + byte[] rawBytes = aSamples.getData().getByteArray(0, aSamples.getSize()); + mLine.write(rawBytes, 0, rawBytes.length); + } } diff --git a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/agent/StreamServerAgent.java b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/agent/StreamServerAgent.java index ba925bf6..3ec96eb2 100644 --- a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/agent/StreamServerAgent.java +++ b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/agent/StreamServerAgent.java @@ -2,7 +2,10 @@ import java.awt.Dimension; import java.awt.image.BufferedImage; +import java.io.ByteArrayOutputStream; import java.net.SocketAddress; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -10,6 +13,12 @@ import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioSystem; +import javax.sound.sampled.DataLine; +import javax.sound.sampled.LineUnavailableException; +import javax.sound.sampled.TargetDataLine; + import org.jboss.netty.bootstrap.ServerBootstrap; import org.jboss.netty.channel.Channel; import org.jboss.netty.channel.group.ChannelGroup; @@ -25,160 +34,213 @@ import com.github.sarxos.webcam.Webcam; public class StreamServerAgent implements IStreamServerAgent{ - protected final static Logger logger = LoggerFactory.getLogger(StreamServer.class); - protected final Webcam webcam; - protected final Dimension dimension; - protected final ChannelGroup channelGroup = new DefaultChannelGroup(); - protected final ServerBootstrap serverBootstrap; - //I just move the stream encoder out of the channel pipeline for the performance - protected final H264StreamEncoder h264StreamEncoder; - protected volatile boolean isStreaming; - protected ScheduledExecutorService timeWorker; - protected ExecutorService encodeWorker; - protected int FPS = 25; - protected ScheduledFuture imageGrabTaskFuture; - public StreamServerAgent(Webcam webcam, Dimension dimension) { - super(); - this.webcam = webcam; - this.dimension = dimension; - //this.h264StreamEncoder = new H264StreamEncoder(dimension,false); - this.serverBootstrap = new ServerBootstrap(); - this.serverBootstrap.setFactory(new NioServerSocketChannelFactory( - Executors.newCachedThreadPool(), - Executors.newCachedThreadPool())); - this.serverBootstrap.setPipelineFactory(new StreamServerChannelPipelineFactory( - new StreamServerListenerIMPL(), - dimension)); - this.timeWorker = new ScheduledThreadPoolExecutor(1); - this.encodeWorker = Executors.newSingleThreadExecutor(); - this.h264StreamEncoder = new H264StreamEncoder(dimension, false); - } - - - - public int getFPS() { - return FPS; - } - - public void setFPS(int fPS) { - FPS = fPS; - } - - @Override - public void start(SocketAddress streamAddress) { - logger.info("Server started :{}",streamAddress); - Channel channel = serverBootstrap.bind(streamAddress); - channelGroup.add(channel); - } - - @Override - public void stop() { - logger.info("server is stoping"); - channelGroup.close(); - timeWorker.shutdown(); - encodeWorker.shutdown(); - serverBootstrap.releaseExternalResources(); - } - - - private class StreamServerListenerIMPL implements StreamServerListener{ + protected final static Logger logger = LoggerFactory.getLogger(StreamServer.class); + protected final Webcam webcam; + protected final Dimension dimension; + protected final static ChannelGroup channelGroup = new DefaultChannelGroup(); + protected final ServerBootstrap serverBootstrap; + //I just move the stream encoder out of the channel pipeline for the performance + protected final H264StreamEncoder h264StreamEncoder; + protected final H264StreamEncoder secondEncoder; + protected volatile boolean isStreaming; + protected ScheduledExecutorService timeWorker; + protected ExecutorService encodeWorker; + protected int FPS = 25; + protected ScheduledFuture imageGrabTaskFuture; + protected TargetDataLine line; + public StreamServerAgent(Webcam webcam, Dimension dimension) { + super(); + this.webcam = webcam; + this.dimension = dimension; + //this.h264StreamEncoder = new H264StreamEncoder(dimension,false); + this.serverBootstrap = new ServerBootstrap(); + this.serverBootstrap.setFactory(new NioServerSocketChannelFactory( + Executors.newCachedThreadPool(), + Executors.newCachedThreadPool())); + this.serverBootstrap.setPipelineFactory(new StreamServerChannelPipelineFactory( + new StreamServerListenerIMPL(), + dimension)); + this.timeWorker = new ScheduledThreadPoolExecutor(1); + this.encodeWorker = Executors.newSingleThreadExecutor(); + this.h264StreamEncoder = new H264StreamEncoder(dimension, false); + this.secondEncoder = new H264StreamEncoder(dimension, false); - @Override - public void onClientConnectedIn(Channel channel) { - //here we just start to stream when the first client connected in - // - channelGroup.add(channel); - if (!isStreaming) { - //do some thing - Runnable imageGrabTask = new ImageGrabTask(); - ScheduledFuture imageGrabFuture = - timeWorker.scheduleWithFixedDelay(imageGrabTask, - 0, - 1000/FPS, - TimeUnit.MILLISECONDS); - imageGrabTaskFuture = imageGrabFuture; - isStreaming = true; - } - logger.info("current connected clients :{}",channelGroup.size()); + AudioFormat format = new AudioFormat( + AudioFormat.Encoding.PCM_SIGNED, + 44100.0F, 16, 2, 4, 44100, false); + + this.line = null; + + // format is an AudioFormat object + DataLine.Info info = new DataLine.Info(TargetDataLine.class, + format); + if (!AudioSystem.isLineSupported(info)) { + System.out.println("Error, line not supported"); + } + // Obtain and open the line. + try { + + this.line = (TargetDataLine) AudioSystem.getLine(info); + this.line.open(format); + } catch (LineUnavailableException ex) { + ex.printStackTrace(); + } + + this.line.start(); } - @Override - public void onClientDisconnected(Channel channel) { - channelGroup.remove(channel); - int size = channelGroup.size(); - logger.info("current connected clients :{}",size); - if (size == 1) { - //cancel the task - imageGrabTaskFuture.cancel(false); - webcam.close(); - isStreaming = false; - } + + + public int getFPS() { + return FPS; + } + + public void setFPS(int fPS) { + FPS = fPS; } @Override - public void onExcaption(Channel channel, Throwable t) { - channelGroup.remove(channel); - channel.close(); - int size = channelGroup.size(); - logger.info("current connected clients :{}",size); - if (size == 1) { - //cancel the task - imageGrabTaskFuture.cancel(false); - webcam.close(); - isStreaming = false; - + public void start(SocketAddress streamAddress) { + logger.info("Server started :{}",streamAddress); + Channel channel = serverBootstrap.bind(streamAddress); + channelGroup.add(channel); } - - } - - protected volatile long frameCount = 0; - - private class ImageGrabTask implements Runnable{ @Override - public void run() { - logger.info("image grabed ,count :{}",frameCount++); - BufferedImage bufferedImage = webcam.getImage(); - /** - * using this when the h264 encoder is added to the pipeline - * */ - //channelGroup.write(bufferedImage); - /** - * using this when the h264 encoder is inside this class - * */ - encodeWorker.execute(new EncodeTask(bufferedImage)); + public void stop() { + logger.info("server is stoping"); + channelGroup.close(); + timeWorker.shutdown(); + encodeWorker.shutdown(); + serverBootstrap.releaseExternalResources(); } - - } - - private class EncodeTask implements Runnable{ - private final BufferedImage image; - - public EncodeTask(BufferedImage image) { - super(); - this.image = image; + + private static void writeData( Object data) { + channelGroup.write(data); } - @Override - public void run() { - try { - Object msg = h264StreamEncoder.encode(image); - if (msg != null) { - channelGroup.write(msg); + + private class StreamServerListenerIMPL implements StreamServerListener{ + + @Override + public void onClientConnectedIn(Channel channel) { + //here we just start to stream when the first client connected in + channelGroup.add(channel); + if (!isStreaming) { + //do some thing + Runnable imageGrabTask = new ImageGrabTask(); + ScheduledFuture imageGrabFuture = + timeWorker.scheduleWithFixedDelay(imageGrabTask, + 0, + 1000/FPS, + TimeUnit.MILLISECONDS); + imageGrabTaskFuture = imageGrabFuture; + AudioGrabTask at = new AudioGrabTask(); + at.start(); + isStreaming = true; + } + logger.info("current connected clients :{}",channelGroup.size()); } - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - - } - - } - - - - + @Override + public void onClientDisconnected(Channel channel) { + channelGroup.remove(channel); + int size = channelGroup.size(); + logger.info("current connected clients :{}",size); + if (size == 1) { + //cancel the task + imageGrabTaskFuture.cancel(false); + webcam.close(); + isStreaming = false; + } + } + + @Override + public void onExcaption(Channel channel, Throwable t) { + channelGroup.remove(channel); + channel.close(); + int size = channelGroup.size(); + logger.info("current connected clients :{}",size); + if (size == 1) { + //cancel the task + imageGrabTaskFuture.cancel(false); + webcam.close(); + isStreaming = false; + + } + + } + + protected volatile long frameCount = 0; + + private class ImageGrabTask implements Runnable{ + + @Override + public void run() { + logger.info("image grabed ,count :{}",frameCount++); + BufferedImage bufferedImage = webcam.getImage(); + /** + * using this when the h264 encoder is added to the pipeline + * */ + //channelGroup.write(bufferedImage); + /** + * using this when the h264 encoder is inside this class + * */ + encodeWorker.execute(new EncodeTask(bufferedImage)); + } + + } + + public class AudioGrabTask extends Thread { + public void run() { + int numBytesRead; + byte[] data = new byte [4096]; + while (true) { + if (data.length > 0 ){ + System.out.println("start read: " + System.currentTimeMillis()); + numBytesRead = line.read(data, 0, 4096); + System.out.println("end read: " + System.currentTimeMillis()); + Object msg2 = null; + try { + if ( numBytesRead > 0 ) { + msg2 = secondEncoder.encode(data,numBytesRead); + } + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + if ( msg2 != null ) { + System.out.println("writing audio"); + writeData(msg2); + } + Thread.yield(); + } + } + } + } + + + private class EncodeTask implements Runnable{ + private final BufferedImage image; + public EncodeTask(BufferedImage image) { + super(); + this.image = image; + } + + @Override + public void run() { + try { + Object msg = h264StreamEncoder.encode(image); + if (msg != null) { + writeData(msg); + } + + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } + } } diff --git a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/H264StreamDecoder.java b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/H264StreamDecoder.java index 4c4a9d0f..194756a7 100644 --- a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/H264StreamDecoder.java +++ b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/H264StreamDecoder.java @@ -15,6 +15,7 @@ import us.sosia.video.stream.handler.frame.FrameDecoder; import com.xuggle.ferry.IBuffer; +import com.xuggle.xuggler.IAudioSamples; import com.xuggle.xuggler.ICodec; import com.xuggle.xuggler.IPacket; import com.xuggle.xuggler.IPixelFormat; @@ -28,222 +29,245 @@ * This codec will encode the bufferedImage to h264 stream * **/ public class H264StreamDecoder extends OneToOneDecoder{ - protected final static Logger logger = LoggerFactory.getLogger(H264StreamDecoder.class); - protected final IStreamCoder iStreamCoder = IStreamCoder.make(Direction.DECODING, ICodec.ID.CODEC_ID_H264); - protected final Type type = ConverterFactory.findRegisteredConverter(ConverterFactory.XUGGLER_BGR_24); - protected final StreamFrameListener streamFrameListener; - protected final Dimension dimension; + protected final static Logger logger = LoggerFactory.getLogger(H264StreamDecoder.class); + protected final IStreamCoder iStreamCoder = IStreamCoder.make(Direction.DECODING, ICodec.ID.CODEC_ID_H264); + protected final IStreamCoder iAudioStreamCoder = IStreamCoder.make(Direction.DECODING, ICodec.ID.CODEC_ID_AAC); + protected final Type type = ConverterFactory.findRegisteredConverter(ConverterFactory.XUGGLER_BGR_24); + protected final StreamFrameListener streamFrameListener; + protected final Dimension dimension; - protected final FrameDecoder frameDecoder; - protected final ExecutorService decodeWorker ; - /** - * - * Cause there may be one or more image in the frame,so we need an Stream listener here to get all the image - * - * */ - - - - - - public H264StreamDecoder(StreamFrameListener streamFrameListener, - Dimension dimension,boolean internalFrameDecoder,boolean decodeInOtherThread) { - super(); - this.streamFrameListener = streamFrameListener; - this.dimension = dimension; - if (internalFrameDecoder) { - frameDecoder = new FrameDecoder(4); - }else { - frameDecoder = null; - } - if (decodeInOtherThread) { - decodeWorker = Executors.newSingleThreadExecutor(); - }else { - decodeWorker = null; - } + protected final FrameDecoder frameDecoder; + protected final ExecutorService decodeWorker ; + /** + * + * Cause there may be one or more image in the frame,so we need an Stream listener here to get all the image + * + * */ + public H264StreamDecoder(StreamFrameListener streamFrameListener, + Dimension dimension,boolean internalFrameDecoder,boolean decodeInOtherThread) { + super(); + this.streamFrameListener = streamFrameListener; + this.dimension = dimension; + if (internalFrameDecoder) { + frameDecoder = new FrameDecoder(4); + }else { + frameDecoder = null; + } + if (decodeInOtherThread) { + decodeWorker = Executors.newSingleThreadExecutor(); + }else { + decodeWorker = null; + } - initialize(); - } + initialize(); + } + private void initialize(){ + iStreamCoder.open(null, null); + iAudioStreamCoder.open(null, null); + } + @Override + protected Object decode(ChannelHandlerContext ctx, Channel channel, + final Object msg) throws Exception { + if (decodeWorker != null) { + decodeWorker.execute(new decodeTask(msg)); + return null; + } + else { + if (msg == null) { + throw new NullPointerException("you cannot pass into an null to the decode"); + } + ChannelBuffer frameBuffer; + if (frameDecoder != null) { + frameBuffer = frameDecoder.decode((ChannelBuffer)msg); + if (frameBuffer == null) { + return null; + } - private void initialize(){ - //iStreamCoder.setNumPicturesInGroupOfPictures(20); - //iStreamCoder.setBitRate(250000); - //iStreamCoder.setBitRateTolerance(9000); - //iStreamCoder.setPixelType(IPixelFormat.Type.YUV420P); - //iStreamCoder.setHeight(dimension.height); - //iStreamCoder.setWidth(dimension.width); - //iStreamCoder.setFlag(IStreamCoder.Flags.FLAG_QSCALE, true); - //iStreamCoder.setGlobalQuality(0); - //rate - //IRational rate = IRational.make(25, 1); - //iStreamCoder.setFrameRate(rate); - //time base - //iStreamCoder.setAutomaticallyStampPacketsForStream(true); - //iStreamCoder.setTimeBase(IRational.make(rate.getDenominator(),rate.getNumerator())); - iStreamCoder.open(null, null); - } - - + } + else { + frameBuffer = (ChannelBuffer)msg; + } - - - - @Override - protected Object decode(ChannelHandlerContext ctx, Channel channel, - final Object msg) throws Exception { - if (decodeWorker != null) { - decodeWorker.execute(new decodeTask(msg)); - return null; - }else { - if (msg == null) { - throw new NullPointerException("you cannot pass into an null to the decode"); - } - ChannelBuffer frameBuffer; - if (frameDecoder != null) { - frameBuffer = frameDecoder.decode((ChannelBuffer)msg); - if (frameBuffer == null) { - return null; - } - - }else { - frameBuffer = (ChannelBuffer)msg; - } + int size = frameBuffer.readableBytes(); + logger.info("decode the frame size :{}",size); - int size = frameBuffer.readableBytes(); - logger.info("decode the frame size :{}",size); - //start to decode - IBuffer iBuffer = IBuffer.make(null, size); - IPacket iPacket = IPacket.make(iBuffer); - iPacket.getByteBuffer().put(frameBuffer.toByteBuffer()); - //decode the packet - if (!iPacket.isComplete()) { - return null; - } + //start to decode + IBuffer iBuffer = IBuffer.make(null, size); + IPacket iPacket = IPacket.make(iBuffer); + iPacket.getByteBuffer().put(frameBuffer.toByteBuffer()); + //decode the packet + if (!iPacket.isComplete()) { + return null; + } + logger.info("packet stream index: " + iPacket.getFlags()); - IVideoPicture picture = IVideoPicture.make(IPixelFormat.Type.YUV420P, - dimension.width, dimension.height); - try { - // decode the packet into the video picture - int postion = 0; - int packageSize = iPacket.getSize(); - while(postion < packageSize){ - postion+= iStreamCoder.decodeVideo(picture, iPacket, postion); - if (postion < 0) - throw new RuntimeException("error " - + " decoding video"); - // if this is a complete picture, dispatch the picture - if (picture.isComplete()){ - IConverter converter = ConverterFactory.createConverter(type - .getDescriptor(), picture); - BufferedImage image = converter.toImage(picture); - //BufferedImage convertedImage = ImageUtils.convertToType(image, BufferedImage.TYPE_3BYTE_BGR); - //here ,put out the image - if (streamFrameListener != null) { - streamFrameListener.onFrameReceived(image); - } - converter.delete(); - }else{ - picture.delete(); - iPacket.delete(); - return null; - } - //clean the picture and reuse it - picture.getByteBuffer().clear(); - } - } finally { - if (picture != null) - picture.delete(); - iPacket.delete(); - // ByteBufferUtil.destroy(data); - } - return null; - } - } + if ( iPacket.getByteBuffer().get() != -1 ) { + IVideoPicture picture = IVideoPicture.make(IPixelFormat.Type.YUV420P, + dimension.width, dimension.height); + try { + // decode the packet into the video picture + int postion = 0; + int packageSize = iPacket.getSize(); + while(postion < packageSize){ + postion+= iStreamCoder.decodeVideo(picture, iPacket, postion); + if (postion < 0) + throw new RuntimeException("error " + + " decoding video"); + // if this is a complete picture, dispatch the picture + if (picture.isComplete()){ + IConverter converter = ConverterFactory.createConverter(type + .getDescriptor(), picture); + BufferedImage image = converter.toImage(picture); + //BufferedImage convertedImage = ImageUtils.convertToType(image, BufferedImage.TYPE_3BYTE_BGR); + //here ,put out the image + if (streamFrameListener != null) { + streamFrameListener.onFrameReceived(image); + } + converter.delete(); + } + else { + picture.delete(); + iPacket.delete(); + return null; + } + //clean the picture and reuse it + picture.getByteBuffer().clear(); + } + } finally { + if (picture != null) + picture.delete(); + iPacket.delete(); + // ByteBufferUtil.destroy(data); + } + } + else { + IAudioSamples samples = IAudioSamples.make(1024, 1); + + /* + * A packet can actually contain multiple sets of samples (or frames of samples + * in audio-decoding speak). So, we may need to call decode audio multiple + * times at different offsets in the packet's data. We capture that here. + */ + int offset = 0; + + /* + * Keep going until we've processed all data + */ + while(offset < iPacket.getSize()) { + int bytesDecoded = iAudioStreamCoder.decodeAudio(samples, iPacket, offset); + if (bytesDecoded < 0) + throw new RuntimeException("got error decoding audio in stream"); + + offset += bytesDecoded; + + /* + * Some decoder will consume data in a packet, but will not be able to construct + * a full set of samples yet. Therefore you should always check if you + * got a complete set of samples from the decoder + */ + if (samples.isComplete()) { + if (streamFrameListener != null) { + streamFrameListener.onAudioRecieved(samples); + } + } + } + } + return null; + } + } - private class decodeTask implements Runnable{ - private final Object msg; + private class decodeTask implements Runnable{ + private final Object msg; - public decodeTask(Object msg) { - super(); - this.msg = msg; - } + public decodeTask(Object msg) { + super(); + this.msg = msg; + } - @Override - public void run() { - if (msg == null) { - return; - } - ChannelBuffer frameBuffer; - if (frameDecoder != null) { - try { - frameBuffer = frameDecoder.decode((ChannelBuffer)msg); - if (frameBuffer == null) { - return ; - } - } catch (Exception e) { - return; - } + @Override + public void run() { + if (msg == null) { + return; + } + ChannelBuffer frameBuffer; + if (frameDecoder != null) { + try { + frameBuffer = frameDecoder.decode((ChannelBuffer)msg); + if (frameBuffer == null) { + return ; + } + } catch (Exception e) { + return; + } - }else { - frameBuffer = (ChannelBuffer)msg; - } - - int size = frameBuffer.readableBytes(); - logger.info("decode the frame size :{}",size); - //start to decode - IBuffer iBuffer = IBuffer.make(null, size); - IPacket iPacket = IPacket.make(iBuffer); - iPacket.getByteBuffer().put(frameBuffer.toByteBuffer()); - //decode the packet - if (!iPacket.isComplete()) { - return ; - } - + } + else { + frameBuffer = (ChannelBuffer)msg; + } - IVideoPicture picture = IVideoPicture.make(IPixelFormat.Type.YUV420P, - dimension.width, dimension.height); - try { - // decode the packet into the video picture - int postion = 0; - int packageSize = iPacket.getSize(); - while(postion < packageSize){ - postion+= iStreamCoder.decodeVideo(picture, iPacket, postion); - if (postion < 0) - throw new RuntimeException("error " - + " decoding video"); - // if this is a complete picture, dispatch the picture - if (picture.isComplete()){ - IConverter converter = ConverterFactory.createConverter(type - .getDescriptor(), picture); - BufferedImage image = converter.toImage(picture); - //BufferedImage convertedImage = ImageUtils.convertToType(image, BufferedImage.TYPE_3BYTE_BGR); - //here ,put out the image - if (streamFrameListener != null) { - streamFrameListener.onFrameReceived(image); - } - converter.delete(); - }else{ - picture.delete(); - iPacket.delete(); - return ; - } - //clean the picture and reuse it - picture.getByteBuffer().clear(); - } - } finally { - if (picture != null) - picture.delete(); - iPacket.delete(); - // ByteBufferUtil.destroy(data); - } - return ; - } + int size = frameBuffer.readableBytes(); + logger.info("decode the frame size :{}",size); + //start to decode + IBuffer iBuffer = IBuffer.make(null, size); + IPacket iPacket = IPacket.make(iBuffer); + iPacket.getByteBuffer().put(frameBuffer.toByteBuffer()); + //decode the packet + if (!iPacket.isComplete()) { + return ; + } + logger.info("packet stream index: " + iPacket.getStreamIndex()); + if ( iPacket.getStreamIndex() == 0 ) { + IVideoPicture picture = IVideoPicture.make(IPixelFormat.Type.YUV420P, + dimension.width, dimension.height); + try { + // decode the packet into the video picture + int postion = 0; + int packageSize = iPacket.getSize(); + while(postion < packageSize){ + postion+= iStreamCoder.decodeVideo(picture, iPacket, postion); + if (postion < 0) { + throw new RuntimeException("error " + + " decoding video"); + } + // if this is a complete picture, dispatch the picture + if (picture.isComplete()){ + IConverter converter = ConverterFactory.createConverter(type + .getDescriptor(), picture); + BufferedImage image = converter.toImage(picture); + //BufferedImage convertedImage = ImageUtils.convertToType(image, BufferedImage.TYPE_3BYTE_BGR); + //here ,put out the image + if (streamFrameListener != null) { + streamFrameListener.onFrameReceived(image); + } + converter.delete(); + } + else { + picture.delete(); + iPacket.delete(); + return ; + } + //clean the picture and reuse it + picture.getByteBuffer().clear(); + } + } finally { + if (picture != null) { + picture.delete(); + } + iPacket.delete(); + } + return ; + } + else { + iPacket.delete(); + logger.info("got audio frame"); + } + } - } + } diff --git a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/H264StreamEncoder.java b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/H264StreamEncoder.java index f4448e99..f44b1087 100644 --- a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/H264StreamEncoder.java +++ b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/H264StreamEncoder.java @@ -4,6 +4,9 @@ import java.awt.image.BufferedImage; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.util.Arrays; + +import javax.sound.sampled.AudioFormat; import org.jboss.netty.buffer.ChannelBuffer; import org.jboss.netty.buffer.ChannelBuffers; @@ -15,6 +18,8 @@ import us.sosia.video.stream.handler.frame.FrameEncoder; +import com.xuggle.ferry.IBuffer; +import com.xuggle.xuggler.IAudioSamples; import com.xuggle.xuggler.ICodec; import com.xuggle.xuggler.IMetaData; import com.xuggle.xuggler.IPacket; @@ -27,105 +32,151 @@ import com.xuggle.xuggler.video.IConverter; public class H264StreamEncoder extends OneToOneEncoder{ - protected final static Logger logger = LoggerFactory.getLogger(Logger.class); - protected final IStreamCoder iStreamCoder = IStreamCoder.make(Direction.ENCODING, ICodec.ID.CODEC_ID_H264); - protected final IPacket iPacket = IPacket.make(); - protected long startTime ; - protected final Dimension dimension; - protected final FrameEncoder frameEncoder; - - - public H264StreamEncoder(Dimension dimension,boolean usingInternalFrameEncoder) { - super(); - this.dimension = dimension; - if (usingInternalFrameEncoder) { - frameEncoder = new FrameEncoder(4); - }else { - frameEncoder = null; - } - initialize(); - } - - private void initialize(){ - //setup - iStreamCoder.setNumPicturesInGroupOfPictures(25); - - iStreamCoder.setBitRate(200000); - iStreamCoder.setBitRateTolerance(10000); - iStreamCoder.setPixelType(Type.YUV420P); - iStreamCoder.setHeight(dimension.height); - iStreamCoder.setWidth(dimension.width); - iStreamCoder.setFlag(IStreamCoder.Flags.FLAG_QSCALE, true); - iStreamCoder.setGlobalQuality(0); - //rate - IRational rate = IRational.make(25, 1); - iStreamCoder.setFrameRate(rate); - //time base - //iStreamCoder.setAutomaticallyStampPacketsForStream(true); - iStreamCoder.setTimeBase(IRational.make(rate.getDenominator(),rate.getNumerator())); - IMetaData codecOptions = IMetaData.make(); - codecOptions.setValue("tune", "zerolatency");// equals -tune zerolatency in ffmpeg - //open it - int revl = iStreamCoder.open(codecOptions, null); - if (revl < 0) { - throw new RuntimeException("could not open the coder"); - } - } - - - - @Override - protected Object encode(ChannelHandlerContext ctx, Channel channel, - Object msg) throws Exception { - return encode(msg); - } - - public Object encode(Object msg) throws Exception { - if (msg == null) { - return null; - } - if (!(msg instanceof BufferedImage)) { - throw new IllegalArgumentException("your need to pass into an bufferedimage"); - } - logger.info("encode the frame"); - BufferedImage bufferedImage = (BufferedImage)msg; - //here is the encode - //convert the image - BufferedImage convetedImage = ImageUtils.convertToType(bufferedImage, BufferedImage.TYPE_3BYTE_BGR); - IConverter converter = ConverterFactory.createConverter(convetedImage, Type.YUV420P); - //to frame - long now = System.currentTimeMillis(); - if (startTime == 0) { - startTime = now; - } - IVideoPicture pFrame = converter.toPicture(convetedImage, (now - startTime)*1000); - //pFrame.setQuality(0); - iStreamCoder.encodeVideo(iPacket, pFrame, 0) ; - //free the MEM - pFrame.delete(); - converter.delete(); - //write to the container - if (iPacket.isComplete()) { - - //iPacket.delete(); - //here we send the package to the remote peer - try{ - ByteBuffer byteBuffer = iPacket.getByteBuffer(); - if (iPacket.isKeyPacket()) { - logger.info("key frame"); - } - ChannelBuffer channelBuffe = ChannelBuffers.copiedBuffer(byteBuffer.order(ByteOrder.BIG_ENDIAN)); - if (frameEncoder != null) { - return frameEncoder.encode(channelBuffe); - } - return channelBuffe; - - }finally{ - iPacket.reset(); - } - }else{ - return null; - } - } + protected final static Logger logger = LoggerFactory.getLogger(Logger.class); + protected final IStreamCoder iStreamCoder = IStreamCoder.make(Direction.ENCODING, ICodec.ID.CODEC_ID_H264); + protected final IStreamCoder iAudioStreamCoder = IStreamCoder.make(Direction.ENCODING, ICodec.ID.CODEC_ID_AAC); + protected final IPacket iPacket = IPacket.make(); + protected long startTime ; + protected final Dimension dimension; + protected final FrameEncoder frameEncoder; + + protected final AudioFormat format = new AudioFormat( + AudioFormat.Encoding.PCM_SIGNED, + 44100.0F, 16, 2, 4, 44100, false); + + + public H264StreamEncoder(Dimension dimension,boolean usingInternalFrameEncoder) { + super(); + this.dimension = dimension; + if (usingInternalFrameEncoder) { + frameEncoder = new FrameEncoder(4); + }else { + frameEncoder = null; + } + initialize(); + } + + private void initialize(){ + //setup + iStreamCoder.setNumPicturesInGroupOfPictures(25); + + iStreamCoder.setBitRate(200000); + iStreamCoder.setBitRateTolerance(10000); + iStreamCoder.setPixelType(Type.YUV420P); + iStreamCoder.setHeight(dimension.height); + iStreamCoder.setWidth(dimension.width); + iStreamCoder.setFlag(IStreamCoder.Flags.FLAG_QSCALE, true); + iStreamCoder.setGlobalQuality(0); + //rate + IRational rate = IRational.make(25, 1); + iStreamCoder.setFrameRate(rate); + //time base + iStreamCoder.setTimeBase(IRational.make(rate.getDenominator(),rate.getNumerator())); + IMetaData codecOptions = IMetaData.make(); + codecOptions.setValue("tune", "zerolatency");// equals -tune zerolatency in ffmpeg + //open it + int revl = iStreamCoder.open(codecOptions, null); + if (revl < 0) { + throw new RuntimeException("could not open the coder"); + } + + iStreamCoder.setNumPicturesInGroupOfPictures(25); + + + iAudioStreamCoder.setChannels(2); + iAudioStreamCoder.setSampleRate(44100); + IRational ratea = IRational.make(44100, 1); + IMetaData codecOptionsa = IMetaData.make(); + revl = iAudioStreamCoder.open(codecOptionsa, null); + if (revl < 0) { + throw new RuntimeException("could not open the audio coder"); + } + } + + + + @Override + protected Object encode(ChannelHandlerContext ctx, Channel channel, + Object msg) throws Exception { + return encode(msg); + } + + public Object encode(Object msg) throws Exception { + if (msg == null) { + return null; + } + if (!(msg instanceof BufferedImage)) { + throw new IllegalArgumentException("your need to pass into an bufferedimage"); + } + logger.info("encode the frame"); + BufferedImage bufferedImage = (BufferedImage)msg; + //here is the encode + //convert the image + BufferedImage convetedImage = ImageUtils.convertToType(bufferedImage, BufferedImage.TYPE_3BYTE_BGR); + IConverter converter = ConverterFactory.createConverter(convetedImage, Type.YUV420P); + //to frame + long now = System.currentTimeMillis(); + if (startTime == 0) { + startTime = now; + } + IVideoPicture pFrame = converter.toPicture(convetedImage, (now - startTime)*1000); + iStreamCoder.encodeVideo(iPacket, pFrame, 0) ; + //free the MEM + pFrame.delete(); + converter.delete(); + //write to the container + iPacket.setStreamIndex(0); + if (iPacket.isComplete()) { + iPacket.setFlags(1); + //here we send the package to the remote peer + try{ + ByteBuffer byteBuffer = iPacket.getByteBuffer(); + if (iPacket.isKeyPacket()) { + logger.info("key frame"); + } + ChannelBuffer channelBuffe = ChannelBuffers.copiedBuffer(byteBuffer.order(ByteOrder.BIG_ENDIAN)); + if (frameEncoder != null) { + return channelBuffe; + } + return channelBuffe; + + }finally{ + iPacket.reset(); + } + }else{ + return null; + } + } + + public Object encode( byte[] data, int numBytesRead) throws Exception { + IBuffer iBuf = IBuffer.make(null, data, 0, numBytesRead); + + IAudioSamples smp = IAudioSamples.make(iBuf,2,IAudioSamples.Format.FMT_S16); + smp.setComplete(true, numBytesRead/4, 44100, 2, IAudioSamples.Format.FMT_S16, 0); + iAudioStreamCoder.encodeAudio(iPacket, smp, 0); + + //write to the container + if (iPacket.isComplete()) { + //here we send the package to the remote peer + try{ + ByteBuffer byteBuffer = iPacket.getByteBuffer(); + if (iPacket.isKeyPacket()) { + logger.info("key frame"); + } + ChannelBuffer channelBuffe = ChannelBuffers.copiedBuffer(byteBuffer.order(ByteOrder.BIG_ENDIAN)); + if (frameEncoder != null) { + return frameEncoder.encode(channelBuffe); + } + return channelBuffe; + + } + finally { + iPacket.reset(); + } + } + else { + return encode(data,numBytesRead); + } + } } diff --git a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/StreamFrameListener.java b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/StreamFrameListener.java index e90b50d5..fd39712f 100644 --- a/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/StreamFrameListener.java +++ b/webcam-capture-examples/webcam-capture-live-streaming/src/main/java/us/sosia/video/stream/handler/StreamFrameListener.java @@ -2,10 +2,14 @@ import java.awt.image.BufferedImage; +import com.xuggle.xuggler.IAudioSamples; + public interface StreamFrameListener { /** * Callback when the image is received from the live stream. * @param image The received and decoded image * */ public void onFrameReceived(BufferedImage image); + + public void onAudioRecieved(IAudioSamples samples); }