Geek Answers Handbook

Streaming WebCamTexture with the new Unity 5.1 transport layer API

As the question says, I am trying to transfer a WebCamTexture from a client with a webcam to a server. Both sides (client and server) are in Unity. Later, the client will be deployed to Android, and the server will be a desktop application.

I am currently getting texture pixels using:

tex.GetPixels32();

and serialize them using a special serializer (to optimize its size). Currently, I have an uncompressed byte array of about 3.5 MB per frame, ready to be sent. I know this is huge, but I wanted it to be transmitted before starting work with the compression part and the real-time part.

The last part of the process should be sent using the new static Unity NetworkTransport class. It has been a long time since I used the sockets, and I'm really rotten. Currently I can’t work.

Here is my server side code (omits the serialization code for clarity):

void Start()
{
    webcamTexture = new WebCamTexture();
    Background.texture = webcamTexture;
    Background.material.mainTexture = webcamTexture;
    webcamTexture.Play();

    if (!_isStarted)
    {
        _isStarted = true;

        NetworkTransport.Init();
        m_Config = new ConnectionConfig();
        m_CommunicationChannel  = m_Config.AddChannel(QosType.ReliableFragmented);
        HostTopology topology = new HostTopology(m_Config, 12);
        m_GenericHostId = NetworkTransport.AddHost(topology, 0);
        byte error;
        m_ConnectionId = NetworkTransport.Connect(m_GenericHostId, ip, port, 0, out error);
    }
}

void Update()
{
    if (!_isStarted)
        return;

    NetworkEventType recData = NetworkTransport.Receive(out recHostId, out connectionId, out channelId, recBuffer, bufferSize, out dataSize, out error);

    switch (recData)
    {
        case NetworkEventType.Nothing:         //1
            break;
        case NetworkEventType.ConnectEvent:    //2
            Debug.Log("Received connection confirmation");
            _readyToSend = true;
            break;
        case NetworkEventType.DataEvent:       //3

            break;
        case NetworkEventType.DisconnectEvent: //4
            //one of the established connection has been disconnected
            Debug.Log(String.Format("Disconnect from host {0} connection {1}", recHostId, connectionId));

            break;
    }

    if (_readyToSend)
    {
        _readyToSend = false; // To send just the first frame

        byte[] colourArray = SerializeObject(MakeSerializable(GetRenderTexturePixels(webcamTexture))); // Serialize the webcam texture

        // Sending total size
        byte[] sizeToSend = BitConverter.GetBytes(colourArray.Length);
        NetworkTransport.Send(m_GenericHostId, m_ConnectionId, m_CommunicationChannel, sizeToSend, sizeToSend.Length, out error);

        byte[] bytes = new byte[bufferLenght];
        int remainingBytes = colourArray.Length;
        int index = 0;
        int i = 1;

        while (remainingBytes >= bufferLenght)
        {
            System.Buffer.BlockCopy(colourArray, index, bytes, 0, bufferLenght);
            NetworkTransport.Send(m_GenericHostId, m_ConnectionId, m_CommunicationChannel, bytes, bytes.Length, out error);
            remainingBytes -= bufferLenght;
            Debug.Log(i++ + "Remaining bytes: " + remainingBytes + " - Error: "+error);
            index += bufferLenght;
        }

        if (remainingBytes > 0) // Send the last fragment below bufferLenght bytes
        {
            System.Buffer.BlockCopy(colourArray, index, bytes, 0, remainingBytes);
            NetworkTransport.Send(m_GenericHostId, m_ConnectionId, m_CommunicationChannel, bytes, remainingBytes, out error);
            Debug.Log("Error: "+error);
        }
    }
}

And this is the client side:

void Start()
{
    if (!_isStarted)
    {
        _isStarted = true;

        NetworkTransport.Init();

        m_Config = new ConnectionConfig();

        m_CommunicationChannel  = m_Config.AddChannel(QosType.ReliableFragmented);

        HostTopology topology = new HostTopology(m_Config, 12);

        m_GenericHostId = NetworkTransport.AddHost(topology, port, null);
    }
}

void Update()
{
    if (!_isStarted)
        return;

    int recHostId; 
    int connectionId; 
    int channelId; 
    byte[] recBuffer = new byte[bufferLenght]; 
    int bufferSize = bufferLenght;
    int dataSize;
    byte error;

    NetworkEventType recData = NetworkTransport.Receive(out recHostId, out connectionId, out channelId, recBuffer, bufferSize, out dataSize, out error);

    switch (recData)
    {
        case NetworkEventType.Nothing:         //1
            break;

        case NetworkEventType.ConnectEvent:    //2

                //somebody else connect to me
            Log.text += string.Format("Connect from host {0} connection {1}\n", recHostId, connectionId);
            break;

        case NetworkEventType.DataEvent:       //3

            if (!sizeReceived)
            {
                sizeReceived = true;

                if (dataSize == 2)
                {
                    bytesToReceive = BitConverter.ToInt16(recBuffer, 0);
                }
                else if (dataSize == 4)
                {
                    bytesToReceive = BitConverter.ToInt32(recBuffer, 0);
                }

                Debug.Log("We will receive: "+bytesToReceive);
            }
            else
            {
                Log.text = string.Format("Received event host {0} connection {1} channel {2} message length {3}\n", recHostId, connectionId, channelId, dataSize);

                Log.text += "Received " + bufferSize + " bytes\n";
                bytesToReceive -= bufferSize;
                Log.text += "Remaining " + bytesToReceive + " bytes\n";
            }
            break;

        case NetworkEventType.DisconnectEvent: //4

            break;

    }
}

I know that it will block the Update function before sending, but now it doesn’t matter to me, since I'm just trying to get the transmitted frame in order to understand how this new system works and proceed from there. I am currently getting this error after sending the first packet with a buffer of 32768 bytes:

no free events for long message
UnityEngine.Networking.NetworkTransport:Send(Int32, Int32, Int32, Byte[], Int32, Byte&)
CameraStreamer:Update() (at Assets/Scripts/Client/CameraStreamer.cs:112)

I also tried using buffer 1024, and this message takes longer (after more than 100 successfully sent packets).

, , , , , Unity .

+4
2

/ Unity 5.2.

Unity 5.3.4f1, , 4 (NetworkError.NoResource) 200 . , , , .

, - 2- ( - , ).

- . , , Update . "", "".

, - , 3 , .

EDIT: , send (while bit) . , .

+1

- . , , .

, , - . , ACK . .

, , , . .

.

EDIT: Unity, Unity 5.2. ( ).

+1

More articles:


All Articles