yichael_smartbow/Assets/InfraredProject/WebCamera/Script/ZIM/ScreenLocate.cs

#define ENABLE_LOG

using Serenegiant.UVC;
using System;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.UI;
using ZIM;
using static Serenegiant.UVC.UVCManager;
using Color = UnityEngine.Color;
using InfraredManager;


[RequireComponent(typeof(Canvas))]
public partial class ScreenLocate : MonoBehaviour
{
    public InfraredCameraHelper InfraredCameraHelper;
    private const string TAG = "ScreenLocate#";
    enum Mode
    {
        InfraredLocate,
        ScreenMap,
        ScreenLocateManual
    }
    enum InfraredCount
    {
        Single,
        Double
    }
    enum Platform
    {
        Window,
        Android
    }
    Platform mPlatform = Platform.Android;

    // 2个灯，顺序根据红外灯的大小 由大到小， 坐标通过 InfraredSpot.ScreenUV 和 InfraredSpot.CameraLocation 获得
    public InfraredSpot[] InfraredSpots
    {
        get
        {
            infraredCount = InfraredCount.Double;
            return infraredSpotBuffer;
        }
    }
    // 1个灯, 坐标通过 InfraredSpot.ScreenUV 和 InfraredSpot.CameraLocation 获得
    public InfraredSpot InfraredSpotSingle
    {
        get
        {
            infraredCount = InfraredCount.Single;
            return infraredSpotBuffer[0];
        }
    }
    public InfraredSpot[] infraredSpotBuffer;
    InfraredCount infraredCount;
    public string GetInfraredCount() { return infraredCount.ToString(); }

    #region UVC 处理的对象
    public UVCManager mUVCManager;
    public UVCDrawer mUVCDrawer;
    public CameraInfo mUVCCameraInfo;
    private Texture mUVCTexture;
    private Texture2D mUVCTexture2D;
    #endregion

    public Text Info;
    public List<RectTransform> CrosshairInCamera;
    public List<RectTransform> CrosshairInScreen;
    public RectTransform ScreenQuad;
    public Toggle SaveToggle;

    public bool ShowScreenQuad = false;

    public RawImage rawImage;
    public RawImage rawImage1;
    public RawImage rawImage2;
    public RawImage rawImage3;
    public RawImage rawImage4;
    public RawImage rawImage5;
    public RawImage FullScreenImage;

    public InfraredSpotSettings InfraredSpotSettings;

    public Texture2D DebugScreenImage;

    public bool bSinglePoint = true;//默认单点识别
    bool bIdentifyRed = true;//默认设备红色
    bool bIdentifyGreen = true;

    #region 性能检测相关
    public Text m_UITime;
    const float m_UIUpdateInterval = 0.1f;
    float m_UIUpdateTimer = 0.0f;
    List<float> m_History = new List<float>(100);
    int m_ValidHistoryFrames = 0;
    float m_AverageTime = float.NaN;
    float m_MedianTime = float.NaN;
    float m_MinTime = float.NaN;
    float m_MaxTime = float.NaN;

    public float updateInterval = 0.5F;
    private double lastInterval;
    private int frames = 0;
    private float fps;
    public Text m_FPS;
    #endregion

    InfraredLocate infraredLocate;
    Mode mode;

    //o0.Project.WebCam o0WebCam = null;
    public o0.Project.ScreenIdentification screenIdentification;

    static public ScreenLocate Main;
    static public List<RawImage> DebugImage = new List<RawImage>();
    static public RectTransform BackQuad = null;
    static public void DebugTexture(int index, Texture texture)
    {
        Destroy(DebugImage[index].texture);
        DebugImage[index].texture = texture;
    }
    static public void SetScreen(UnityEngine.Color? color = null)
    {
        if (BackQuad == null)
        {
            var canvas = GameObject.Find("WebCameraView").GetComponent<RectTransform>();
            var background = canvas.Find("Background");
            BackQuad = background.GetChild(0).GetComponent<RectTransform>();
        }
        BackQuad.parent.gameObject.SetActive(color != null);
        BackQuad.GetComponent<RawImage>().color = color ?? Color.black;
    }
    static public void SetScreen(Rect rect, UnityEngine.Color? color = null)
    {
        if (BackQuad == null)
        {
            var canvas = GameObject.Find("WebCameraView").GetComponent<RectTransform>();
            var background = canvas.Find("Background");
            BackQuad = background.GetChild(0).GetComponent<RectTransform>();
        }
        BackQuad.parent.gameObject.SetActive(color != null);
        BackQuad.anchorMin = rect.min;
        BackQuad.anchorMax = rect.max;
        BackQuad.GetComponent<RawImage>().color = color ?? Color.black;
    }
    static void DebugBackQuad(Rect? rect = null)
    {
        if (BackQuad)
        {
            BackQuad.parent.GetComponent<RawImage>().enabled = false;
            BackQuad.GetComponent<RawImage>().color = Color.white;
            BackQuad.parent.gameObject.SetActive(!BackQuad.parent.gameObject.activeSelf);
            if (rect.HasValue)
            {
                BackQuad.anchorMin = rect.Value.min;
                BackQuad.anchorMax = rect.Value.max;
            }
        }
    }

    void Awake()
    {
        Main = this;

        if (mUVCDrawer)
            mUVCDrawer.StartPreviewAction += UVCIsReady;
    }

    void OnDestroy()
    {
        if (mUVCDrawer)
            mUVCDrawer.StartPreviewAction -= UVCIsReady;
    }

    void Start()
    {
        DebugImage.Add(rawImage);
        DebugImage.Add(rawImage1);
        DebugImage.Add(rawImage2);
        DebugImage.Add(rawImage3);
        DebugImage.Add(rawImage4);
        DebugImage.Add(rawImage5);
        DebugImage.Add(FullScreenImage);

        mode = Mode.InfraredLocate;
        infraredCount = InfraredCount.Single;

        #region 性能检测相关
        for (var i = 0; i < m_History.Capacity; ++i)
        {
            m_History.Add(0.0f);
        }
        lastInterval = Time.realtimeSinceStartup;
        frames = 0;
        #endregion
    }
    //ZIMWebCamera场景使用
    public void WebCamIsReady(Texture texture)
    {
        mPlatform = Platform.Window;
        mUVCTexture = texture;
        mUVCCameraInfo = new CameraInfo(mUVCTexture);
    }
    //手机端UVCCamra使用
    public void UVCIsReady(Texture texture)
    {
        mPlatform = Platform.Android;
        //this.startUVCBtn.interactable = true;

        mUVCTexture = texture;
        //ARGB32
        //mUVCTexture2D = Texture2D.CreateExternalTexture(texture.width, texture.height, TextureFormat.ARGB32, false, true, texture.GetNativeTexturePtr()); //TextureToTexture2D(texture);
        //Debug.Log("mUVCTexture2D isReable:" + mUVCTexture2D.isReadable);
        //Debug.Log("mUVCTexture2D mipmapCount:" + (mUVCTexture2D.mipmapCount > 1));

        List<CameraInfo> cameraInfos = mUVCManager.GetAttachedDevices();
        mUVCCameraInfo = cameraInfos[cameraInfos.Count - 1];
    }

    void Update()
    {
        ++frames;
        float timeNow = Time.realtimeSinceStartup;
        if (timeNow > lastInterval + updateInterval)
        {
            fps = (float)(frames / (timeNow - lastInterval));
            frames = 0;
            lastInterval = timeNow;
        }

        if (m_FPS != null)
            m_FPS.text = "FPS:" + fps.ToString("f2");

        if (mUVCCameraInfo == null) return;

        if (screenIdentification == null)
            screenIdentification = new o0.Project.ScreenIdentification(mUVCTexture);

        if (infraredLocate == null)
            infraredLocate = new InfraredLocate(mUVCCameraInfo, screenIdentification, InfraredSpotSettings);

        if (mode == Mode.ScreenLocateManual)
        {
            //if (Input.GetMouseButtonDown(0))
            //{
            //    var mouse = Input.mousePosition;
            //    var u = mouse.x / Screen.width;
            //    var v = mouse.y / Screen.height;
            //    u = Math.Clamp(u, 0, 1);
            //    v = Math.Clamp(v, 0, 1);
            //    pointManual.Add(new Vector2(u * mUVCTexture.width, v * mUVCTexture.height));
            //    var obj = Instantiate(Resources.Load("Point")) as GameObject;
            //    obj.transform.SetParent(FullScreenImage.transform);
            //    obj.transform.localPosition = new Vector2(u, v).pixelToLocalPosition_AnchorCenter(new Vector2(1, 1), FullScreenImage.rectTransform.rect);

            //    if (pointManual.Count == 1)
            //        Info.text = "左键单击屏幕 右下角";
            //    else if (pointManual.Count == 2)
            //        Info.text = "左键单击屏幕 右上角";
            //    else if (pointManual.Count == 3)
            //        Info.text = "左键单击屏幕 左上角";
            //    else if (pointManual.Count == 4)
            //    {
            //        screenIdentification.LocateScreenManual(new OrdinalQuadrilateral(pointManual[0].o0Vector(), pointManual[1].o0Vector(), pointManual[3].o0Vector(), pointManual[2].o0Vector()));
            //        pointManual.Clear();
            //        ShowScreen(screenIdentification.Screen.Quad);
            //        foreach (Transform i in FullScreenImage.transform)
            //            Destroy(i.gameObject);
            //        ToMode(Mode.InfraredLocate);
            //    }
            //}
            return;
        }

        var t0 = Time.realtimeSinceStartup;

        /* New*/
        if (mUVCCameraInfo != null && mUVCCameraInfo.IsPreviewing && screenIdentification.Screen.Active)     // 成功定位屏幕后才做红外识别
        {
          
            CreateUVCTexture2DIfNeeded();
            if (!screenIdentification.Update(mUVCTexture2D))
            {
                if (mode == Mode.InfraredLocate)
                {
                    //0,0, cameraTexture2D.width, cameraTexture2D.height,0
                    var pixels = mUVCTexture2D.GetPixels();       // 从左往右、从下往上
                    //InfraredSpots = infraredLocate.Update(pixels);
                    if (bSinglePoint)
                        infraredSpotBuffer = infraredLocate.UpdateSingle(pixels);
                    else
                        infraredSpotBuffer = infraredLocate.Update(pixels);

                    if (mPlatform == Platform.Window) //渲染ui上面的点。进入游戏可以隐藏
                    {
                        for (int i = 0; i < infraredSpotBuffer.Length; i++)
                        {
                            if (infraredSpotBuffer[i].CameraLocation != null)
                            {
                                // 检测到光点
                                var posInCanvas = infraredSpotBuffer[i].CameraLocation.Value.pixelToLocalPosition_AnchorCenter(mUVCCameraInfo.Size, rawImage.rectTransform.rect);
                                CrosshairInCamera[i].gameObject.SetActive(true);
                                CrosshairInCamera[i].anchoredPosition = posInCanvas;

                            }
                            else
                                CrosshairInCamera[i].gameObject.SetActive(false);
                        }
                    }

                    //手机端使用
                    if (mPlatform == Platform.Android && infraredSpotBuffer.Length > 0)
                    {
                        int redIndex = 0;
                        int greenIndex = 1;
                        //仅仅第一个点显示(如果最大点出界了会闪烁)
                        if (bSinglePoint)
                        {
                            redIndex = 0; //单点识别是，可以选择切换颜色
                            if (infraredSpotBuffer[redIndex].ScreenUV != null)
                            {
                                string str = "Single:";
                                Info.text = str + infraredSpotBuffer[redIndex].ScreenUV.Value.ToString("F4");
                                //InfraredManager.ConnetDevicesSingle.ins.posAction?.Invoke(new Vector3(infraredSpotBuffer[redIndex].ScreenUV.Value.x * Screen.width, infraredSpotBuffer[redIndex].ScreenUV.Value.y * Screen.height, 0));
                                onFilterPos(infraredSpotBuffer[redIndex].ScreenUV.Value);

                            }
                        }
                        else
                        {
                            //雙點模式下選擇第一個點
                            if (bIdentifyRed && !bIdentifyGreen)
                            {
                                if (infraredSpotBuffer[redIndex].ScreenUV != null)
                                {
                                    Info.text = "Red" + redIndex + ":" + infraredSpotBuffer[redIndex].ScreenUV.Value.ToString("F4");
                                    //InfraredManager.ConnetDevicesSingle.ins.posAction?.Invoke(new Vector3(infraredSpotBuffer[redIndex].ScreenUV.Value.x * Screen.width, infraredSpotBuffer[redIndex].ScreenUV.Value.y * Screen.height, 0));
                                    onFilterPos2(infraredSpotBuffer[redIndex].ScreenUV.Value, redIndex);
                                }
                                else
                                {
                                    Info.text = "未检测到红色最大点！";
                                }
                            }
                            else if (!bIdentifyRed && bIdentifyGreen) {
                                if (infraredSpotBuffer[greenIndex].ScreenUV != null)
                                {
                                    Info.text = "Green:" + infraredSpotBuffer[greenIndex].ScreenUV.Value.ToString("F4");
                                    //InfraredManager.ConnetDevicesSingle.ins.posAction?.Invoke(new Vector3(infraredSpotBuffer[greenIndex].ScreenUV.Value.x * Screen.width, infraredSpotBuffer[greenIndex].ScreenUV.Value.y * Screen.height, 0));
                                    onFilterPos2(infraredSpotBuffer[greenIndex].ScreenUV.Value, greenIndex);

                                }
                                else
                                {
                                    Info.text = "未检测到绿色点！";
                                }
                            }
                            else
                            {
                                //两个不选择和两个全选都跑识别两个点
                                //自動切換 检测到光点
                                if (infraredSpotBuffer[redIndex].ScreenUV != null)
                                {
                                    Info.text = "Red:" + infraredSpotBuffer[redIndex].ScreenUV.Value.ToString("F4");
                                    //InfraredManager.ConnetDevicesSingle.ins.posAction?.Invoke(new Vector3(infraredSpotBuffer[redIndex].ScreenUV.Value.x * Screen.width, infraredSpotBuffer[redIndex].ScreenUV.Value.y * Screen.height, 0));
                                    onFilterPos2(infraredSpotBuffer[redIndex].ScreenUV.Value, redIndex);
                                }
                                else if (infraredSpotBuffer[greenIndex].ScreenUV != null)
                                {

                                    Info.text = "Green:" + infraredSpotBuffer[greenIndex].ScreenUV.Value.ToString("F4");
                                    //InfraredManager.ConnetDevicesSingle.ins.posAction?.Invoke(new Vector3(infraredSpotBuffer[greenIndex].ScreenUV.Value.x * Screen.width, infraredSpotBuffer[greenIndex].ScreenUV.Value.y * Screen.height, 0));
                                    onFilterPos2(infraredSpotBuffer[greenIndex].ScreenUV.Value, greenIndex);
                                }
                                else
                                {
                                    Info.text = "未检测到点！";
                                }

                            }
                          

                        }

                    }

                }

            }
        }

        var t1 = Time.realtimeSinceStartup;
        var dt = t1 - t0;
        m_History[m_ValidHistoryFrames % m_History.Count] = dt;
        ++m_ValidHistoryFrames;

        m_UIUpdateTimer += Time.deltaTime;

        if (m_UIUpdateTimer >= m_UIUpdateInterval)
        {
            m_UIUpdateTimer = 0.0f;

            if (m_ValidHistoryFrames >= m_History.Count)
            {
                m_ValidHistoryFrames = 0;

                m_AverageTime = 0.0f;

                m_MinTime = float.PositiveInfinity;
                m_MaxTime = float.NegativeInfinity;

                {
                    for (var i = 0; i < m_History.Count; i++)
                    {
                        var time = m_History[i];
                        m_AverageTime += time;

                        m_MinTime = Mathf.Min(m_MinTime, time);
                        m_MaxTime = Mathf.Max(m_MaxTime, time);
                    }
                    m_AverageTime /= m_History.Count;
                }
                {
                    m_History.Sort();

                    // Odd-length history?
                    if ((m_History.Count & 1) != 0)
                    {
                        m_MedianTime = m_History[m_History.Count / 2];
                    }
                    else
                    {
                        m_MedianTime = (m_History[m_History.Count / 2] + m_History[m_History.Count / 2 - 1]) / 2.0f;
                    }
                }

            }
            var statistics = $"{m_History.Count} 帧样本:\naverage: {m_AverageTime * 1000.0f:F2}ms\nmedian: {m_MedianTime * 1000.0f:F2}ms\nmin: {m_MinTime * 1000.0f:F2}ms\nmax: {m_MaxTime * 1000.0f:F2}ms\n";

            //Method: {m_Method} {UnityEngine.SceneManagement.SceneManager.GetActiveScene().name} |
            if (m_UITime != null)
                m_UITime.text = $"Cam: {mUVCCameraInfo.CurrentWidth}x{mUVCCameraInfo.CurrentHeight}{(mUVCTexture2D? ",T2D:" : "")}{(mUVCTexture2D? mUVCTexture2D.width+ "x" : "")}{(mUVCTexture2D ? mUVCTexture2D.height:"")} \nLast Frame: {dt * 1000.0f:F2}ms \n{statistics}";
        
        }
    }

    Vector2 targetPos = Vector2.zero;
    public float filterDis = 3.0f;
    void onFilterPos(Vector2 _vector2Pos) {

        Vector2 np = new Vector2(_vector2Pos.x * Screen.width, _vector2Pos.y * Screen.height); //_vector2Pos.pixelToLocalPosition_AnchorCenter(Vector2.one, (transform as RectTransform).rect);
        if (Vector2.Distance(np, targetPos) >= filterDis)
        {
            targetPos = np;
            InfraredCameraHelper.InvokeOnPositionUpdate(targetPos);
        }
    }

    Vector2[] _targetPoints2 = new Vector2[] { Vector2.zero, Vector2.zero };
    void onFilterPos2(Vector2 _vector2Pos, int index)
    {
        Vector2 np = new Vector2(_vector2Pos.x * Screen.width, _vector2Pos.y * Screen.height);
        if (Vector2.Distance(np, _targetPoints2[index]) >= filterDis)
        {
            _targetPoints2[index] = np;
            InfraredCameraHelper.InvokeOnPositionUpdate2(_targetPoints2[index], index);
        }
    }

    // 标记屏幕的四个角
    public void ShowScreen(OrdinalQuadrilateral quad)
    {
        if (quad == null)
        {
            Info.text = "识别屏幕失败";
            return;
        }
        Info.text = "已识别到屏幕";

        if (ShowScreenQuad)
        {
            ScreenQuad.gameObject.SetActive(true);
            for (int i = 0; i < 4; i++)
            {
                RectTransform t = ScreenQuad.GetChild(i) as RectTransform;
                //mUVCCameraInfo.Size
                t.anchoredPosition = quad[i].UnityVector().pixelToLocalPosition_AnchorCenter(mUVCCameraInfo.Size, ScreenQuad.rect);
            }
        }

    }

    void ToMode(Mode mode)
    {
        if (this.mode == mode)
            return;

        if (mode == Mode.ScreenMap)
        {
            if (!screenIdentification.Screen.Active)
            {
                Info.text = "先定位屏幕";
                return;
            }
            Info.text = "按ESC退出";
            SetScreen(Color.black);
            Info.transform.SetAsLastSibling();
            this.mode = Mode.ScreenMap;
        }
        else if (mode == Mode.InfraredLocate)
        {
            Info.text = screenIdentification.Screen.Active ? "已定位屏幕" : "定位屏幕失败";
            //Info.text = "已识别到屏幕"; 
            SetScreen(null);
            foreach (var i in CrosshairInScreen)
                i.gameObject.SetActive(false);
            FullScreenImage.gameObject.SetActive(false);
            Info.transform.SetSiblingIndex(transform.childCount - 4);
            this.mode = Mode.InfraredLocate;
            DebugTexture(6, null);
            //DebugTexture(1, null); //null
            // rawImage1.texture = null;

#if (!NDEBUG && DEBUG && ENABLE_LOG)
            Console.WriteLine($"{TAG} Mode.InfraredLocate:已识别到屏幕:{ screenIdentification.Screen.Active}");
#endif
        }
        else if (mode == Mode.ScreenLocateManual)
        {
            //Info.text = "左键单击屏幕 左下角";
            //FullScreenImage.gameObject.SetActive(true);
            //Info.transform.SetSiblingIndex(transform.childCount - 1);

            //// var newTex = WebCamera.webCamTexture.AutoLight(10);
            ////DebugTexture(1, TextureToTexture2D(rawImage.texture));

            //CreateUVCTexture2DIfNeeded();
            //DebugTexture(6, mUVCTexture2D.zimAutoLight(brightness));
            ////mUVCTexture2DTemp = TextureToTexture2D(mUVCCameraInfo.previewTexture);
            ////DebugTexture(6, mUVCTexture2DTemp.zimAutoLight(brightness));
            //this.mode = Mode.ScreenLocateManual;
        }
    }

    private Texture2D TextureToTexture2D(Texture texture)
    {
        Texture2D _texture2D = new Texture2D(texture.width, texture.height, TextureFormat.ARGB32, false, true);
        RenderTexture currentRT = RenderTexture.active;
        RenderTexture renderTexture = RenderTexture.GetTemporary(
                    texture.width,
                    texture.height,
                    0,
                    RenderTextureFormat.ARGB32,
                    RenderTextureReadWrite.Linear);
        Graphics.Blit(texture, renderTexture);
        RenderTexture.active = renderTexture;
        _texture2D.ReadPixels(new Rect(0, 0, renderTexture.width, renderTexture.height), 0, 0);
        _texture2D.Apply();
        RenderTexture.active = currentRT;
        RenderTexture.ReleaseTemporary(renderTexture);
        return _texture2D;
    }

    private void CreateUVCTexture2DIfNeeded()
    {
        if (mUVCTexture2D != null)
            Destroy(mUVCTexture2D);
        mUVCTexture2D = TextureToTexture2D(mUVCTexture);
    }
}