slambb_smartbow-infrared/Assets/WebCamera/Script/ZIM/ScreenLocate.cs

#define ENABLE_LOG

using Serenegiant.UVC;
using System;
using System.Collections.Generic;
using System.Threading;
using UnityEngine;
using UnityEngine.UI;
using ZIM;
using static Serenegiant.UVC.UVCManager;
using Color = UnityEngine.Color;


[RequireComponent(typeof(Canvas))]
public class ScreenLocate : MonoBehaviour
{
    private const string TAG = "ScreenLocate#";
    enum Mode
    {
        InfraredLocate,
        ScreenMap,
        ScreenLocateManual
    }
    enum InfraredCount
    {
        Single,
        Double
    }
    enum Platform
    {
        Window,
        Android
    }
    Platform mPlatform = Platform.Android;

    // 2个灯，顺序根据红外灯的大小 由大到小， 坐标通过 InfraredSpot.ScreenUV 和 InfraredSpot.CameraLocation 获得
    public InfraredSpot[] InfraredSpots
    {
        get
        {
            infraredCount = InfraredCount.Double;
            return infraredSpotBuffer;
        }
    }
    // 1个灯, 坐标通过 InfraredSpot.ScreenUV 和 InfraredSpot.CameraLocation 获得
    public InfraredSpot InfraredSpotSingle
    {
        get
        {
            infraredCount = InfraredCount.Single;
            return infraredSpotBuffer[0];
        }
    }
    InfraredSpot[] infraredSpotBuffer;
    InfraredCount infraredCount;

    #region UVC 处理的对象
    public UVCManager mUVCManager;
    public UVCDrawer mUVCDrawer;
    public CameraInfo mUVCCameraInfo;
    private Texture mUVCTexture;
    private Texture2D mUVCTexture2D;
    // [SerializeField] Texture2DArray mUVCOutArray;
    #endregion


    public Text Info;
    public List<RectTransform> CrosshairInCamera;
    public List<RectTransform> CrosshairInScreen;
    public RectTransform ScreenQuad;
    public Toggle SaveToggle;

    public bool ShowScreenQuad = false;

    public RawImage rawImage;
    public RawImage rawImage1;
    public RawImage rawImage2;
    public RawImage rawImage3;
    public RawImage rawImage4;
    public RawImage rawImage5;
    public RawImage FullScreenImage;

    public InfraredSpotSettings InfraredSpotSettings;

    public Texture2D DebugScreenImage;



    public Button startUVCBtn;
    public Button manualDebugBtn;

    public InfraredManager.UIManagerSingle mUIManagerSingle;
    // private SynchronizationContext mainContext;

    //是否单点显示
    bool bSinglePoint = true;//默认单点识别
    public Toggle SinglePointToggle;
    bool bIdentifyRed = true;//默认设备红色
    public Toggle SinglePointToggleColor;

    bool bIdentifyGreen = true;
    public Toggle SinglePointToggleColorGreen;

    #region 性能检测相关
    public Text m_UITime;
    const float m_UIUpdateInterval = 0.1f;
    float m_UIUpdateTimer = 0.0f;
    List<float> m_History = new List<float>(100);
    int m_ValidHistoryFrames = 0;
    float m_AverageTime = float.NaN;
    float m_MedianTime = float.NaN;
    float m_MinTime = float.NaN;
    float m_MaxTime = float.NaN;

    public float updateInterval = 0.5F;
    private double lastInterval;
    private int frames = 0;
    private float fps;
    public Text m_FPS;
    #endregion


    InfraredLocate infraredLocate;
    RectTransform canvas;
    Mode mode;
    List<Vector2> pointManual = new List<Vector2>();

    //o0.Project.WebCam o0WebCam = null;
    o0.Project.ScreenIdentification screenIdentification;

    static public ScreenLocate Main;
    static public List<RawImage> DebugImage = new List<RawImage>();
    static public RectTransform BackQuad = null;
    static public void DebugTexture(int index, Texture texture)
    {
        Destroy(DebugImage[index].texture);
        DebugImage[index].texture = texture;
    }
    static public void SetScreen(UnityEngine.Color? color = null)
    {
        if (BackQuad == null)
        {
            var canvas = GameObject.Find("WebCameraView").GetComponent<RectTransform>();
            var background = canvas.Find("Background");
            BackQuad = background.GetChild(0).GetComponent<RectTransform>();
        }
        BackQuad.parent.gameObject.SetActive(color != null);
        BackQuad.GetComponent<RawImage>().color = color ?? Color.black;
    }
    static public void SetScreen(Rect rect, UnityEngine.Color? color = null)
    {
        if (BackQuad == null)
        {
            var canvas = GameObject.Find("WebCameraView").GetComponent<RectTransform>();
            var background = canvas.Find("Background");
            BackQuad = background.GetChild(0).GetComponent<RectTransform>();
        }
        BackQuad.parent.gameObject.SetActive(color != null);
        BackQuad.anchorMin = rect.min;
        BackQuad.anchorMax = rect.max;
        BackQuad.GetComponent<RawImage>().color = color ?? Color.black;
    }
    static void DebugBackQuad(Rect? rect = null)
    {
        if (BackQuad)
        {
            BackQuad.parent.GetComponent<RawImage>().enabled = false;
            BackQuad.GetComponent<RawImage>().color = Color.white;
            BackQuad.parent.gameObject.SetActive(!BackQuad.parent.gameObject.activeSelf);
            if (rect.HasValue)
            {
                BackQuad.anchorMin = rect.Value.min;
                BackQuad.anchorMax = rect.Value.max;
            }
        }
    }

    void Awake()
    {
        if (mUVCDrawer)
            mUVCDrawer.StartPreviewAction += UVCIsReady;
    }

    void OnDestroy()
    {
        if (mUVCDrawer)
            mUVCDrawer.StartPreviewAction -= UVCIsReady;
    }

    void Start()
    {
        //mainContext = SynchronizationContext.Current;

        Main = this;

        DebugImage.Add(rawImage);
        DebugImage.Add(rawImage1);
        DebugImage.Add(rawImage2);
        DebugImage.Add(rawImage3);
        DebugImage.Add(rawImage4);
        DebugImage.Add(rawImage5);
        DebugImage.Add(FullScreenImage);

        canvas = transform.GetComponent<RectTransform>();

        mode = Mode.InfraredLocate;

        infraredCount = InfraredCount.Single;

        if (SinglePointToggle)
        {
            //如果是单点显示
            bSinglePoint = PlayerPrefs.GetInt("bSinglePoint", 1) == 1;
            SinglePointToggle.isOn = bSinglePoint;
        }
        if (SinglePointToggleColor)
        {
            bIdentifyRed = PlayerPrefs.GetInt("bIdentifyRed", 1) == 1;
            SinglePointToggleColor.isOn = bIdentifyRed;
        }

        if (SinglePointToggleColorGreen) { 
            bIdentifyGreen = PlayerPrefs.GetInt("bIdentifyGreen", 1) == 1;
            SinglePointToggleColorGreen.isOn = bIdentifyGreen;
        }

        #region 性能检测相关
        for (var i = 0; i < m_History.Capacity; ++i)
        {
            m_History.Add(0.0f);
        }
        lastInterval = Time.realtimeSinceStartup;
        frames = 0;
        #endregion
    }
    //ZIMWebCamera场景使用
    public void WebCamIsReady(Texture texture)
    {
        mPlatform = Platform.Window;
        mUVCTexture = texture;
        mUVCCameraInfo = new CameraInfo(mUVCTexture);
        brightness = 0;
    }
    //手机端UVCCamra使用
    public void UVCIsReady(Texture texture)
    {
        mPlatform = Platform.Android;
        //this.startUVCBtn.interactable = true;

        mUVCTexture = texture;
        //ARGB32
        //mUVCTexture2D = Texture2D.CreateExternalTexture(texture.width, texture.height, TextureFormat.ARGB32, false, true, texture.GetNativeTexturePtr()); //TextureToTexture2D(texture);
        //Debug.Log("mUVCTexture2D isReable:" + mUVCTexture2D.isReadable);
        //Debug.Log("mUVCTexture2D mipmapCount:" + (mUVCTexture2D.mipmapCount > 1));

        List<CameraInfo> cameraInfos = mUVCManager.GetAttachedDevices();
        mUVCCameraInfo = cameraInfos[cameraInfos.Count - 1];
        manualDebugBtn.interactable = true;
    }

    public void OnChangeSinglePoint()
    {
        bSinglePoint = SinglePointToggle.isOn;
        PlayerPrefs.SetInt("bSinglePoint", bSinglePoint ? 1 : 0);
    }

    public void OnChangeSinglePointColor()
    {

        bIdentifyRed = SinglePointToggleColor.isOn;
        PlayerPrefs.SetInt("bIdentifyRed", bIdentifyRed ? 1 : 0);
    }

    public void OnChangeSinglePointColorGreen() {

        bIdentifyGreen = SinglePointToggleColorGreen.isOn;
        PlayerPrefs.SetInt("bIdentifyGreen", bIdentifyGreen ? 1 : 0);
    }

    public void startUVC()
    {

    }
    int brightness = 0;
    //public Text brightnessText;
    //public void SliderBrightness(Slider slider)
    //{
    //    var _value = slider.value;
    //    brightness = (int)_value;
    //    brightnessText.text = (2 + brightness) + "";
    //}


    void Update()
    {
        ++frames;
        float timeNow = Time.realtimeSinceStartup;
        if (timeNow > lastInterval + updateInterval)
        {
            fps = (float)(frames / (timeNow - lastInterval));
            frames = 0;
            lastInterval = timeNow;
        }

        if (m_FPS != null)
            m_FPS.text = "FPS:" + fps.ToString("f2");

        if (mUVCCameraInfo == null) return;

        if (screenIdentification == null)
            screenIdentification = new o0.Project.ScreenIdentification(mUVCTexture);

        if (infraredLocate == null)
            infraredLocate = new InfraredLocate(mUVCCameraInfo, screenIdentification, InfraredSpotSettings);

        if (mode == Mode.ScreenLocateManual)
        {
            if (Input.GetMouseButtonDown(0))
            {
                var mouse = Input.mousePosition;
                var u = mouse.x / Screen.width;
                var v = mouse.y / Screen.height;
                u = Math.Clamp(u, 0, 1);
                v = Math.Clamp(v, 0, 1);
                pointManual.Add(new Vector2(u * mUVCTexture.width, v * mUVCTexture.height));
                var obj = Instantiate(Resources.Load("Point")) as GameObject;
                obj.transform.SetParent(FullScreenImage.transform);
                obj.transform.localPosition = new Vector2(u, v).pixelToLocalPosition_AnchorCenter(new Vector2(1, 1), FullScreenImage.rectTransform.rect);

                if (pointManual.Count == 1)
                    Info.text = "左键单击屏幕 右下角";
                else if (pointManual.Count == 2)
                    Info.text = "左键单击屏幕 右上角";
                else if (pointManual.Count == 3)
                    Info.text = "左键单击屏幕 左上角";
                else if (pointManual.Count == 4)
                {
                    screenIdentification.LocateScreenManual(new OrdinalQuadrilateral(pointManual[0].o0Vector(), pointManual[1].o0Vector(), pointManual[3].o0Vector(), pointManual[2].o0Vector()));
                    pointManual.Clear();
                    ShowScreen(screenIdentification.Screen.Quad);
                    foreach (Transform i in FullScreenImage.transform)
                        Destroy(i.gameObject);
                    ToMode(Mode.InfraredLocate);
                }
            }
            return;
        }

        var t0 = Time.realtimeSinceStartup;

        /* New*/
        if (mUVCCameraInfo != null && mUVCCameraInfo.IsPreviewing && screenIdentification.Screen.Active)     // 成功定位屏幕后才做红外识别
        {
          
            CreateUVCTexture2DIfNeeded();
            if (!screenIdentification.Update(mUVCTexture2D))
            {
                if (mode == Mode.InfraredLocate)
                {
                    //0,0, cameraTexture2D.width, cameraTexture2D.height,0
                    var pixels = mUVCTexture2D.GetPixels();       // 从左往右、从下往上
                    //InfraredSpots = infraredLocate.Update(pixels);
                    if (bSinglePoint)
                        infraredSpotBuffer = infraredLocate.UpdateSingle(pixels);
                    else
                        infraredSpotBuffer = infraredLocate.Update(pixels);

                    if (mPlatform == Platform.Window || mUIManagerSingle.bGetPanelActive) //渲染ui上面的点。进入游戏可以隐藏
                    {
                        for (int i = 0; i < infraredSpotBuffer.Length; i++)
                        {
                            if (infraredSpotBuffer[i].CameraLocation != null)
                            {
                                // 检测到光点
                                var posInCanvas = infraredSpotBuffer[i].CameraLocation.Value.pixelToLocalPosition_AnchorCenter(mUVCCameraInfo.Size, rawImage.rectTransform.rect);
                                CrosshairInCamera[i].gameObject.SetActive(true);
                                CrosshairInCamera[i].anchoredPosition = posInCanvas;

                            }
                            else
                                CrosshairInCamera[i].gameObject.SetActive(false);
                        }
                    }

                    //手机端使用
                    if (mPlatform == Platform.Android && infraredSpotBuffer.Length > 0)
                    {
                        int redIndex = 0;
                        int greenIndex = 1;
                        //仅仅第一个点显示(如果最大点出界了会闪烁)
                        if (bSinglePoint)
                        {
                            redIndex = 0; //单点识别是，可以选择切换颜色
                            if (infraredSpotBuffer[redIndex].ScreenUV != null)
                            {
                                string str = "Single:";
                                Info.text = str + infraredSpotBuffer[redIndex].ScreenUV.Value.ToString("F4");
                                InfraredManager.ConnetDevicesSingle.ins.posAction?.Invoke(new Vector3(infraredSpotBuffer[redIndex].ScreenUV.Value.x * Screen.width, infraredSpotBuffer[redIndex].ScreenUV.Value.y * Screen.height, 0));
                            }
                        }
                        else
                        {
                            //雙點模式下選擇第一個點
                            if (bIdentifyRed && !bIdentifyGreen)
                            {
                                if (infraredSpotBuffer[redIndex].ScreenUV != null)
                                {
                                    Info.text = "Red" + redIndex + ":" + infraredSpotBuffer[redIndex].ScreenUV.Value.ToString("F4");
                                    InfraredManager.ConnetDevicesSingle.ins.posAction?.Invoke(new Vector3(infraredSpotBuffer[redIndex].ScreenUV.Value.x * Screen.width, infraredSpotBuffer[redIndex].ScreenUV.Value.y * Screen.height, 0));
                                }
                                else
                                {
                                    Info.text = "未检测到红色最大点！";
                                }
                            }
                            else if (!bIdentifyRed && bIdentifyGreen) {
                                if (infraredSpotBuffer[greenIndex].ScreenUV != null)
                                {
                                    Info.text = "Green:" + infraredSpotBuffer[greenIndex].ScreenUV.Value.ToString("F4");
                                    InfraredManager.ConnetDevicesSingle.ins.posAction?.Invoke(new Vector3(infraredSpotBuffer[greenIndex].ScreenUV.Value.x * Screen.width, infraredSpotBuffer[greenIndex].ScreenUV.Value.y * Screen.height, 0));
                                }
                                else
                                {
                                    Info.text = "未检测到绿色点！";
                                }
                            }
                            else
                            {
                                //两个不选择和两个全选都跑识别两个点
                                //自動切換 检测到光点
                                if (infraredSpotBuffer[redIndex].ScreenUV != null)
                                {
                                    Info.text = "Red:" + infraredSpotBuffer[redIndex].ScreenUV.Value.ToString("F4");
                                    InfraredManager.ConnetDevicesSingle.ins.posAction?.Invoke(new Vector3(infraredSpotBuffer[redIndex].ScreenUV.Value.x * Screen.width, infraredSpotBuffer[redIndex].ScreenUV.Value.y * Screen.height, 0));
                                }
                                else if (infraredSpotBuffer[greenIndex].ScreenUV != null)
                                {

                                    Info.text = "Green:" + infraredSpotBuffer[greenIndex].ScreenUV.Value.ToString("F4");
                                    InfraredManager.ConnetDevicesSingle.ins.posAction?.Invoke(new Vector3(infraredSpotBuffer[greenIndex].ScreenUV.Value.x * Screen.width, infraredSpotBuffer[greenIndex].ScreenUV.Value.y * Screen.height, 0));
                                }
                                else
                                {
                                    Info.text = "未检测到点！";
                                }

                            }
                          

                        }

                    }

                }

            }
        }

        var t1 = Time.realtimeSinceStartup;
        var dt = t1 - t0;
        m_History[m_ValidHistoryFrames % m_History.Count] = dt;
        ++m_ValidHistoryFrames;

        m_UIUpdateTimer += Time.deltaTime;

        if (m_UIUpdateTimer >= m_UIUpdateInterval)
        {
            m_UIUpdateTimer = 0.0f;

            if (m_ValidHistoryFrames >= m_History.Count)
            {
                m_ValidHistoryFrames = 0;

                m_AverageTime = 0.0f;

                m_MinTime = float.PositiveInfinity;
                m_MaxTime = float.NegativeInfinity;

                {
                    for (var i = 0; i < m_History.Count; i++)
                    {
                        var time = m_History[i];
                        m_AverageTime += time;

                        m_MinTime = Mathf.Min(m_MinTime, time);
                        m_MaxTime = Mathf.Max(m_MaxTime, time);
                    }
                    m_AverageTime /= m_History.Count;
                }
                {
                    m_History.Sort();

                    // Odd-length history?
                    if ((m_History.Count & 1) != 0)
                    {
                        m_MedianTime = m_History[m_History.Count / 2];
                    }
                    else
                    {
                        m_MedianTime = (m_History[m_History.Count / 2] + m_History[m_History.Count / 2 - 1]) / 2.0f;
                    }
                }

            }
            var statistics = $"{m_History.Count} 帧样本:\naverage: {m_AverageTime * 1000.0f:F2}ms\nmedian: {m_MedianTime * 1000.0f:F2}ms\nmin: {m_MinTime * 1000.0f:F2}ms\nmax: {m_MaxTime * 1000.0f:F2}ms\n";

            //Method: {m_Method} {UnityEngine.SceneManagement.SceneManager.GetActiveScene().name} |
            if (m_UITime != null)
                m_UITime.text = $"Cam: {mUVCCameraInfo.CurrentWidth}x{mUVCCameraInfo.CurrentHeight}{(mUVCTexture2D? ",T2D:" : "")}{(mUVCTexture2D? mUVCTexture2D.width+ "x" : "")}{(mUVCTexture2D ? mUVCTexture2D.height:"")} \nLast Frame: {dt * 1000.0f:F2}ms \n{statistics}";
        
        }
        UpdateInputs();

      
    }


    public void BtnScreenLocate()
    {
        screenIdentification.LocateScreen();
    }

    public void BtnScreenMap()
    {
        ToMode(Mode.ScreenMap);
    }

    //进入手动定位屏幕
    public void BtnScreenLocateManual()
    {
        ToMode(Mode.ScreenLocateManual);
    }

    // 标记屏幕的四个角
    public void ShowScreen(OrdinalQuadrilateral quad)
    {
        if (quad == null)
        {
            Info.text = "识别屏幕失败";
            return;
        }
        Info.text = "已识别到屏幕";

        if (ShowScreenQuad)
        {
            ScreenQuad.gameObject.SetActive(true);
            for (int i = 0; i < 4; i++)
            {
                RectTransform t = ScreenQuad.GetChild(i) as RectTransform;
                //mUVCCameraInfo.Size
                t.anchoredPosition = quad[i].UnityVector().pixelToLocalPosition_AnchorCenter(mUVCCameraInfo.Size, ScreenQuad.rect);
            }
        }

    }

    void ToMode(Mode mode)
    {
        if (this.mode == mode)
            return;

        if (mode == Mode.ScreenMap)
        {
            if (!screenIdentification.Screen.Active)
            {
                Info.text = "先定位屏幕";
                return;
            }
            Info.text = "按ESC退出";
            SetScreen(Color.black);
            Info.transform.SetAsLastSibling();
            this.mode = Mode.ScreenMap;
        }
        else if (mode == Mode.InfraredLocate)
        {
            Info.text = screenIdentification.Screen.Active ? "已定位屏幕" : "定位屏幕失败";
            //Info.text = "已识别到屏幕"; 
            SetScreen(null);
            foreach (var i in CrosshairInScreen)
                i.gameObject.SetActive(false);
            FullScreenImage.gameObject.SetActive(false);
            Info.transform.SetSiblingIndex(transform.childCount - 4);
            this.mode = Mode.InfraredLocate;
            DebugTexture(6, null);
            //DebugTexture(1, null); //null
            // rawImage1.texture = null;

#if (!NDEBUG && DEBUG && ENABLE_LOG)
            Console.WriteLine($"{TAG} Mode.InfraredLocate:已识别到屏幕:{ screenIdentification.Screen.Active}");
#endif
        }
        else if (mode == Mode.ScreenLocateManual)
        {
            Info.text = "左键单击屏幕 左下角";
            FullScreenImage.gameObject.SetActive(true);
            Info.transform.SetSiblingIndex(transform.childCount - 1);

            // var newTex = WebCamera.webCamTexture.AutoLight(10);
            //DebugTexture(1, TextureToTexture2D(rawImage.texture));

            CreateUVCTexture2DIfNeeded();
            DebugTexture(6, mUVCTexture2D.zimAutoLight(brightness));
            //mUVCTexture2DTemp = TextureToTexture2D(mUVCCameraInfo.previewTexture);
            //DebugTexture(6, mUVCTexture2DTemp.zimAutoLight(brightness));
            this.mode = Mode.ScreenLocateManual;
        }
    }

    private Texture2D TextureToTexture2D(Texture texture)
    {
        Texture2D _texture2D = new Texture2D(texture.width, texture.height, TextureFormat.ARGB32, false, true);
        RenderTexture currentRT = RenderTexture.active;
        RenderTexture renderTexture = RenderTexture.GetTemporary(
                    texture.width,
                    texture.height,
                    0,
                    RenderTextureFormat.ARGB32,
                    RenderTextureReadWrite.Linear);
        Graphics.Blit(texture, renderTexture);
        RenderTexture.active = renderTexture;
        _texture2D.ReadPixels(new Rect(0, 0, renderTexture.width, renderTexture.height), 0, 0);
        _texture2D.Apply();
        RenderTexture.active = currentRT;
        RenderTexture.ReleaseTemporary(renderTexture);
        return _texture2D;
    }

    private void CreateUVCTexture2DIfNeeded()
    {
        if (mUVCTexture2D != null)
            Destroy(mUVCTexture2D);
        mUVCTexture2D = TextureToTexture2D(mUVCTexture);
    }



    #region DoubleButton 

    private DateTime m_firstTime;
    private DateTime m_secondTime;

    private void Press()
    {
        Debug.Log("进入手动定位");
        BtnScreenLocateManual();
        resetTime();
    }

    public void OnDoubleClick()
    {
        //超时重置
        if (!m_firstTime.Equals(default(DateTime)))
        {
            var intervalTime = DateTime.Now - m_firstTime;
            float milliSeconds = intervalTime.Seconds * 1000 + intervalTime.Milliseconds;
            if (milliSeconds >= 400)
                resetTime();
        }

        // 按下按钮时对两次的时间进行记录
        if (m_firstTime.Equals(default(DateTime)))
            m_firstTime = DateTime.Now;
        else
            m_secondTime = DateTime.Now;

        // 在第二次点击触发,时差小于400ms触发
        if (!m_firstTime.Equals(default(DateTime)) && !m_secondTime.Equals(default(DateTime)))
        {
            var intervalTime = m_secondTime - m_firstTime;
            float milliSeconds = intervalTime.Seconds * 1000 + intervalTime.Milliseconds;
            if (milliSeconds < 400)
                Press();
            else
                resetTime();
        }
    }

    private void resetTime()
    {
        m_firstTime = default(DateTime);
        m_secondTime = default(DateTime);
    }
    #endregion


    #region 性能检测相关
    void InvalidateTimings()
    {
        m_ValidHistoryFrames = 0;
        m_AverageTime = float.NaN;
        m_MedianTime = float.NaN;
        m_MinTime = float.NaN;
        m_MaxTime = float.NaN;
    }
    void UpdateInputs()
    {
        
        //重置
        if (Input.GetKeyDown(KeyCode.UpArrow))
        {
            InvalidateTimings();
        }
    }
    #endregion
}