#define ENABLE_LOG using InfraredManager; using o0; using o0.Geometry2D.Float; using o0InfraredLocate.ZIM; using SixLabors.ImageSharp.PixelFormats; using SLAMUVC; using System; using System.Collections; using System.Collections.Generic; using System.Linq; using UnityEngine; using UnityEngine.Experimental.AI; using UnityEngine.UI; using ZIM; using ZIM.Unity; using static SLAMUVC.UVCManager; using Color = UnityEngine.Color; using Time = UnityEngine.Time; [RequireComponent(typeof(Canvas))] public partial class ScreenLocate : o0InfraredCameraHandler { public InfraredCameraHelper InfraredCameraHelper; private const string TAG = "ScreenLocate#"; enum Mode { InfraredLocate, ScreenMap, ScreenLocateManual } enum Platform { Window, Android } Platform mPlatform = Platform.Android; public enum ScreenIdentificationTag { // 屏幕定位的方式,手动、半自动、自动 Manual, SemiAuto, Auto } // 2个灯,顺序根据红外灯的大小 由大到小, 坐标通过 InfraredSpot.ScreenUV 和 InfraredSpot.CameraLocation 获得 public InfraredSpot[] InfraredSpots { get { return infraredSpotBuffer; } } // 1个灯, 坐标通过 InfraredSpot.ScreenUV 和 InfraredSpot.CameraLocation 获得 public InfraredSpot InfraredSpotSingle { get { return infraredSpotBuffer[0]; } } public InfraredSpot[] infraredSpotBuffer => infraredLocate.InfraredSpotBuffer; // 当前 应用/生效 的屏幕四边形数据(QuadrilateralInCamera类) public QuadrilateralInCamera CurrentScreenQuad => screenIdentification.Screen.QuadInCamera; /// /// 定位之后,可能有3种结果(手动、半自动、自动),从中选择一种作为最终识别到的屏幕。 /// 如果选择的是null,即没有识别到屏幕,则返回false,否则返回true /// public bool SelectScreenAfterLocate(ScreenIdentificationTag tag) => ScreenIdentification.SelectScreenAfterLocate(tag); /// 上一次屏幕定位的情况, 还未识别(或识别失败)的时候返回值是null public QuadrilateralInCamera LastQuadState(ScreenIdentificationTag tag) { QuadrilateralInCamera target = tag switch { ScreenLocate.ScreenIdentificationTag.Manual => ScreenIdentification.QuadManual, ScreenLocate.ScreenIdentificationTag.SemiAuto => ScreenIdentification.QuadSemiAuto, ScreenLocate.ScreenIdentificationTag.Auto => ScreenIdentification.QuadAuto, _ => null }; return target; } /// /// 上一次半自动识别的情况, 还未识别的时候数组是null /// 通过索引获取布尔值,false代表这条边识别失败(回退应用了手动数据), 0-下、1-右、2-上、3-左 /// public bool[] LastQuadSemiAutoState() => screenIdentification.LastQuadSemiAutoState; /// /// 获取算法执行过程中输出的纹理,0原图,1半自动识别到的全部线段,2屏幕黑白色差,3识别结果,4屏幕色差叠加识别结果,5半自动时的备选线段 /// public Texture2D[] OutputTextures => outputTexture2D; public Vector2 CameraLocationOffset { get=>infraredLocate.CameraLocationOffset; set{ infraredLocate.CameraLocationOffset = value; } } public Vector2 UVOffset { get=>infraredLocate.UVOffset; set{ infraredLocate.UVOffset = value; } } //用来记录最后一次更新的数据 Vector2 OldCameraLocationOffset { get; set; } = new Vector2(0, 0); //用来记录最后一次更新的数据 Vector2 OldUVOffset { get; set; } = new Vector2(0, 0); //是否单点显示 public bool bSinglePoint => infraredLocate.bSinglePoint; // public InfraredDemo InfraredDemoMain => FindObjectOfType(); #region UVC 处理的对象 //public UVCManager mUVCManager; public CameraInfo mUVCCameraInfo; public bool getUVCCameraInfo => mUVCCameraInfo != null ? true : false; public Vector2 getUVCCameraInfoSize => getUVCCameraInfo ? mUVCCameraInfo.Size : new Vector2(320, 240); private Texture mUVCTexture; public Texture getUVCTexture => mUVCTexture; public Texture setUVCTexture { set { mUVCTexture = value; } } private Texture2D mUVCTexture2D; // [SerializeField] Texture2DArray mUVCOutArray; #endregion public Text Info; public List CrosshairInCamera; public List CrosshairInScreen; public RectTransform ScreenQuad; public Toggle SaveToggle; public Toggle FullScreenToggle; public Toggle SingleToggle; public LineGenerator UILineGenerator; public bool ShowScreenQuad = false; // 显示在demo上的rawImage public List outputRawImages; readonly Texture2D[] outputTexture2D = new Texture2D[8]; public RawImage FullScreenImage; public PixelCheaker ScreenPixelCheaker; public List DebugScreenImages = new List(); public bool DebugOnZIMDemo = false; // private SynchronizationContext mainContext; public float ReDoLocateCalibrationRatio { get; private set; } // 半自动定位时校准的距离比例,以手动的结果来校准,离手动太远的线段会被舍弃 public void SetCameraSize(Vector size) => cameraSize = size; public override Vector CameraSize => cameraSize; // 记录算法中的CameraSize,红外识别和屏幕识别都会使用到 Vector cameraSize; bool bIdentifyRed = true;//默认设备红色 bool bIdentifyGreen = true; #region 性能检测相关 public Text m_UITime; const float m_UIUpdateInterval = 0.1f; float m_UIUpdateTimer = 0.0f; List m_History = new List(100); int m_ValidHistoryFrames = 0; float m_AverageTime = float.NaN; float m_MedianTime = float.NaN; float m_MinTime = float.NaN; float m_MaxTime = float.NaN; public float updateInterval = 0.5F; private double lastInterval; private int frames = 0; private float fps; public Text m_FPS; #endregion #region PC部分参数 //亮度 public float pcBrightness { get; set; } = 0.0f; //对比度 public float pcContrast { get; set; } = 0.0f; #endregion // 红外灯识别算法 InfraredLocate infraredLocate; // 屏幕识别算法 o0.Project.ScreenIdentification screenIdentification; public o0.Project.ScreenIdentification ScreenIdentification => screenIdentification; RectTransform canvas; Mode mode; //List<(Vector2 pos, GameObject go)> pointManual = new List<(Vector2, GameObject)>(); //o0.Project.WebCam o0WebCam = null; /// /// 正在识别的状态,自动识别时候记录 /// bool bAutomaticRecognition { get; set; } = false;//进行捕获时 bool bAutomaticRecognitionStart { get; set; } = false;//是否进行捕获 bool bAutomaticRecognitionEnd { get; set; } = false;//是否结束捕获 [NonSerialized] public RectTransform BackQuad = null; static public ScreenLocate Main { get; private set; } static public void AutoLightPixels(Color[] pixels, int width, int height) { if (Main.DebugOnZIMDemo) { var newTex = pixels.zimAutoLightSimple(width, height); DebugTexture(7, newTex); try { Main.FullScreenImage.texture = newTex; } catch { } } } static public void DebugTexture(int index, Texture2D texture) { LateDestory(Main.outputTexture2D[index]); Main.outputTexture2D[index] = texture; try { Main.outputRawImages[index].texture = texture; } catch { } } static void LateDestory(UnityEngine.Object o) => Main.StartCoroutine(Main.LateDestoryIEnum(o)); static public void SetScreen(UnityEngine.Color? color = null) { if (Main.BackQuad == null) { var canvas = GameObject.Find("WebCameraView").GetComponent(); var background = canvas.Find("Background"); Main.BackQuad = background.GetChild(0).GetComponent(); } Main.BackQuad.parent.gameObject.SetActive(color != null); Main.BackQuad.GetComponent().color = color ?? Color.black; //Debug.Log("Set Screen " + color.GetColorName()); } static public void SetScreen(Rect rect, UnityEngine.Color? color = null) { if (Main.BackQuad == null) { var canvas = GameObject.Find("WebCameraView").GetComponent(); var background = canvas.Find("Background"); Main.BackQuad = background.GetChild(0).GetComponent(); } Main.BackQuad.parent.gameObject.SetActive(color != null); Main.BackQuad.anchorMin = rect.min; Main.BackQuad.anchorMax = rect.max; Main.BackQuad.GetComponent().color = color ?? Color.black; //Debug.Log("Set Screen " + color.GetColorName()); } static void DebugBackQuad(Rect? rect = null) { if (Main.BackQuad) { Main.BackQuad.parent.GetComponent().enabled = false; Main.BackQuad.GetComponent().color = Color.white; Main.BackQuad.parent.gameObject.SetActive(!Main.BackQuad.parent.gameObject.activeSelf); if (rect.HasValue) { Main.BackQuad.anchorMin = rect.Value.min; Main.BackQuad.anchorMax = rect.Value.max; } } } //public void ReSizeTexture(int width, int height) //{ // Debug.Log("Cur mUVCTexture Size: [" + mUVCTexture.width + "," + mUVCTexture.height + "]"); // if (mUVCTexture.width < width || mUVCTexture.height < height) // 如果当前分辨率太小,则重新new一个texture // { // Texture2D tex = new Texture2D( // width, height, // TextureFormat.ARGB32, // false, /* mipmap */ // true /* linear */); // tex.filterMode = FilterMode.Point; // tex.Apply(); // mUVCTexture = tex; // mUVCCameraInfo.previewTexture = tex; // var nativeTexPtr = mUVCCameraInfo.previewTexture.GetNativeTexturePtr(); // } //} void Awake() { if (Main != null) throw new Exception("[ScreenLocaer] 不允许多个实例"); Main = this; #if !UNITY_EDITOR_WIN DebugOnZIMDemo = false; #endif //if (mUVCDrawer) // mUVCDrawer.StartPreviewAction += UVCIsReady; } void OnDestroy() { //if (mUVCDrawer) // mUVCDrawer.StartPreviewAction -= UVCIsReady; } void Start() { //mainContext = SynchronizationContext.Current; canvas = transform.GetComponent(); mode = Mode.InfraredLocate; if (DebugScreenImages.Count != 0 && DebugOnZIMDemo) { screenIdentification = new o0.Project.ScreenIdentification(this); screenIdentification.LocateScreen(); } ReDoLocateCalibrationRatio = 0.125f; #region 性能检测相关 for (var i = 0; i < m_History.Capacity; ++i) { m_History.Add(0.0f); } lastInterval = Time.realtimeSinceStartup; frames = 0; #endregion } // 初始化算法 void AlgorithmInit() { if (screenIdentification == null) { screenIdentification = new o0.Project.ScreenIdentification(this); Debug.Log("[ScreenLocate] 初始化屏幕识别"); //screenIdentification.OnLocateScreenEnter += OnLocateScreenEnter; screenIdentification.OnLocateScreenEnd += OnLocateScreenEnd; //初始化屏幕数据 InfraredCameraHelper.InitScreenLocateManual(); } if (infraredLocate == null) { infraredLocate = new InfraredLocate(this, screenIdentification.Screen); cameraSize = new Vector(getUVCCameraInfoSize.x, getUVCCameraInfoSize.y); InfraredSpot.RefreshMinVerifyLength(new o0.Geometry2D.Float.Vector(getUVCCameraInfoSize.x, getUVCCameraInfoSize.y)); Debug.Log($"[ScreenLocate] 初始化红外灯识别, 当前相机分辨率: {CameraSize.x}×{CameraSize.y},红外算法追踪距离: {InfraredSpot.MinVerifyLength}"); //InfraredDemo 初始化 //float redfilterValue = PlayerPrefs.GetFloat("Init redFilterSliderValue", 0.8f); //Debug.Log("Init Red filterValue:" + redfilterValue); //infraredLocate.SetBrightnessThreshold(redfilterValue); // 参数是 红外灯的亮度阈值,阈值越小能够检测到的亮度就越低,默认值是0.93 // UI相关 infraredLocate.SetSinglePoint(SingleToggle.isOn); SingleToggle.onValueChanged.AddListener((i) => { infraredLocate.SetSinglePoint(i); }); } } IEnumerator LateDestoryIEnum(UnityEngine.Object o) { if (o) { yield return new WaitForEndOfFrame(); Destroy(o); } } //ZIMWebCamera场景使用 public void WebCamIsReady(Texture texture) { mPlatform = Platform.Window; mUVCTexture = texture; mUVCCameraInfo = new CameraInfo(mUVCTexture); brightness = 0; //UVC准备好 InfraredCameraHelper?.InvokeOnUVCIsReady(mUVCCameraInfo); } /// /// UVCManager 创建初始化时候,更新此函数 /// /// public void UVCIsReady(CameraInfo cameraInfo) { mPlatform = Platform.Android; mUVCTexture = cameraInfo.previewTexture; mUVCCameraInfo = cameraInfo; Debug.Log("UVCIsReady:" + mUVCCameraInfo); //UVC准备好 InfraredCameraHelper?.InvokeOnUVCIsReady(mUVCCameraInfo); } /// /// 获取新的 previewTexture /// public void UVCUpdate(bool bChange) { mUVCTexture = mUVCCameraInfo.previewTexture; Debug.Log("[ScreenLocate] UVCUpdate:" + mUVCCameraInfo + ",bChange:" + bChange); InfraredCameraHelper?.InvokeOnUVCIsUpdate(); //这里判断是否进入自动识别? if (bAutomaticRecognitionStart) { bAutomaticRecognitionStart = false; Debug.Log("[ScreenLocate] UVCUpdate 开始自动识别 Capture:" + Capture + " ,Delay: " + Delay); screenIdentification.LocateScreen(Capture, Delay); } if (bAutomaticRecognitionEnd) { bAutomaticRecognitionEnd = false; Debug.Log("[ScreenLocate] UVCUpdate 结束捕获,当前摄像机分辨率为: " + mUVCCameraInfo.Size); bAutomaticRecognition = false; } } /// /// 选择模式后更新 quadUnityVectorList /// public void UpdateQuadUnityVectorList() { quadUnityVectorList = screenIdentification.Screen.QuadInCamera.GetUnityVertexNormalizedList(); SaveScreenLocateVectorList(); } int brightness = 0; /// /// 设置算法红外灯的亮度值 /// /// public void SetInfraredLocateBrightnessThreshold(float value) { if (infraredLocate != null) infraredLocate.SetBrightnessThreshold(value); // 参数是 红外灯的亮度阈值,阈值越小能够检测到的亮度就越低,默认值是0.93 } void Update() { //++frames; //float timeNow = Time.realtimeSinceStartup; //if (timeNow > lastInterval + updateInterval) //{ // fps = (float)(frames / (timeNow - lastInterval)); // frames = 0; // lastInterval = timeNow; //} //if (m_FPS != null) // m_FPS.text = "FPS:" + fps.ToString("f2"); if (mUVCCameraInfo == null) return; AlgorithmInit(); /* New*/ //Debug.Log((mUVCCameraInfo != null) +" = "+ mUVCCameraInfo.IsPreviewing + " = "+ screenIdentification.Screen.Active); if (mUVCCameraInfo != null && mUVCCameraInfo.IsPreviewing) { //if (bAutomaticRecognition) //{ // //识别的过程使用的分辨率 // //CreateUVCTexture2DIfNeeded((int)getUVCCameraInfoSize.x, (int)getUVCCameraInfoSize.y); // if (log1) // { // log1 = false; // Debug.Log("[ScreenLocate] log1:[" + (int)getUVCCameraInfoSize.x + ", " + (int)getUVCCameraInfoSize.y + "]"); // } //} //else //{ // //自动识别完成后使用相机分辨率大小 getUVCCameraInfoSize // //CreateUVCTexture2DIfNeeded((int)getUVCCameraInfoSize.x, (int)getUVCCameraInfoSize.y); // if (log2) // { // log2 = false; // Debug.Log("[ScreenLocate] log2:[" + (int)getUVCCameraInfoSize.x + ", " + (int)getUVCCameraInfoSize.y + "]"); // } //} //如果是连接了蓝牙设备,并且不是9轴设备。不进行识别算法处理 if (BluetoothAim.ins?.status == BluetoothStatusEnum.ConnectSuccess && AimHandler.ins && AimHandler.ins.bRuning9Axis()) return; //根据getUVCCameraInfoSize 分辨率渲染 CreateUVCTexture2DIfNeeded((int)getUVCCameraInfoSize.x, (int)getUVCCameraInfoSize.y); if (!screenIdentification.Update(mUVCTexture2D)) { // 同步分辨率, 分辨率变化后还需同步到InfraredDemo if (RefreshCameraSize()) { if (screenIdentification.Screen.QuadInCamera != null) { quadUnityVectorList = screenIdentification.Screen.QuadInCamera.GetUnityVertexNormalizedList(); if (!ContainsNaN(quadUnityVectorList)) { SaveScreenLocateVectorList(); //SyncInfraredDemo(); //SyncInfraredScreenPositioningView(); InfraredCameraHelper?.InvokeOnUVCPosUpdate(quadUnityVectorList); Debug.Log("[ScreenLocate] RefreshCameraSize 屏幕size改变:[" + (int)getUVCCameraInfoSize.x + "," + (int)getUVCCameraInfoSize.y + "]"); Debug.Log("[ScreenLocate] RefreshCameraSize 屏幕size改变,刷新quadUnityVectorList:" + PrintVector2List(quadUnityVectorList)); } else { Debug.LogError("[ScreenLocate] RefreshCameraSize 屏幕size改变,存在NaN值,重新校准:" + PrintVector2List(quadUnityVectorList)); } } if (DebugOnZIMDemo) Main.ShowScreen(screenIdentification.Screen.QuadInCamera); } if (CameraSize.x != mUVCTexture2D.width || CameraSize.y != mUVCTexture2D.height) { Debug.Log($"[ScreenLocate] 分辨率不匹配,相机分辨率为: {getUVCCameraInfoSize}, mUVCTexture2D纹理尺寸: {mUVCTexture2D.width}×{mUVCTexture2D.height}"); return; } // 获取像素,用于后续操作 var pixels = mUVCTexture2D.GetPixels(); // 从左往右、从下往上 AutoLightPixels(pixels, CameraWidth, CameraHeight); InfraredUpdate(pixels); if (mode == Mode.ScreenLocateManual) { for (int i = 0; i < infraredSpotBuffer.Length; i++) { if (infraredSpotBuffer[i].CameraLocation != null) { // 检测到光点 var posInCanvas = infraredSpotBuffer[i].CameraLocation.Value.pixelToLocalPosition_AnchorCenter(CameraSize, FullScreenImage.rectTransform.rect); CrosshairInCamera[i].gameObject.SetActive(true); CrosshairInCamera[i].anchoredPosition = posInCanvas; } else CrosshairInCamera[i].gameObject.SetActive(false); } } else if (mode == Mode.InfraredLocate) { if (mPlatform == Platform.Window) //渲染ui上面的点。进入游戏可以隐藏 { for (int i = 0; i < infraredSpotBuffer.Length; i++) { if (infraredSpotBuffer[i].CameraLocation != null) { // 检测到光点 var posInCanvas = infraredSpotBuffer[i].CameraLocation.Value.pixelToLocalPosition_AnchorCenter(CameraSize, outputRawImages[0].rectTransform.rect); CrosshairInCamera[i].gameObject.SetActive(true); CrosshairInCamera[i].anchoredPosition = posInCanvas; } else CrosshairInCamera[i].gameObject.SetActive(false); } } //手机端使用 mPlatform == Platform.Android && //通用,手机 和 PC if (infraredSpotBuffer.Length > 0) { int redIndex = 0; int greenIndex = 1; //仅仅第一个点显示(如果最大点出界了会闪烁) if (bSinglePoint) { redIndex = 0; //单点识别是,可以选择切换颜色 if (infraredSpotBuffer[redIndex].ScreenUV != null) { string str = "Single:"; Info.text = str + infraredSpotBuffer[redIndex].ScreenUV.Value.ToString("F4"); //InfraredManager.ConnetDevicesSingle.ins.posAction?.Invoke(new Vector3(infraredSpotBuffer[redIndex].ScreenUV.Value.x * Screen.width, infraredSpotBuffer[redIndex].ScreenUV.Value.y * Screen.height, 0)); onFilterPos(infraredSpotBuffer[redIndex].ScreenUV.Value); } } else { //雙點模式下選擇第一個點 if (bIdentifyRed && !bIdentifyGreen) { if (infraredSpotBuffer[redIndex].ScreenUV != null) { Info.text = "Red" + redIndex + ":" + infraredSpotBuffer[redIndex].ScreenUV.Value.ToString("F4"); //InfraredManager.ConnetDevicesSingle.ins.posAction?.Invoke(new Vector3(infraredSpotBuffer[redIndex].ScreenUV.Value.x * Screen.width, infraredSpotBuffer[redIndex].ScreenUV.Value.y * Screen.height, 0)); onFilterPos2(infraredSpotBuffer[redIndex].ScreenUV.Value, redIndex); } else { Info.text = "未检测到红色最大点!"; } } else if (!bIdentifyRed && bIdentifyGreen) { if (infraredSpotBuffer[greenIndex].ScreenUV != null) { Info.text = "Green:" + infraredSpotBuffer[greenIndex].ScreenUV.Value.ToString("F4"); //InfraredManager.ConnetDevicesSingle.ins.posAction?.Invoke(new Vector3(infraredSpotBuffer[greenIndex].ScreenUV.Value.x * Screen.width, infraredSpotBuffer[greenIndex].ScreenUV.Value.y * Screen.height, 0)); onFilterPos2(infraredSpotBuffer[greenIndex].ScreenUV.Value, greenIndex); } else { Info.text = "未检测到绿色点!"; } } else { //两个不选择和两个全选都跑识别两个点 //自動切換 检测到光点 if (infraredSpotBuffer[redIndex].ScreenUV != null) { Info.text = "Red:" + infraredSpotBuffer[redIndex].ScreenUV.Value.ToString("F4"); //InfraredManager.ConnetDevicesSingle.ins.posAction?.Invoke(new Vector3(infraredSpotBuffer[redIndex].ScreenUV.Value.x * Screen.width, infraredSpotBuffer[redIndex].ScreenUV.Value.y * Screen.height, 0)); onFilterPos2(infraredSpotBuffer[redIndex].ScreenUV.Value, redIndex); } else if (infraredSpotBuffer[greenIndex].ScreenUV != null) { Info.text = "Green:" + infraredSpotBuffer[greenIndex].ScreenUV.Value.ToString("F4"); //InfraredManager.ConnetDevicesSingle.ins.posAction?.Invoke(new Vector3(infraredSpotBuffer[greenIndex].ScreenUV.Value.x * Screen.width, infraredSpotBuffer[greenIndex].ScreenUV.Value.y * Screen.height, 0)); onFilterPos2(infraredSpotBuffer[greenIndex].ScreenUV.Value, greenIndex); } else { Info.text = "未检测到点!"; } } } } } else if (mode == Mode.ScreenMap && DebugOnZIMDemo) { for (int i = 0; i < infraredSpotBuffer.Length; i++) { if (infraredSpotBuffer[i].ScreenUV != null) { // 检测到光点 var posInCanvas = infraredSpotBuffer[i].ScreenUV.Value.pixelToLocalPosition_AnchorCenter(new Vector2(1, 1), canvas.rect); CrosshairInScreen[i].gameObject.SetActive(true); CrosshairInScreen[i].anchoredPosition = posInCanvas; } else CrosshairInScreen[i].gameObject.SetActive(false); } if (Input.GetKeyDown(KeyCode.Escape)) ToMode(Mode.InfraredLocate); } } } //var t1 = Time.realtimeSinceStartup; //var dt = t1 - t0; //m_History[m_ValidHistoryFrames % m_History.Count] = dt; //++m_ValidHistoryFrames; //m_UIUpdateTimer += Time.deltaTime; //if (m_UIUpdateTimer >= m_UIUpdateInterval) //{ // m_UIUpdateTimer = 0.0f; // if (m_ValidHistoryFrames >= m_History.Count) // { // m_ValidHistoryFrames = 0; // m_AverageTime = 0.0f; // m_MinTime = float.PositiveInfinity; // m_MaxTime = float.NegativeInfinity; // { // for (var i = 0; i < m_History.Count; i++) // { // var time = m_History[i]; // m_AverageTime += time; // m_MinTime = Mathf.Min(m_MinTime, time); // m_MaxTime = Mathf.Max(m_MaxTime, time); // } // m_AverageTime /= m_History.Count; // } // { // m_History.Sort(); // // Odd-length history? // if ((m_History.Count & 1) != 0) // { // m_MedianTime = m_History[m_History.Count / 2]; // } // else // { // m_MedianTime = (m_History[m_History.Count / 2] + m_History[m_History.Count / 2 - 1]) / 2.0f; // } // } // } // var statistics = $"{m_History.Count} 帧样本:\naverage: {m_AverageTime * 1000.0f:F2}ms\nmedian: {m_MedianTime * 1000.0f:F2}ms\nmin: {m_MinTime * 1000.0f:F2}ms\nmax: {m_MaxTime * 1000.0f:F2}ms\n"; // //Method: {m_Method} {UnityEngine.SceneManagement.SceneManager.GetActiveScene().name} | // if (m_UITime != null) // m_UITime.text = $"Cam: {mUVCCameraInfo.CurrentWidth}x{mUVCCameraInfo.CurrentHeight}{(mUVCTexture2D? ",T2D:" : "")}{(mUVCTexture2D? mUVCTexture2D.width+ "x" : "")}{(mUVCTexture2D ? mUVCTexture2D.height:"")} \nLast Frame: {dt * 1000.0f:F2}ms \n{statistics}"; //} //UpdateInputs(); if (DebugOnZIMDemo) { if (Input.GetKeyDown(KeyCode.Z)) SelectScreenAfterLocate(ScreenIdentificationTag.Manual); if (Input.GetKeyDown(KeyCode.X)) SelectScreenAfterLocate(ScreenIdentificationTag.SemiAuto); if (Input.GetKeyDown(KeyCode.C)) SelectScreenAfterLocate(ScreenIdentificationTag.Auto); } } public void InfraredUpdate(Color[] cameraPixels) { infraredLocate.InfraredUpdate(cameraPixels); if (DebugOnZIMDemo) DebugPixelSpotArea(infraredLocate.DebugAreas); } private bool RefreshCameraSize() { var sizeNew = new Vector(getUVCCameraInfoSize.x, getUVCCameraInfoSize.y); var sizeNewFloat = getUVCCameraInfoSize.o0Vector(); if (sizeNew != CameraSize || (screenIdentification?.Screen?.QuadInCamera != null && sizeNewFloat != screenIdentification.Screen.QuadInCamera.CameraSize)) { Debug.Log($"[ScreenLocate] 分辨率变化,刷新分辨率(from {CameraSize.x}×{CameraSize.y} to {sizeNew.x}×{sizeNew.y}), 是否有屏幕数据: {screenIdentification.Screen.QuadInCamera != null}, 是否有手动数据: {screenIdentification.QuadManual != null}"); // 同步相机分辨率 cameraSize = sizeNew; screenIdentification.Screen.RefreshCameraSize(sizeNewFloat); screenIdentification.QuadAuto?.ReSize(sizeNewFloat, ScreenMap.ViewAspectRatioSetting); screenIdentification.QuadManual?.ReSize(sizeNewFloat, ScreenMap.ViewAspectRatioSetting); screenIdentification.QuadSemiAuto?.ReSize(sizeNewFloat, ScreenMap.ViewAspectRatioSetting); InfraredSpot.RefreshMinVerifyLength(sizeNewFloat); return true; } return false; } Vector2 targetPos = Vector2.zero; Vector2 movePos = Vector2.zero; int moveSpeed = 20; public float filterDis = 3.0f; void onFilterPos(Vector2 _vector2Pos) { //主要用于模拟九轴时候的 //添加一个偏移量,使得最后输出的准心是指向正中心 Vector2 np = new Vector2((_vector2Pos.x - UVOffset.x) * Screen.width, (_vector2Pos.y - UVOffset.y) * Screen.height); //_vector2Pos.pixelToLocalPosition_AnchorCenter(Vector2.one, (transform as RectTransform).rect); if (Vector2.Distance(np, targetPos) >= filterDis) { targetPos = np; //InfraredManager.ConnetDevicesSingle.ins.posAction?.Invoke(new Vector3(targetPos.x, targetPos.y, 0)); //Vector2 np = new Vector2(uvCenterOffset.x * Screen.width, uvCenterOffset.y * Screen.height); //point -= np; InfraredCameraHelper?.InvokeOnPositionUpdate(targetPos); } //movePos = Vector3.Lerp(movePos, targetPos, Time.deltaTime * moveSpeed); //InfraredManager.ConnetDevicesSingle.ins.posAction?.Invoke(new Vector3(movePos.x, movePos.y, 0)); } Vector2[] _targetPoints2 = new Vector2[] { Vector2.zero, Vector2.zero }; void onFilterPos2(Vector2 _vector2Pos, int index) { Vector2 np = new Vector2((_vector2Pos.x - UVOffset.x) * Screen.width, (_vector2Pos.y - UVOffset.y) * Screen.height); if (Vector2.Distance(np, _targetPoints2[index]) >= filterDis) { _targetPoints2[index] = np; InfraredCameraHelper?.InvokeOnPositionUpdate2(_targetPoints2[index], index); } } public void DebugPixelSpotArea(List areas) { if (areas!=null) { Info.transform.GetChild(0).GetComponent().text = $"areas.Count: {areas.Count}"; PixelSpotArea a0 = null; // 表示最大半径的区域 PixelSpotArea a1 = null; // 表示第二大半径的区域 foreach (var a in areas) { if (a0 == null || a.Radius > a0.Radius) { a1 = a0; // 更新第二大为之前最大 a0 = a; // 更新最大为当前的 } else if (a1 == null || a.Radius > a1.Radius) { a1 = a; // 更新第二大 } } Texture2D texture = new Texture2D(CameraWidth, CameraHeight); Color[] blackPixels = new Color[texture.width * texture.height]; for (int i = 0; i < blackPixels.Length; i++) blackPixels[i] = Color.black; texture.SetPixels(blackPixels); if (a0 != null) { foreach (var p in a0.Pixels0) texture.SetPixel((int)p.x, (int)p.y, Color.yellow); foreach (var p in a0.Pixels1) texture.SetPixel((int)p.x, (int)p.y, Color.white); } if (a1 != null) { foreach (var p in a1.Pixels0) texture.SetPixel((int)p.x, (int)p.y, Color.green); foreach (var p in a1.Pixels1) texture.SetPixel((int)p.x, (int)p.y, Color.blue); } texture.Apply(); DebugTexture(6, texture); } } #region 自动识别 int Capture = 30; int Delay = 30; Vector2 EnterResolution; // int DefaultResolutionIndex; // readonly public int HighScreenLocateResolutionIndex = 2; // 自动识别时,摄像机分辨率固定为1280 * 720 ( 对应索引是2 ) public void BtnScreenLocate() { if (DebugScreenImages.Count != 0) { //screenIdentification = new o0.Project.ScreenIdentification(); cameraSize = new Vector(DebugScreenImages[0].width, DebugScreenImages[0].height); WebCamIsReady(DebugScreenImages[0]); CreateUVCTexture2DIfNeeded(); } //Debug.Log("BtnScreenLocate Capture:" + Capture + " ,Delay: " + Delay); //screenIdentification.LocateScreen(Capture, Delay); OnLocateScreenEnter(); } // bool log1 = false, log2 = false; public void OnLocateScreenEnter() { bAutomaticRecognition = true; bAutomaticRecognitionStart = true; ResetScreenIdentification(); //DefaultResolutionIndex = InfraredDemoMain?.ResolutionIndex ?? 0; // 记录一下进入前的分辨率(游戏场景的分辨率,比识别时更低) //HighScreenLocateResolutionIndex = InfraredDemoMain.getTextureToResolutionNewIndex(); // index = 0 // Debug.Log("[ScreenLocate] 开始捕获 DefaultResolutionIndex:" + DefaultResolutionIndex + " ,HighScreenLocateResolutionIndex:" + HighScreenLocateResolutionIndex); // InfraredDemoMain?.SetResolutionNew(HighScreenLocateResolutionIndex); EnterResolution = mUVCCameraInfo.Size;// 记录一下进入前的分辨率(游戏场景的分辨率,比识别时更低) Vector2 _HighResolution = mUVCCameraInfo.CurrentCalibrationResolution; //最高的分辨率 Resize((int)_HighResolution.x, (int)_HighResolution.y); //CreateUVCTexture2DIfNeeded(); // log1 = true; // log2 = true; screenIdentification.LocateScreen(); // 自动识别开始的入口 if (DebugOnZIMDemo) { var webCam = GetComponent(); webCam.AdjustResolution(1920, 1080); mUVCCameraInfo.SetSize(webCam.webCamTexture.width, webCam.webCamTexture.height); } } /// /// 屏幕识别结束 /// public void OnLocateScreenEnd() { bAutomaticRecognitionEnd = true; // 记录本次屏幕识别的分辨率(目前采用高分辨率做识别,识别结束后调回低分辨率) //InfraredDemoMain?.SetResolutionNew(DefaultResolutionIndex); Resize((int)EnterResolution.x, (int)EnterResolution.y); if (DebugOnZIMDemo) { var webCam = GetComponent(); GetComponent().AdjustResolution((int)EnterResolution.x, (int)EnterResolution.y); mUVCCameraInfo.SetSize(webCam.webCamTexture.width, webCam.webCamTexture.height); } // 前面的数据ContainsNaN(quadUnityVectorList) 如果没处理到,这里再进行判断一次 // 识别结束后,也要判断半自动数据,即P5? if (screenIdentification.QuadAuto == null && screenIdentification.QuadSemiAuto == null) { InfraredCameraHelper?.InvokeOnUVCPosUpdate(new List()); Debug.LogError("[ScreenLocate] OnLocateScreenEnd 屏幕未识别"); } } /** * 修改相机的实际分辨率 */ public void Resize(int width, int height) { if (mUVCCameraInfo == null) return; #if UNITY_ANDROID //发送修改指令给相机实际分辨率 mUVCCameraInfo.SetCameraSize(width, height); #endif #if UNITY_STANDALONE_WIN // pc todo 看看怎么处理 // ResizePC(width, height); #endif //mUVCCameraInfo.SetSize(width, height); // 手动记录分辨率,这里可能会有问题 width和height是期望的分辨率而不是当前摄像机实际分辨率 Debug.Log($"[ScreenLocate] 开始修改分辨率 mUVCCameraInfo origin:[{mUVCCameraInfo.CurrentWidth},{mUVCCameraInfo.CurrentHeight}]=>target:[{width},{height}]"); // if (screenIdentification.isInitLocateScreen()) screenIdentification.bStartLocateScreen = true; } /// /// pc修改分辨率 /// /// /// public void ResizePC(int width, int height) { if (mUVCCameraInfo == null) return; //if (screenIdentification.isInitLocateScreen()) screenIdentification.bStartLocateScreen = true; // PcWebCamera pcWebCamera = GetComponent(); // if(pcWebCamera.webCamTexture == null || !pcWebCamera.webCamTexture.isPlaying) return; //StartCoroutine(ResetWebCam(pcWebCamera, width, height)); mUVCCameraInfo.SetSize(width, height); // 手动记录分辨率,这里可能会有问题 width和height是期望的分辨率而不是当前摄像机实际分辨率 Debug.Log("[ScreenLocate] Resize mUVCCameraInfo.SetSize: [" + mUVCCameraInfo.CurrentWidth + "," + mUVCCameraInfo.CurrentHeight + "]"); } private System.Collections.IEnumerator ResetWebCam(PcWebCamera pcWebCamera, int newWidth, int newHeight) { WebCamTexture _webCamTexture = pcWebCamera.webCamTexture; // Stop the current WebCamTexture _webCamTexture.Stop(); // Trigger OnWebCamStopped event // OnWebCamStopped?.Invoke(); // Wait for a short time to ensure resources are released yield return new WaitForSeconds(0.5f); // Create a new WebCamTexture with the new dimensions _webCamTexture = new WebCamTexture(newWidth, newHeight); pcWebCamera.webCamTexture = _webCamTexture; mUVCTexture = _webCamTexture; // Restart the camera yield return StartCoroutine(StartWebCam(pcWebCamera)); } private System.Collections.IEnumerator StartWebCam(PcWebCamera pcWebCamera) { WebCamTexture _webCamTexture = pcWebCamera.webCamTexture; _webCamTexture.Play(); // Wait until the WebCamTexture is playing while (!_webCamTexture.isPlaying) { yield return null; } // Trigger OnWebCamStarted event //OnWebCamStarted?.Invoke(); mUVCCameraInfo.SetSize(_webCamTexture.width, _webCamTexture.height); // 手动记录分辨率,这里可能会有问题 width和height是期望的分辨率而不是当前摄像机实际分辨率 Debug.Log("[ScreenLocate] ResizePc mUVCCameraInfo.SetSize: [" + mUVCCameraInfo.CurrentWidth + "," + mUVCCameraInfo.CurrentHeight + "]"); // if(screenIdentification.isInitLocateScreen())screenIdentification.bStartLocateScreen = true; } #endregion public void BtnScreenMap() { ToMode(Mode.ScreenMap); } //进入手动定位屏幕 public void BtnScreenLocateManual() { ToMode(Mode.ScreenLocateManual); } // 重置屏幕识别的数据 public void ResetScreenIdentification() { screenIdentification.Screen.Active = false; } // threshold 的值是0-1,0代表最近,1代表最远 public void SetReDoLocateCalibrationRatio(float threshold) { const float MIN = 0.02f; const float MAX = 0.32f; ReDoLocateCalibrationRatio = MIN + (MAX - MIN) * threshold; } /// /// 固定的顶点顺序: 左下,右下,左上,右上 /// public static List quadUnityVectorList = new(); /// /// 打印信息 /// /// 左下,右下,左上,右上 /// public string PrintVector2List(List list) { if (screenIdentification == null || !screenIdentification.Screen.Active) return "[]"; string result = ""; if (list.Count == 4) { result = "左下" + list[0].ToString() + ",右下" + list[1].ToString() + ",左上" + list[2].ToString() + ",右上" + list[3].ToString(); } else { result = "count != 4 error"; } //foreach (Vector2 vector in list) //{ // result += vector.ToString() + " "; //} //Debug.Log(result); return result; } /// /// 判断是否存在NaN /// /// /// public bool ContainsNaN(List vectors) { foreach (var v in vectors) { if (float.IsNaN(v.x) || float.IsNaN(v.y)) { return true; } } return false; } // 标记屏幕的四个角, ScreenQuadObject 下挂了4个子节点用于标记 public void ShowScreen(RectTransform ScreenQuadObject, QuadrilateralInCamera screen) { if (screen == null) { Info.text = "识别屏幕失败"; return; } Info.text = "已识别到屏幕"; //if (ScreenQuadObject && ScreenQuadObject.childCount >= 4) //{ // ScreenQuadObject.gameObject.SetActive(true); // for (int i = 0; i < 4; i++) // { // if (DebugOnZIMDemo) // { // RectTransform t = ScreenQuadObject.GetChild(i) as RectTransform; // t.anchoredPosition = screen.Quad[i].pixelToLocalPosition_AnchorCenter(screen.CameraSize, ScreenQuadObject.rect); // } // } //} quadUnityVectorList = screen.GetUnityVertexNormalizedList(); // 记录四个点 if (!ContainsNaN(quadUnityVectorList)) { SaveScreenLocateVectorList(); //SyncInfraredDemo(); if (DebugOnZIMDemo) SyncInfraredScreenPositioningView(); InfraredCameraHelper?.InvokeOnUVCPosUpdate(quadUnityVectorList); Debug.Log("[ScreenLocate] ShowScreen 已识别到屏幕,更新quadUnityVectorList:" + PrintVector2List(quadUnityVectorList)); } else { Debug.LogError("[ScreenLocate] RefreshCameraSize 屏幕size改变,存在NaN值,重新校准:" + PrintVector2List(quadUnityVectorList)); } } public void ShowScreen(QuadrilateralInCamera screen) => ShowScreen(ScreenQuad, screen); /// /// 校准点位置存储到本地 /// static public void SaveScreenLocateVectorList() { //string saveStr = string.Join(";", quadUnityVectorList.Select(v => $"{v.x},{v.y}")); //,{v.z} // 如果列表为空,保存空字符串或自定义标记 string saveStr = quadUnityVectorList.Count > 0 ? string.Join(";", quadUnityVectorList.Select(v => $"{v.x},{v.y}")) : ""; Debug.Log("SaveScreenLocateVectorList: " + saveStr); PlayerPrefs.SetString("ScreenLocateVectorList", saveStr); } /// /// 获取本地存储校准点位置 /// static public bool GetScreenLocateVectorList() { string posListStr = PlayerPrefs.GetString("ScreenLocateVectorList", ""); Debug.Log("GetScreenLocateVectorList:" + posListStr); if (!string.IsNullOrWhiteSpace(posListStr)) { quadUnityVectorList.Clear(); quadUnityVectorList = posListStr.Split(';') .Select(s => { string[] parts = s.Split(','); return new Vector2(float.Parse(parts[0]), float.Parse(parts[1])); }) .ToList(); return true; } else return false; } public void SetSinglePoint(bool value)=> infraredLocate.SetSinglePoint(value); public bool IsSinglePoint() => infraredLocate.IsSinglePoint(); public Vector2 AdjustPointsOffset(Vector2 inputPoint, string type = "CameraLocation") => infraredLocate.GetCenterOffset(inputPoint, type); /// /// 重置偏移量 /// public void ResetPointsOffest() => infraredLocate.ResetCenterOffset(); /// /// 初始化记录值 /// /// /// /// public Vector2 SetPointsOffset(Vector2 inputPointOffset, string type = "CameraLocation") { // 计算从原始中心到输入点的偏移量 if (type == "CameraLocation") { OldCameraLocationOffset = CameraLocationOffset = inputPointOffset; return CameraLocationOffset; } else { //ScreenUV OldUVOffset = UVOffset = inputPointOffset; return UVOffset; } } /// /// 撤销操作, /// public void RevokePointsOffest() { CameraLocationOffset = OldCameraLocationOffset; UVOffset = OldUVOffset; } /// /// 这里计算一个偏移后的cameraLocatoin位置 /// /// /// public Vector2 GetOffsetCameraLocation(Vector2 cameraLocatoin) { return cameraLocatoin - CameraLocationOffset; } void ToMode(Mode mode) { if (this.mode == mode) return; if (mode == Mode.ScreenMap) { if (!screenIdentification.Screen.Active) { Info.text = "先定位屏幕"; return; } Info.text = "按ESC退出"; SetScreen(Color.black); //Info.transform.SetAsLastSibling(); this.mode = Mode.ScreenMap; } else if (mode == Mode.InfraredLocate) { Info.text = screenIdentification.Screen.Active ? "已定位屏幕" : "定位屏幕失败"; //Info.text = "已识别到屏幕"; SetScreen(null); foreach (var i in CrosshairInScreen) i.gameObject.SetActive(false); FullScreenImage.gameObject.SetActive(false); ScreenPixelCheaker?.HideImage(); //Info.transform.SetSiblingIndex(transform.childCount - 4); this.mode = Mode.InfraredLocate; #if (!NDEBUG && DEBUG && ENABLE_LOG) Console.WriteLine($"{TAG} Mode.InfraredLocate:已识别到屏幕:{screenIdentification.Screen.Active}"); #endif } else if (mode == Mode.ScreenLocateManual) { Info.text = "左键单击屏幕 左下角"; FullScreenImage.gameObject.SetActive(true); ScreenPixelCheaker?.ShowImage(); //Info.transform.SetSiblingIndex(transform.childCount - 1); // var newTex = WebCamera.webCamTexture.AutoLight(10); //DebugTexture(1, TextureToTexture2D(rawImage.texture)); CreateUVCTexture2DIfNeeded(); DebugTexture(7, mUVCTexture2D.zimAutoLight(brightness)); //mUVCTexture2DTemp = TextureToTexture2D(mUVCCameraInfo.previewTexture); //DebugTexture(6, mUVCTexture2DTemp.zimAutoLight(brightness)); this.mode = Mode.ScreenLocateManual; } } private Texture2D TextureToTexture2D(Texture texture, int width = 0, int height = 0) { if (width == 0) width = texture.width; if (height == 0) height = texture.height; Texture2D _texture2D = new Texture2D(width, height, TextureFormat.ARGB32, false, true); RenderTexture currentRT = RenderTexture.active; RenderTexture renderTexture = RenderTexture.GetTemporary( width, height, 0, RenderTextureFormat.ARGB32, RenderTextureReadWrite.Linear); Graphics.Blit(texture, renderTexture); RenderTexture.active = renderTexture; _texture2D.ReadPixels(new Rect(0, 0, width, height), 0, 0); _texture2D.Apply(); RenderTexture.active = currentRT; RenderTexture.ReleaseTemporary(renderTexture); return _texture2D; } //public void CreateUVCTexture2DFocusSizeIfNeeded(int width, int height) //{ // if (mUVCTexture2D != null) // Destroy(mUVCTexture2D); // mUVCTexture2D = TextureToTexture2D(mUVCTexture, width, height); //} /// /// 使用默认的mUVCTexture宽高 /// private void CreateUVCTexture2DIfNeeded() { if (mUVCTexture2D != null) Destroy(mUVCTexture2D); mUVCTexture2D = TextureToTexture2D(mUVCTexture); } /// /// 根据宽高调整mUVCTexture2D /// /// /// private void CreateUVCTexture2DIfNeeded(int width = 0, int height = 0) { if (mUVCTexture2D != null) Destroy(mUVCTexture2D); mUVCTexture2D = TextureToTexture2D(mUVCTexture, width, height); } #region DoubleButton private DateTime m_firstTime; private DateTime m_secondTime; private void Press() { Debug.Log("进入手动定位"); BtnScreenLocateManual(); resetTime(); } public void OnDoubleClick() { //超时重置 if (!m_firstTime.Equals(default(DateTime))) { var intervalTime = DateTime.Now - m_firstTime; float milliSeconds = intervalTime.Seconds * 1000 + intervalTime.Milliseconds; if (milliSeconds >= 400) resetTime(); } // 按下按钮时对两次的时间进行记录 if (m_firstTime.Equals(default(DateTime))) m_firstTime = DateTime.Now; else m_secondTime = DateTime.Now; // 在第二次点击触发,时差小于400ms触发 if (!m_firstTime.Equals(default(DateTime)) && !m_secondTime.Equals(default(DateTime))) { var intervalTime = m_secondTime - m_firstTime; float milliSeconds = intervalTime.Seconds * 1000 + intervalTime.Milliseconds; if (milliSeconds < 400) Press(); else resetTime(); } } private void resetTime() { m_firstTime = default(DateTime); m_secondTime = default(DateTime); } #endregion #region 性能检测相关 void InvalidateTimings() { m_ValidHistoryFrames = 0; m_AverageTime = float.NaN; m_MedianTime = float.NaN; m_MinTime = float.NaN; m_MaxTime = float.NaN; } void UpdateInputs() { //重置 if (Input.GetKeyDown(KeyCode.UpArrow)) { InvalidateTimings(); } } #endregion }