using o0.Geometry2D.Float; using System; using System.Collections; using System.Collections.Generic; using System.IO; using System.Linq; using System.Net.NetworkInformation; using System.Threading.Tasks; using TMPro; using Unity.VisualScripting; using UnityEngine; using UnityEngine.UI; using ZIM; using ZIM.Unity; public class ScreenLocateTestOld : MonoBehaviour { public ZIMWebCamera webCamera; public TextMeshProUGUI info; public TextMeshProUGUI location; public RawImage crosshairInScreen; public RawImage background; public Transform Screen; //public Texture2D TextureTest; public RawImage testImage; // 测试用 public RawImage testImage2; // 测试用 public InfraredLocateTest infraredLocateTest; Rect canvasRect; Quadrilateral screenQuad; int record; float[] brightnessRecord; // record > 0 时记录一段时间内图像亮度的均值 void Start() { canvasRect = webCamera.GetComponent().rect; record = 0; //TextureTest = new Texture2D(2, 2); //TextureTest.wrapMode = TextureWrapMode.Clamp; //TextureTest.filterMode = FilterMode.Point; //TextureTest.SetPixel(0, 0, new UnityEngine.Color(1f, 0f, 0f)); //TextureTest.SetPixel(1, 0, new UnityEngine.Color(0f, 1f, 0f)); //TextureTest.SetPixel(0, 1, new UnityEngine.Color(0f, 0f, 1f)); //TextureTest.SetPixel(1, 1, UnityEngine.Color.white); //TextureTest.Apply(); //Color[] pixels = TextureTest.GetPixels(); //location.text = pixels[0].ToString() + pixels[1].ToString() + pixels[2].ToString(); } //int frame = 2; void Update() { WebCamTexture texture = webCamera?.webCamTexture; if (texture == null) return; if (texture.didUpdateThisFrame) // 检查camera是否更新帧 { //frame--; //if (frame != 0) // return; //frame = 2; //var watch = new System.Diagnostics.Stopwatch(); //watch.Start(); //var times = new List() { 0.0 }; Color[] pixels = infraredLocateTest.pixels; if (record > 0) { if (record == 1) brightnessRecord = new float[pixels.Length]; Parallel.For(0, pixels.Length, (i) => { brightnessRecord[i] += (pixels[i].Brightness(64) - brightnessRecord[i]) / record; }); record++; } else { //var il = infraredLocateTest.InfraredLocation; //if (il != null) //{ // // 检测到亮点 // if (screenQuad != null) // 已定位了屏幕区域 // { // var uv = screenQuad.InterpolationFactors(il.Value.o0Vector()); // var posInScreen = new Vector2(canvasRect.width * (uv.x - 0.5f), canvasRect.height * (uv.y - 0.5f)); // crosshairInScreen.gameObject.SetActive(true); // crosshairInScreen.rectTransform.anchoredPosition = posInScreen; // location.text = "uv: " + uv; // } //} //else //{ // // 没有检测到亮点 // crosshairInScreen.gameObject.SetActive(false); //} } //times.Add(watch.ElapsedMilliseconds); //UnityEngine.Debug.Log("time: " + (times[times.Count - 1] - times[times.Count - 2])); } //if (Input.GetKeyUp(KeyCode.Q)) //{ // background.gameObject.SetActive(!background.gameObject.activeSelf); // testImage.gameObject.SetActive(!testImage.gameObject.activeSelf); // testImage2.gameObject.SetActive(!testImage2.gameObject.activeSelf); //} //if (Input.GetKeyUp(KeyCode.R)) //{ // SavePng("test"); //} } public void SavePng(string name, Color[] pixels = null) { var testTexture = new Texture2D((int)webCamera.width, (int)webCamera.height); testTexture.wrapMode = TextureWrapMode.Clamp; testTexture.filterMode = FilterMode.Point; if (pixels == null) pixels = webCamera.webCamTexture.GetPixels(); for (int i = 0; i < pixels.Count(); i++) { var vec = webCamera.IndexToCoord(i); testTexture.SetPixel(vec.x, vec.y, pixels[i]); } testTexture.Apply(); var bytes = testTexture.EncodeToPNG(); File.WriteAllBytes($"{name}.png", bytes); } public void SavePngFloat(string name, float[] pixels, int scale = 10) { var testTexture = new Texture2D((int)webCamera.width, (int)webCamera.height); testTexture.wrapMode = TextureWrapMode.Clamp; testTexture.filterMode = FilterMode.Point; for (int i = 0; i < pixels.Count(); i++) { var vec = webCamera.IndexToCoord(i); var f = Math.Min(pixels[i] / 64 * scale, 1); testTexture.SetPixel(vec.x, vec.y, new Color(f, f, f, 1)); } testTexture.Apply(); var bytes = testTexture.EncodeToPNG(); File.WriteAllBytes($"{name}.png", bytes); } public void DoScreenLocate() { info.gameObject.SetActive(false); StartCoroutine(ScreenLocateCorutine2()); //StartCoroutine(ScreenTestCoruine()); } // 用协程控制游戏画面闪烁,用来识别屏幕 IEnumerator ScreenLocateCorutine() { background.color = Color.white; background.gameObject.SetActive(true); yield return new WaitForSeconds(0.3f); //摄像机画面有延时 //while (true) //{ // if (webCamera.webCamTexture.didUpdateThisFrame) // break; // else // yield return null; //} record = 1; yield return new WaitForSeconds(0.8f); record = 0; float[] brightness0 = (float[])brightnessRecord.Clone(); Color[] piexl0 = webCamera.webCamTexture.GetPixels(); background.color = Color.black; yield return new WaitForSeconds(0.3f); record = 1; yield return new WaitForSeconds(0.8f); float[] brightness1 = (float[])brightnessRecord.Clone(); record = 0; Color[] piexl1 = webCamera.webCamTexture.GetPixels(); background.color = Color.white; yield return new WaitForSeconds(0.3f); record = 1; yield return new WaitForSeconds(0.8f); float[] brightness2 = (float[])brightnessRecord.Clone(); record = 0; Color[] piexl2 = webCamera.webCamTexture.GetPixels(); background.color = Color.black; yield return new WaitForSeconds(0.3f); record = 1; yield return new WaitForSeconds(0.8f); float[] brightness3 = (float[])brightnessRecord.Clone(); record = 0; Color[] piexl3 = webCamera.webCamTexture.GetPixels(); //background.gameObject.SetActive(false); yield return null; // 用 int数组 记录随着屏幕亮度变化的点,变化的记为 1,其他为 0 int[] pixelsInScreen = new int[brightness0.Length]; const float threshold0 = 0.1f; const float threshold1 = 300f; Parallel.For(0, (int)webCamera.width, (i) => { for (int j = 0; j < webCamera.height; j++) { var index = webCamera.CoordToIndex(i, j); var a0 = brightness0[index]; var a1 = brightness1[index]; var a2 = brightness2[index]; var a3 = brightness3[index]; var b0 = Math.Pow(a0 + 1, 8); var b1 = Math.Pow(a1 + 1, 8); var b2 = Math.Pow(a2 + 1, 8); var b3 = Math.Pow(a3 + 1, 8); //var b0 = getBrightness(piexl0[index]); //var b1 = getBrightness(piexl1[index]); //var b2 = getBrightness(piexl2[index]); //var b3 = getBrightness(piexl3[index]); if (a0 - a1 > threshold0 && a2 - a1 > threshold0 && a0 - a3 > threshold0 && a2 - a3 > threshold0 && b0 - b1 > threshold1 && b2 - b1 > threshold1 && b0 - b3 > threshold1 && b2 - b3 > threshold1) pixelsInScreen[index] = 1; } }); // 测试亮度变化的屏幕区域 var testTexture = new Texture2D((int)webCamera.width, (int)webCamera.height); testTexture.wrapMode = TextureWrapMode.Clamp; testTexture.filterMode = FilterMode.Point; for (int i = 0; i < pixelsInScreen.Count(); i++) { var vec = webCamera.IndexToCoord(i); var c = 0.5f + 0.5f * pixelsInScreen[i]; testTexture.SetPixel(vec.x, vec.y, new Color(c, c, c, 1)); } testTexture.Apply(); testImage.gameObject.SetActive(true); testImage.texture = testTexture; // 处理得到屏幕的边缘像素点,拟合成四边形 var Edge = PixlesProcessToEdge(pixelsInScreen); //screenQuad = Quadrilateral.Fit(Edge, webCamera.Size); //Debug.Log(ScreenQuad); info.gameObject.SetActive(true); info.text = "Locate Complete!\r\n" + screenQuad; // 标记屏幕的四个角 Screen.gameObject.SetActive(true); for (int i = 0; i < 4; i++) { RectTransform t = (RectTransform)Screen.GetChild(i); t.anchoredPosition = screenQuad[i].UnityVector().pixelToLocalPosition_AnchorCenter(webCamera.Size, webCamera.rawImage.rectTransform.rect); } //if (ScreenQuad.IsInScreen(cameraSize)) // info.text = "Screen Locate succeed"; //else // info.text = "Screen Locate failed"; // 测试识别的屏幕区域 var testTexture2 = new Texture2D(webCamera.width,webCamera.height); testTexture2.wrapMode = TextureWrapMode.Clamp; testTexture2.filterMode = FilterMode.Point; for (int i = 0; i < pixelsInScreen.Count(); i++) { var vec = webCamera.IndexToCoord(i); var c = 0.5f + 0.5f * pixelsInScreen[i]; testTexture2.SetPixel(vec.x, vec.y, new Color(c, c, c, 1)); } testTexture2.Apply(); testImage2.gameObject.SetActive(true); testImage2.texture = testTexture2; } IEnumerator ScreenLocateCorutine2() { background.color = Color.black; background.gameObject.SetActive(true); yield return new WaitForSeconds(0.4f); //摄像机画面有延时 record = 1; yield return new WaitForSeconds(1.0f); record = 0; float[] brightness0 = (float[])brightnessRecord.Clone(); var mean0 = brightnessRecord.Sum() / brightnessRecord.Length; //Debug.Log(mean0); SavePngFloat("显示器黑色", brightness0); background.color = Color.white; yield return new WaitForSeconds(0.4f); // 测试当前画面 Color[] _piexl = webCamera.webCamTexture.GetPixels(); var testTexture = new Texture2D(webCamera.width, webCamera.height); testTexture.wrapMode = TextureWrapMode.Clamp; testTexture.filterMode = FilterMode.Point; for (int i = 0; i < _piexl.Count(); i++) { var vec = webCamera.IndexToCoord(i); testTexture.SetPixel(vec.x, vec.y, _piexl[i]); } testTexture.Apply(); int[] BrightnessValue = new int[brightness0.Length]; int? threshold = null; for (int i = 0; i < 20; i++) { record = 1; yield return new WaitForSeconds(0.3f); record = 0; // 求阈值 if (threshold == null) { var mean = brightnessRecord.Sum() / brightnessRecord.Length; //Debug.Log(mean); threshold = (int)Math.Ceiling(Math.Pow(mean - mean0, 2) * 4); threshold = (int)Math.Pow((double)threshold, 2); //Debug.Log("threshold: " + threshold); SavePngFloat("显示器白色", brightnessRecord); } Parallel.For(0, brightness0.Length, (j) => { if (BrightnessValue[j] >= 0) { var value = (brightnessRecord[j] - brightness0[j]) * 100; if (value > threshold) BrightnessValue[j] = 1; else { lock (BrightnessValue) BrightnessValue[j] = -1; } } }); } background.color = Color.black; //background.gameObject.SetActive(false); yield return null; // 处理得到屏幕的边缘像素点,拟合成四边形 var Edge = PixlesProcessToEdge(BrightnessValue); //screenQuad = Quadrilateral.Fit(Edge, webCamera.Size); //Debug.Log(ScreenQuad); info.gameObject.SetActive(true); info.text = $"Locate Complete!\r\nthreshold: {threshold}\r\n{screenQuad}"; // 标记屏幕的四个角 Screen.gameObject.SetActive(true); for (int i = 0; i < 4; i++) { RectTransform t = (RectTransform)Screen.GetChild(i); t.anchoredPosition =screenQuad[i].UnityVector().pixelToLocalPosition_AnchorCenter(webCamera.Size, webCamera.rawImage.rectTransform.rect); } //if (ScreenQuad.IsInScreen(cameraSize)) // info.text = "Screen Locate succeed"; //else // info.text = "Screen Locate failed"; // 测试识别的屏幕区域 testImage.gameObject.SetActive(true); testImage.texture = testTexture; var testTexture2 = new Texture2D(webCamera.width, webCamera.height); testTexture2.wrapMode = TextureWrapMode.Clamp; testTexture2.filterMode = FilterMode.Point; for (int i = 0; i < BrightnessValue.Count(); i++) { var vec = webCamera.IndexToCoord(i); var c = 0.5f + 0.5f * BrightnessValue[i]; testTexture2.SetPixel(vec.x, vec.y, new Color(c, c, c, 1)); } testTexture2.Apply(); testImage2.gameObject.SetActive(true); testImage2.texture = testTexture2; } IEnumerator ScreenTestCoruine() { background.color = Color.black; background.gameObject.SetActive(true); yield return new WaitForSeconds(0.4f); //摄像机画面有延时 record = 1; yield return new WaitForSeconds(3.0f); record = 0; float[] brightness0 = (float[])brightnessRecord.Clone(); SavePngFloat("显示器黑色", brightness0); background.color = Color.white; yield return new WaitForSeconds(0.4f); record = 1; yield return new WaitForSeconds(3.0f); record = 0; SavePngFloat("显示器白色", brightnessRecord); var sub = new float[brightnessRecord.Count()]; Parallel.For(0, brightnessRecord.Count(), (i) => { sub[i] = brightnessRecord[i] - brightness0[i]; }); SavePngFloat("亮度差值", sub); background.gameObject.SetActive(false); } HashSet PixlesProcessToEdge(int[] pixelsInScreen) { // 去噪 { int[] pixelsInScreen2 = new int[pixelsInScreen.Length]; int[] pixelsIn, pixelsOut; for (int n = 0; n < 6; n++) { if (n % 2 == 0) { pixelsIn = pixelsInScreen; pixelsOut = pixelsInScreen2; } else { pixelsIn = pixelsInScreen2; pixelsOut = pixelsInScreen; } Parallel.For(0, (int)(webCamera.width), (i) => { for (int j = 0; j < webCamera.height; j++) { var index = webCamera.CoordToIndex(i, j); pixelsOut[index] = 0; if (pixelsIn[index] == 1) { var conv = Conv2DAverageInt(pixelsIn, new Vector2Int(i, j), 7, 7 - n % 4); if (conv > 0.25f) { //Debug.Log("white"); pixelsOut[index] = 1; } } } }); } } // 边缘提取 HashSet Edge = new HashSet(); int[] pixelsLast = new int[pixelsInScreen.Length]; Parallel.For(0, (int)(webCamera.width), (i) => { for (int j = 0; j < webCamera.height; j++) { var index = webCamera.CoordToIndex(i, j); pixelsLast[index] = 0; if (pixelsInScreen[index] == 1) { if (EdgeCalculation(pixelsInScreen, new Vector2Int(i, j), 5)) { lock (Edge) Edge.Add(new Vector2(i, j)); pixelsLast[index] = 1; } } } }); return Edge; } //float Conv2DAverage(Color[] pixels, int index, int kernel_size, int scale) //{ // var vec = indexToVector2(index); // var sum = 0f; // for (int i = vec.x - kernel_size / 2 * scale; i < vec.x + kernel_size / 2 * scale; i += scale) // { // for (int j = vec.y - kernel_size / 2 * scale; j < vec.y + kernel_size / 2 * scale; j += scale) // { // var index2 = Vector2ToIndex(i, j); // if (index2 > 0 && index2 < pixels.Length) // 超出边缘不计算 // sum += getBrightness(pixels[index2]); // } // } // return sum / kernel_size / kernel_size; //} float Conv2DAverageInt(int[] pixels, Vector2Int vec, int kernel_size, int scale) { var sum = 0f; for (int i = vec.x - kernel_size / 2 * scale; i < vec.x + kernel_size / 2 * scale; i += scale) { for (int j = vec.y - kernel_size / 2 * scale; j < vec.y + kernel_size / 2 * scale; j += scale) { var index2 = webCamera.CoordToIndex(i, j); if (index2 > 0 && index2 < pixels.Length) // 超出边缘不计算 sum += pixels[index2]; } } return sum / kernel_size / kernel_size; } bool EdgeCalculation(int[] pixels, Vector2Int vec, int kernel_size) { var sum = 0f; for (int i = vec.x - kernel_size / 2; i < vec.x + kernel_size / 2; i++) { for (int j = vec.y - kernel_size / 2; j < vec.y + kernel_size / 2; j++) { var index2 = webCamera.CoordToIndex(i, j); if (i == vec.x && j == vec.y) sum += (kernel_size * kernel_size - 1) * pixels[index2]; else if (index2 > 0 && index2 < pixels.Length) // 超出边缘不计算 sum -= pixels[index2]; } } return sum > kernel_size * kernel_size / 2; } }