#define ENABLE_LOG using o0.Geometry2D.Float; using System; using System.Collections.Generic; using System.IO; using System.Threading.Tasks; using UnityEngine; using ZIM; using ZIM.Unity; namespace o0.Project { public partial class ScreenIdentification { private const string TAG = "ScreenIdentification#"; //static Rect[][] LocateAreaData = new Rect[][] { // new Rect[] { new Rect(0f, 0f, 0.3f, 0.3f), new Rect(0f, 0f, 0.4f, 0.4f), new Rect(0f, 0f, 0.5f, 0.5f), new Rect(0f, 0f, 0.6f, 0.6f) }, // new Rect[] { new Rect(0.7f, 0f, 0.3f, 0.3f), new Rect(0.6f, 0f, 0.4f, 0.4f), new Rect(0.5f, 0f, 0.5f, 0.5f), new Rect(0.4f, 0f, 0.6f, 0.6f) }, // new Rect[] { new Rect(0f, 0.7f, 0.3f, 0.3f), new Rect(0f, 0.6f, 0.4f, 0.4f), new Rect(0f, 0.5f, 0.5f, 0.5f), new Rect(0f, 0.4f, 0.6f, 0.6f) }, // new Rect[] { new Rect(0.7f, 0.7f, 0.3f, 0.3f), new Rect(0.6f, 0.6f, 0.4f, 0.4f), new Rect(0.5f, 0.5f, 0.5f, 0.5f), new Rect(0.4f, 0.4f, 0.6f, 0.6f) } //}; static Rect[][] LocateAreaData = new Rect[][] { new Rect[] { new Rect(0f, 0f, 0.3f, 0.3f), new Rect(0f, 0f, 0.4f, 0.4f), new Rect(0f, 0f, 0.5f, 0.5f) }, new Rect[] { new Rect(0.7f, 0f, 0.3f, 0.3f), new Rect(0.6f, 0f, 0.4f, 0.4f), new Rect(0.5f, 0f, 0.5f, 0.5f) }, new Rect[] { new Rect(0f, 0.7f, 0.3f, 0.3f), new Rect(0f, 0.6f, 0.4f, 0.4f), new Rect(0f, 0.5f, 0.5f, 0.5f) }, new Rect[] { new Rect(0.7f, 0.7f, 0.3f, 0.3f), new Rect(0.6f, 0.6f, 0.4f, 0.4f), new Rect(0.5f, 0.5f, 0.5f, 0.5f) } }; //static bool LocateDebug = false; static bool LocateDebug = true; public Geometry2D.Vector Size { get; private set; } public ScreenMap Screen; // 识别到的屏幕,用于执行透视变换 int capture = 0; int delay = 0; int maxCapture; int maxDelay; Geometry.Vector[] ScreenBlackTexture; Geometry.Vector[] ScreenWhiteTexture; int locateIndex = -1; List locateArea = new List { new Rect(0f, 0f, 0.5f, 0.5f), new Rect(0.5f, 0f, 0.5f, 0.5f), new Rect(0f, 0.5f, 0.5f, 0.5f), new Rect(0.5f, 0.5f, 0.5f, 0.5f) }; // 屏幕显示白色的区域大小 float areaPercent => locateArea[locateIndex].size.x; // 当前白色区域的占比 int areaSelected = -1; // 选择哪个区域,顺序与Quadrilateral对应 List sumTemp = new List(); List quadTemp = new List(); //public ScreenIdentification(WebCamTexture texture) //{ // Size = new Geometry2D.Vector(texture.width, texture.height); // Screen = new ScreenMap(); //} public ScreenIdentification(Texture texture) { Size = new Geometry2D.Vector(texture.width, texture.height); Screen = new ScreenMap(); } public ScreenIdentification(Geometry2D.Vector size) { Size = size; Screen = new ScreenMap(); } public void LocateScreenManual(OrdinalQuadrilateral quad) => Screen.Quad = quad; public void LocateScreen(int Capture = 30, int Delay = 30)//frame { if (ScreenLocate.Main.DebugScreenImage != null) // 测试图片 { DebugImage(ScreenLocate.Main.DebugScreenImage); return; } delay = Delay; capture = Capture; maxDelay = Delay; maxCapture = Capture; ScreenLocate.SetScreen(new Rect(0f, 0f, 1f, 1f), UnityEngine.Color.black); //ScreenLocate.SetScreen(new Rect(0f, 0f, 0.6f, 0.6f), UnityEngine.Color.white); } void DebugImage(Texture2D image) { QuadrilateralFit(out Texture2D LocateTex, out Texture2D DrawLineTex, image); ScreenLocate.DebugTexture(2, LocateTex); ScreenLocate.DebugTexture(3, DrawLineTex); //Debug.Log(quadTemp[0]); //var watch = new System.Diagnostics.Stopwatch(); //watch.Start(); //var times = new List() { 0.0 }; #if (!NDEBUG && DEBUG && ENABLE_LOG) Console.WriteLine($"{TAG} quadTemp.Count:{ quadTemp.Count}"); #endif if (quadTemp.Count > 0) { // 透视变换 var quad = quadTemp[0]; var srcWidth = LocateTex.width; var transformWidth = (int)((quad.B.x - quad.A.x + quad.D.x - quad.C.x) / 2); var transformHeight = (int)((quad.C.y - quad.A.y + quad.D.y - quad.B.y) / 2); var transformTex = new Texture2D(transformWidth, transformHeight); var pt = new ZIMPerspectiveTransform(new OrdinalQuadrilateral(new Vector(0, 0), new Vector(transformWidth, 0), new Vector(0, transformHeight), new Vector(transformWidth, transformHeight)), quad); var dstPixel = new UnityEngine.Color[transformWidth * transformHeight]; var srcPixel = LocateTex.GetPixels(); Parallel.For(0, transformWidth, (x) => { for (int y = 0; y < transformHeight; y++) { var index = y * transformWidth + x; var sampleCoord = pt.TransformRound(x, y); dstPixel[index] = srcPixel[sampleCoord.y * srcWidth + sampleCoord.x]; } }); transformTex.SetPixels(dstPixel); transformTex.Apply(); ScreenLocate.DebugTexture(1, transformTex); #if (!NDEBUG && DEBUG && ENABLE_LOG) Console.WriteLine($"{TAG} ScreenLocate.DebugTexture 1:{ transformTex.GetNativeTexturePtr()}"); #endif } //times.Add(watch.ElapsedMilliseconds); //UnityEngine.Debug.Log("time: " + (times[times.Count - 1] - times[times.Count - 2])); } public void NextScreen() { // 测试用 if (LocateDebug && areaSelected == -1) { LocateAreaData = new Rect[][] { new Rect[] { new Rect(0, 0, 1f, 1f) } }; locateIndex = 3; areaSelected = 0; locateArea.AddRange(LocateAreaData[0]); } // index从-1开始 locateIndex++; if (locateIndex < locateArea.Count) // 依次点亮屏幕区域 { ScreenLocate.SetScreen(locateArea[locateIndex], UnityEngine.Color.white); delay = maxDelay; capture = maxCapture; } else // 退出屏幕黑白控制 { ScreenLocate.SetScreen(null); ScreenLocate.Main.ShowScreen(Screen.Quad); Reset(); } } void Reset() { delay = 0; capture = 0; ScreenWhiteTexture = null; ScreenBlackTexture = null; locateIndex = -1; areaSelected = -1; locateArea.RemoveRange(4, LocateAreaData[0].Length); quadTemp.Clear(); sumTemp.Clear(); } public void CaptureBlack(Texture2D cam) { if (ScreenBlackTexture == null) ScreenBlackTexture = new Geometry.Vector[Size.x * Size.y]; var pixel = cam.GetPixels(); Parallel.For(0, Size.x * Size.y, i => { var ip = pixel[i]; ScreenBlackTexture[i] += new Geometry.Vector(ip.r / maxCapture, ip.g / maxCapture, ip.b / maxCapture); }); } public void CaptureWhite(Texture2D cam) { if (ScreenWhiteTexture == null) ScreenWhiteTexture = new Geometry.Vector[Size.x * Size.y]; var pixel = cam.GetPixels(); Parallel.For(0, Size.x * Size.y, i => { var ip = pixel[i]; ScreenWhiteTexture[i] += new Geometry.Vector(ip.r / maxCapture, ip.g / maxCapture, ip.b / maxCapture); }); } public void CaptureStay(Texture2D cam) { if (locateIndex == -1) // 屏幕黑色 { CaptureBlack(cam); } else // 屏幕部分为白色 { CaptureWhite(cam); } } public void CaptureEnd() { //Debug.Log("locateIndex: " + locateIndex + ", quad: " + quadTemp.Count); if (locateIndex == -1) return; if (locateIndex < 4) { sumTemp.Add(GetBrightness()); ScreenWhiteTexture = null; // 选择亮度差最大的区域 if (locateIndex == 3) { areaSelected = sumTemp.MaxIndex(); locateArea.AddRange(LocateAreaData[areaSelected]); } } else if (locateIndex >= 4 && locateIndex < locateArea.Count - 1) { QuadrilateralFit(out _,out _); ScreenWhiteTexture = null; } else { QuadrilateralFit(out Texture2D LocateTex, out Texture2D DrawLineTex); //ScreenLocate.DebugTexture(2, LocateTex); // ScreenLocate.DebugTexture(3, DrawLineTex); if (quadTemp.Count != LocateAreaData[0].Length) { Debug.Log($"拟合四边形失败, quadTemp.Count: {quadTemp.Count}"); } else if (quadTemp.Count == 1) { Screen.Quad = quadTemp[0]; } else { // 线性拟合 var xValue = new List() { 0 }; var predicts = new List(); foreach (var i in LocateAreaData[0]) xValue.Add(i.size.x); Vector baseVertex = Vector.Zero; // x==0 时的点 { foreach (var q in quadTemp) { baseVertex += q[areaSelected]; } baseVertex /= quadTemp.Count; } double rs = 0.0; for (int i = 0; i < 4; i++) { if (i == areaSelected) { predicts.Add(baseVertex); } else { var yValue = new List() { baseVertex }; foreach (var q in quadTemp) { yValue.Add(q[i]); } var lr = LinerRegression1D.Fit(2, xValue.ToArray(), yValue.ToArray()); rs += lr.RSquared / 3; predicts.Add(lr.Predict(1)); } } Screen.Quad = new OrdinalQuadrilateral(predicts); Debug.Log($"[InfraredIdentification2拟合结果] RSquared: {rs}, Quad: {Screen.Quad}"); if (rs < 0.8) Screen.Quad = null; } } } public bool Update(Texture2D cam) { if (delay != 0) { delay--; return true; } if (capture != 0) { CaptureStay(cam); capture--; if (capture == 0) { CaptureEnd(); NextScreen(); } return true; } return false; #region Old /* if (delay != 0) { delay--; return true; } if (capture != 0) { capture--; if (ScreenBlackTexture == null) ScreenBlackTexture = new Geometry.Vector[Size.x * Size.y]; var pixel = cam.GetPixels(); Parallel.For(0, Size.x * Size.y, i => { var ip = pixel[i]; ScreenBlackTexture[i] += new Geometry.Vector(ip.r, ip.g, ip.b); }); if (capture == 0) ScreenLocate.SetScreen(UnityEngine.Color.black); return true; } if (delay != 0) { delay--; return true; } if (capture != 0) { capture--; if (ScreenWhiteTexture == null) ScreenWhiteTexture = new Geometry.Vector[Size.x * Size.y]; var pixel = cam.GetPixels(); Parallel.For(0, Size.x * Size.y, i => { var ip = pixel[i]; ScreenWhiteTexture[i] += new Geometry.Vector(ip.r, ip.g, ip.b); }); if (capture == 0) ScreenLocate.SetScreen(UnityEngine.Color.black); return true; } if (delay != 0) { delay--; return true; } if (capture != 0) { capture--; var pixel = cam.GetPixels(); Parallel.For(0, Size.x * Size.y, i => { var ip = pixel[i]; ScreenWhiteTexture[i] -= new Geometry.Vector(ip.r, ip.g, ip.b); }); if (capture == 0) { ScreenLocate.SetScreen(null); UnityEngine.Color[] newPixel = new UnityEngine.Color[Size.x * Size.y]; Parallel.For(0, Size.x * Size.y, i => { var pi = ScreenWhiteTexture[i] /= capture; newPixel[i] = new UnityEngine.Color(pi.x, pi.y, pi.z); }); //读取数据 //{ // var fileName = "3.bin"; // ScreenLocateTexture = $"2023 04 16 厦门测试数据/{fileName}".FileReadByte[]>(); // Debug.Log($"Read {fileName}"); // Parallel.For(0, Size.x * Size.y, i => // { // var pi = ScreenLocateTexture[i]; // newPixel[i] = new UnityEngine.Color(pi.x, pi.y, pi.z); // }); //} var ScreenLocateTex = new Texture2D(Size.x, Size.y); ScreenLocateTex.SetPixels(newPixel); ScreenLocateTex.Apply(); //ScreenLocate.DebugTexture(2, ScreenLocateTex); var ScreenLocateTexLighted = ScreenLocateTex.AutoLight(10); //ScreenLocate.DebugTexture(2, ScreenLocateTexLighted); //var FileSavePath = Application.persistentDataPath + "/ScreenLocateTexture.bin"; bool Save = ScreenLocate.Main.SaveToggle.isOn; string time; if (Save) { time = DateTime.Now.ToString("yyyyMMdd_HHmmss"); var FileSavePath = $"屏幕定位数据{time}.bin"; FileSavePath.FileWriteByte(ScreenWhiteTexture); var bytes = ScreenLocateTexLighted.EncodeToPNG(); File.WriteAllBytes($"屏幕定位数据{time}.png", bytes); Debug.Log("ScreenLocateTexture Saved To: " + FileSavePath); } var ScreenLocateTexR = ScreenLocateTexLighted.ToRGB(ColorChannel.Red); var ScreenLocateTexG = ScreenLocateTexLighted.ToRGB(ColorChannel.Green); var ScreenLocateTexB = ScreenLocateTexLighted.ToRGB(ColorChannel.Blue); ScreenLocate.DebugTexture(2, ScreenLocateTexR); //ScreenLocate.DebugTexture(4, ScreenLocateTexG); //ScreenLocate.DebugTexture(5, ScreenLocateTexB); var watch = new System.Diagnostics.Stopwatch(); watch.Start(); var times = new List() { 0.0 }; var ScreenLocateTexLightedMat = ScreenLocateTexLighted.Too0Mat(); //var ScreenLocateTexLightedMat = texture.Too0Mat(); //var (edge, edgeDir) = ScreenLocateTexLightedMat.IdentifyEdge(); var (edge, edgeDir) = ScreenLocateTexLightedMat.zimIdentifyEdgeGradientAny(15); //ScreenLocate.DebugTexture(4, ScreenLocateTexLighted.Too0Mat().IdentifyEdgeGradient().ToTex()); //ScreenLocate.DebugTexture(4, edge.ToTex()); var quadLines = ScreenLocateTexLightedMat.IdentifyQuadLSD(edge, edgeDir, out List lightLines, 30); var drawLineMap = new MatrixF2D(edge..Size.x, edge.Size.y); int lineCount = 0; foreach (var l in quadLines) { if (l != null) { drawLineMap.DrawLine(l, (x, y) => 1, new Geometry2D.Float.Vector(0, 10)); lineCount++; } } if (lineCount == 4) { var a = quadLines[0].Intersect(quadLines[3], false).Value; var b = quadLines[0].Intersect(quadLines[1], false).Value; var c = quadLines[2].Intersect(quadLines[3], false).Value; var d = quadLines[1].Intersect(quadLines[2], false).Value; Quad = new Quadrilateral(a, b, c, d); if (!Quad.IsInScreen(ScreenLocate.Main.WebCamera.Size)) Quad = null; } ScreenLocate.Main.ShowScreen(Quad); //var lines = edge.IdentifyLineLSD(edgeDir, 100); ////var lines = ScreenLocateTexLightedMat.IdentifyLineLSD(); //var drawLineMap = new MatrixF2D(edge..Size.x, edge.Size.y); //var returnMaxLines = lines.Sub(0, 10); //foreach (var (line, sum, gradient) in returnMaxLines) // drawLineMap.DrawLine(line, (x, y) => 1, new Geometry2D.Float.Vector(0, 10)); ScreenLocate.DebugTexture(3, drawLineMap.ToTex()); //{ // var bytes = drawLineMap.ToTex().EncodeToPNG(); // File.WriteAllBytes($"屏幕定位数据DrawLineMap.png", bytes); //} times.Add(watch.ElapsedMilliseconds); UnityEngine.Debug.Log("time: " + (times[times.Count - 1] - times[times.Count - 2])); //ScreenLocate.DebugTexture(5, edge.IdentifyLine(edgeDir).ToTex()); //ScreenLocate.DebugTexture(4, ScreenLocateTexLighted.Too0Mat().IdentifyEdgeGradientX().ToTex()); //ScreenLocate.DebugTexture(5, ScreenLocateTexLighted.Too0Mat().IdentifyEdgeGradientY().ToTex()); //var convolutionLighted2 = ScreenLocateTexLighted.Too0Mat().IdentifyEdgeVariance().ToTex(); // opecncv处理 // zim { //var cvLines = edge.cvHoughLinesP(); //ScreenLocate.DebugTexture(5, cvLines); //var myLines = Hough.Transform(edgeMat); //var cvLines = edge.cvLine(myLines); //ScreenLocate.DebugTexture(5, cvLines); } UnityEngine.Object.Destroy(ScreenLocateTex); //ScreenLocate.DebugTexture(4, convolutionLighted2); } return true; } /* var avg = new Geometry4D.Vector(); var pixel = texture.GetPixels(); foreach(var i in pixel.Index()) { var iP = pixel[i]; avg += new Geometry4D.Vector(iP.r, iP.g, iP.b, iP.a); } avg /= pixel.Count(); /* var (texLightedR, texLightedG, texLightedB) = ToRGB(newTex); ScreenLocate.DebugTexture(3, texLightedR); ScreenLocate.DebugTexture(4, texLightedG); ScreenLocate.DebugTexture(5, texLightedB); //Debug.Log(avg); return false; /**/ #endregion } float GetBrightness() { UnityEngine.Color[] differPixel = new UnityEngine.Color[Size.x * Size.y]; Parallel.For(0, Size.x * Size.y, i => { var pi = ScreenWhiteTexture[i] - ScreenBlackTexture[i]; differPixel[i] = new UnityEngine.Color(pi.x, pi.y, pi.z); }); var sum = 0f; foreach (var i in differPixel) { sum += i.Brightness(); } sum /= differPixel.Length; //Debug.Log(sum); return sum; } void QuadrilateralFit(out Texture2D LocateTex, out Texture2D DrawLineTex, Texture2D debugImage = null) { OrdinalQuadrilateral quad = null; UnityEngine.Color[] differPixel = new UnityEngine.Color[Size.x * Size.y]; //读取数据 if (debugImage != null) { Debug.Log($"Debug {debugImage.name}"); differPixel = debugImage.GetPixels(); } else // 获得屏幕差值 { Parallel.For(0, Size.x * Size.y, i => { var pi = ScreenWhiteTexture[i] - ScreenBlackTexture[i]; differPixel[i] = new UnityEngine.Color(pi.x, pi.y, pi.z); }); } var ScreenLocateTex = new Texture2D(Size.x, Size.y); ScreenLocateTex.SetPixels(differPixel); ScreenLocateTex.Apply(); //ScreenLocate.DebugTexture(2, ScreenLocateTex); var ScreenLocateTexLighted = ScreenLocateTex.AutoLight(10); //ScreenLocate.DebugTexture(2, ScreenLocateTexLighted); var ScreenLocateTexR = ScreenLocateTexLighted.ToRGB(ColorChannel.Red); var ScreenLocateTexG = ScreenLocateTexLighted.ToRGB(ColorChannel.Green); var ScreenLocateTexB = ScreenLocateTexLighted.ToRGB(ColorChannel.Blue); LocateTex = ScreenLocateTexR; //ScreenLocate.DebugTexture(2, ScreenLocateTexR); //ScreenLocate.DebugTexture(4, ScreenLocateTexG); //ScreenLocate.DebugTexture(5, ScreenLocateTexB); //var watch = new System.Diagnostics.Stopwatch(); //watch.Start(); //var times = new List() { 0.0 }; var ScreenLocateTexLightedMat = ScreenLocateTexLighted.Too0Mat(); //var ScreenLocateTexLightedMat = texture.Too0Mat(); //var (edge, edgeDir) = ScreenLocateTexLightedMat.IdentifyEdge(); int conSize = 15; var (edge, edgeDir) = ScreenLocateTexLightedMat.zimIdentifyEdgeGradientAny(conSize); //ScreenLocate.DebugTexture(4, ScreenLocateTexLighted.Too0Mat().IdentifyEdgeGradient().ToTex()); //ScreenLocate.DebugTexture(4, edge.ToTex()); var minLength = locateIndex == -1 ? 50 : 50 * areaPercent; var quadLines = ScreenLocateTexLightedMat.ZIMIdentifyQuadLSD(edge, edgeDir, out List lightLines, conSize, minLength); var drawLineMap = new Matrix(edge.Size, Tiling: true); int lineCount = 0; foreach (var l in quadLines) { if (l != null) { drawLineMap.DrawLine(l, (x, y) => 1, new Geometry2D.Float.Vector(0, 10)); lineCount++; } } //foreach (var l in lightLines) //{ // if (l != null) // { // if (quadLines.Contains(l)) // { // drawLineMap.DrawLine(l, (x, y) => 1, new Geometry2D.Float.Vector(0, 10)); // lineCount++; // } // else // { // drawLineMap.DrawLine(l, (x, y) => 1, new Geometry2D.Float.Vector(0, 2)); // } // } //} if (lineCount == 4) { var a = quadLines[0].Intersect(quadLines[3], false).Value; var b = quadLines[0].Intersect(quadLines[1], false).Value; var c = quadLines[2].Intersect(quadLines[3], false).Value; var d = quadLines[1].Intersect(quadLines[2], false).Value; quad = new OrdinalQuadrilateral(a, b, c, d); if (!quad.IsInScreen(ScreenLocate.Main.mUVCCameraInfo.Size)) quad = null; } //if (quad != null && debugImage == null) if (quad != null) quadTemp.Add(quad); //var lines = edge.IdentifyLineLSD(edgeDir, 100); ////var lines = ScreenLocateTexLightedMat.IdentifyLineLSD(); //var drawLineMap = new MatrixF2D(edge..Size.x, edge.Size.y); //var returnMaxLines = lines.Sub(0, 10); //foreach (var (line, sum, gradient) in returnMaxLines) // drawLineMap.DrawLine(line, (x, y) => 1, new Geometry2D.Float.Vector(0, 10)); DrawLineTex = drawLineMap.ToTex(); //ScreenLocate.DebugTexture(3, drawLineMap.ToTex()); //var FileSavePath = Application.persistentDataPath + "/ScreenLocateTexture.bin"; bool Save = ScreenLocate.Main.SaveToggle.isOn; string time; if (Save) { time = DateTime.Now.ToString("yyyyMMdd_HHmmss"); var FileSavePath = $"{time}屏幕定位数据.bin"; FileSavePath.FileWriteByte(ScreenWhiteTexture); var bytes = ScreenLocateTexLighted.EncodeToPNG(); File.WriteAllBytes($"{time}屏幕.png", bytes); bytes = DrawLineTex.EncodeToPNG(); File.WriteAllBytes($"{time}屏幕边框识别.png", bytes); Debug.Log("ScreenLocateTexture Saved To: " + FileSavePath); } //times.Add(watch.ElapsedMilliseconds); //UnityEngine.Debug.Log("time: " + (times[times.Count - 1] - times[times.Count - 2])); //ScreenLocate.DebugTexture(5, edge.IdentifyLine(edgeDir).ToTex()); //ScreenLocate.DebugTexture(4, ScreenLocateTexLighted.Too0Mat().IdentifyEdgeGradientX().ToTex()); //ScreenLocate.DebugTexture(5, ScreenLocateTexLighted.Too0Mat().IdentifyEdgeGradientY().ToTex()); //var convolutionLighted2 = ScreenLocateTexLighted.Too0Mat().IdentifyEdgeVariance().ToTex(); // opecncv处理 // zim { //var cvLines = edge.cvHoughLinesP(); //ScreenLocate.DebugTexture(5, cvLines); //var myLines = Hough.Transform(edgeMat); //var cvLines = edge.cvLine(myLines); //ScreenLocate.DebugTexture(5, cvLines); } UnityEngine.Object.Destroy(ScreenLocateTex); } } }