#define ENABLE_LOG using o0.Geometry2D.Float; using o0.Num; using o0InfraredLocate; using o0InfraredLocate.ZIM; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading.Tasks; using UnityEngine; using UnityEngine.UIElements; using UnityStandardAssets.Utility; using ZIM; using ZIM.Unity; namespace o0.Project { public partial class ScreenIdentification { private const string TAG = "ScreenIdentification#"; // LocateAreaData表示每次屏幕的色差变化的区域,可能有多次。通过设置LocateSingleStep可调整为仅识别一次色差 static Rect[][] LocateAreaData = new Rect[][] { new Rect[] { new Rect(0f, 0f, 0.3f, 0.3f), new Rect(0f, 0f, 0.4f, 0.4f), new Rect(0f, 0f, 0.5f, 0.5f) }, new Rect[] { new Rect(0.7f, 0f, 0.3f, 0.3f), new Rect(0.6f, 0f, 0.4f, 0.4f), new Rect(0.5f, 0f, 0.5f, 0.5f) }, new Rect[] { new Rect(0f, 0.7f, 0.3f, 0.3f), new Rect(0f, 0.6f, 0.4f, 0.4f), new Rect(0f, 0.5f, 0.5f, 0.5f) }, new Rect[] { new Rect(0.7f, 0.7f, 0.3f, 0.3f), new Rect(0.6f, 0.6f, 0.4f, 0.4f), new Rect(0.5f, 0.5f, 0.5f, 0.5f) } }; //static Rect[][] LocateAreaData = new Rect[][] { // new Rect[] { new Rect(0f, 0f, 0.3f, 0.3f), new Rect(0f, 0f, 0.4f, 0.4f), new Rect(0f, 0f, 0.5f, 0.5f), new Rect(0f, 0f, 0.6f, 0.6f) }, // new Rect[] { new Rect(0.7f, 0f, 0.3f, 0.3f), new Rect(0.6f, 0f, 0.4f, 0.4f), new Rect(0.5f, 0f, 0.5f, 0.5f), new Rect(0.4f, 0f, 0.6f, 0.6f) }, // new Rect[] { new Rect(0f, 0.7f, 0.3f, 0.3f), new Rect(0f, 0.6f, 0.4f, 0.4f), new Rect(0f, 0.5f, 0.5f, 0.5f), new Rect(0f, 0.4f, 0.6f, 0.6f) }, // new Rect[] { new Rect(0.7f, 0.7f, 0.3f, 0.3f), new Rect(0.6f, 0.6f, 0.4f, 0.4f), new Rect(0.5f, 0.5f, 0.5f, 0.5f), new Rect(0.4f, 0.4f, 0.6f, 0.6f) } //}; //static bool LocateSingleStep = false; static bool LocateSingleStep = true; public Vector Size => ScreenLocate.Main.CameraSize; public Geometry.Vector o0Size => new Geometry.Vector((int)Size.x, (int)Size.y); public QuadrilateralInCamera QuadManual; public QuadrilateralInCamera QuadAuto; // 全自动,可以给用户选择(赋值给Screen.QuadInCamera即生效) public QuadrilateralInCamera QuadSemiAuto; // 半自动,可以给用户选择(赋值给Screen.QuadInCamera即生效) public ScreenMap Screen; // 识别到的屏幕,用于执行透视变换 int capture = 0; int delay = 0; int maxCapture; int maxDelay; Geometry.Vector[] ScreenBlackTexture; Geometry.Vector[] ScreenWhiteTexture; int locateIndex = -1; readonly List locateArea = new List { new Rect(0f, 0f, 0.5f, 0.5f), new Rect(0.5f, 0f, 0.5f, 0.5f), new Rect(0f, 0.5f, 0.5f, 0.5f), new Rect(0.5f, 0.5f, 0.5f, 0.5f) }; // 屏幕显示白色的区域大小 float areaPercent => locateArea[locateIndex].size.x; // 当前白色区域的占比 int areaSelected = -1; // 选择哪个区域,顺序与Quadrilateral对应 readonly List sumTemp = new List(); readonly List quadTemp = new List(); //public ScreenIdentification(WebCamTexture texture) //{ // Size = new Geometry2D.Vector(texture.width, texture.height); // Screen = new ScreenMap(); //} public static UnityEngine.Color FloatValueToColor(float i) { return i switch { 1 => UnityEngine.Color.yellow, 2 => new UnityEngine.Color(0,1,1,1), 3 => UnityEngine.Color.green, 4 => UnityEngine.Color.white, 5 => UnityEngine.Color.red, _ => UnityEngine.Color.black, }; } public ScreenIdentification(o0InfraredCameraHandler cameraHandler) { Screen = new ScreenMap(cameraHandler); //OnLocateScreenEnter += () => Application.targetFrameRate = 30; // 固定识别的帧率,确保摄像机拍到正确的画面 //OnLocateScreenEnd += () => Application.targetFrameRate = 60; } public void SetScreenQuad(QuadrilateralInCamera quad) => Screen.QuadInCamera = quad; // 上一次半自动识别的情况,false代表这条边识别失败,线段顺序: 下、右、上、左 public bool[] LastQuadSemiAutoState; public event Action OnLocateScreenEnter; public event Action OnLocateScreenEnd; public bool bStartLocateScreen { get; set; } = false;//是否进行捕获 public bool SelectScreenAfterLocate(ScreenLocate.ScreenIdentificationTag tag) { QuadrilateralInCamera target = tag switch { ScreenLocate.ScreenIdentificationTag.Manual => QuadManual, ScreenLocate.ScreenIdentificationTag.SemiAuto => QuadSemiAuto, ScreenLocate.ScreenIdentificationTag.Auto => QuadAuto, _ => null }; if (target == null) return false; Debug.Log($"[ScreenIdentification] 选择已识别到的屏幕({Enum.GetName(typeof(ScreenLocate.ScreenIdentificationTag), tag)}), {target}"); SetScreenQuad(target); return true; } // 自动识别开始的入口 public void LocateScreen(int Capture = 30, int Delay = 30) //数值单位是frame { if (ScreenLocate.Main.DebugScreenImages.Count != 0 && ScreenLocate.Main.DebugOnZIMDemo) // 这段仅用于测试图片 { ScreenLocate.Main.SetCameraSize(new Vector(ScreenLocate.Main.DebugScreenImages[0].width, ScreenLocate.Main.DebugScreenImages[0].height)); DebugImage(ScreenLocate.Main.DebugScreenImages); Screen.QuadInCamera = quadTemp[0]; ScreenLocate.SetScreen(null); ScreenLocate.Main.ShowScreen(ScreenLocate.Main.ScreenQuad, Screen.QuadInCamera); delay = 0; capture = 0; ScreenWhiteTexture = null; ScreenBlackTexture = null; locateIndex = -1; areaSelected = -1; quadTemp.Clear(); sumTemp.Clear(); //ScreenLocate.Main.DebugScreenImages.Clear(); return; } delay = Math.Max(Delay, 5); capture = Math.Max(Capture, 5); maxDelay = Delay; maxCapture = Capture; ScreenLocate.SetScreen(new Rect(0f, 0f, 1f, 1f), UnityEngine.Color.black); //ScreenLocate.SetScreen(new Rect(0f, 0f, 0.6f, 0.6f), UnityEngine.Color.white); //bStartLocateScreen = false; ScreenWhiteTexture = null; ScreenBlackTexture = null; OnLocateScreenEnter?.Invoke(); } /// /// 开始进行捕获 /// 初始化了两个数据 capture 和 delay /// /// public bool isInitLocateScreen() { return capture != 0 && delay != 0; } void DebugImage(List images) { QuadrilateralFit(images); //var watch = new System.Diagnostics.Stopwatch(); //watch.Start(); //var times = new List() { 0.0 }; #if (!NDEBUG && DEBUG && ENABLE_LOG) Console.WriteLine($"{TAG} quadTemp.Count:{quadTemp.Count}"); #endif if (quadTemp.Count > 0) { ScreenLocate.Main.ShowScreen(ScreenLocate.Main.outputRawImages[4].transform.GetChild(0) as RectTransform, quadTemp[0]); // 透视变换 // var srcWidth = LocateLightedRedTex.width; // var transformWidth = (int)((quad.B.x - quad.A.x + quad.D.x - quad.C.x) / 2); // var transformHeight = (int)((quad.C.y - quad.A.y + quad.D.y - quad.B.y) / 2); // var transformTex = new Texture2D(transformWidth, transformHeight); // var pt = new ZIMPerspectiveTransform(new OrdinalQuadrilateral(new Vector(0, 0), new Vector(transformWidth, 0), new Vector(0, transformHeight), new Vector(transformWidth, transformHeight)), quad); // var dstPixel = new UnityEngine.Color[transformWidth * transformHeight]; // var srcPixel = LocateLightedRedTex.GetPixels(); // Parallel.For(0, transformWidth, (x) => // { // for (int y = 0; y < transformHeight; y++) // { // var index = y * transformWidth + x; // var sampleCoord = pt.TransformRound(x, y); // dstPixel[index] = srcPixel[sampleCoord.y * srcWidth + sampleCoord.x]; // } // }); // transformTex.SetPixels(dstPixel); // transformTex.Apply(); // //ScreenLocate.DebugTexture(1, transformTex); //#if (!NDEBUG && DEBUG && ENABLE_LOG) // Console.WriteLine($"{TAG} ScreenLocate.DebugTexture 1:{transformTex.GetNativeTexturePtr()}"); //#endif } //times.Add(watch.ElapsedMilliseconds); //Debug.Log("time: " + (times[times.Count - 1] - times[times.Count - 2])); } public void NextScreen() { // 只识别一次色差变化 if (LocateSingleStep && areaSelected == -1) { LocateAreaData = new Rect[][] { new Rect[] { new Rect(0, 0, 1f, 1f) } }; locateIndex = 3; areaSelected = 0; locateArea.AddRange(LocateAreaData[0]); } // index从-1开始 locateIndex++; if (locateIndex < locateArea.Count) // 依次点亮屏幕区域 { ScreenLocate.SetScreen(locateArea[locateIndex], UnityEngine.Color.white); delay = maxDelay; capture = maxCapture; } else // 退出屏幕黑白控制 { ScreenLocate.SetScreen(null); ScreenLocate.Main.ShowScreen(ScreenLocate.Main.ScreenQuad, Screen.QuadInCamera); Reset(); } } // 清除记录的屏幕识别数据(手动、自动等) public void ClearQuadCache() { SetScreenQuad(null); QuadManual = null; QuadSemiAuto = null; QuadAuto = null; } public void Reset() { // bStartLocateScreen = false; delay = 0; capture = 0; ScreenWhiteTexture = null; ScreenBlackTexture = null; locateIndex = -1; areaSelected = -1; if (locateArea.Count > 4) locateArea.RemoveRange(4, LocateAreaData[0].Length); quadTemp.Clear(); sumTemp.Clear(); } public void CaptureBlack(Texture2D cam) { if (ScreenBlackTexture == null) ScreenBlackTexture = new Geometry.Vector[(int)(Size.x * Size.y)]; var pixel = cam.GetPixels(); Parallel.For(0, (int)(Size.x * Size.y), i => { var ip = pixel[i]; ScreenBlackTexture[i] += new Geometry.Vector(ip.r / maxCapture, ip.g / maxCapture, ip.b / maxCapture); }); } public void CaptureWhite(Texture2D cam) { if (ScreenWhiteTexture == null) ScreenWhiteTexture = new Geometry.Vector[(int)(Size.x * Size.y)]; var pixel = cam.GetPixels(); Parallel.For(0, (int)(Size.x * Size.y), i => { var ip = pixel[i]; ScreenWhiteTexture[i] += new Geometry.Vector(ip.r / maxCapture, ip.g / maxCapture, ip.b / maxCapture); }); } public void CaptureStay(Texture2D cam) { if (locateIndex == -1) // 屏幕黑色 { CaptureBlack(cam); } else // 屏幕部分为白色 { CaptureWhite(cam); } } public void CaptureEnd() { //Debug.Log("locateIndex: " + locateIndex + ", quad: " + quadTemp.Count); if (locateIndex == -1) return; if (locateIndex < 4) { sumTemp.Add(GetBrightness()); ScreenWhiteTexture = null; // 选择亮度差最大的区域 if (locateIndex == 3) { areaSelected = sumTemp.MaxIndex(); locateArea.AddRange(LocateAreaData[areaSelected]); } } else if (locateIndex >= 4 && locateIndex < locateArea.Count - 1) { QuadrilateralFit(); ScreenWhiteTexture = null; } else { QuadrilateralFit(); if (quadTemp.Count != LocateAreaData[0].Length) { Debug.Log($"[ScreenIdentification] 拟合四边形失败, quadTemp.Count: {quadTemp.Count}"); } else if (quadTemp.Count == 1) { SetScreenQuad(quadTemp[0]); Debug.Log($"[ScreenIdentification] 拟合成功,识别数据: {Screen.QuadInCamera}"); } else { // Debug.Log($"拟合四边形 2 , quadTemp.Count: {quadTemp.Count}"); // 线性拟合 var xValue = new List() { 0 }; var predicts = new List(); foreach (var i in LocateAreaData[0]) xValue.Add(i.size.x); Vector baseVertex = Vector.Zero; // x==0 时的点 { foreach (var q in quadTemp) { baseVertex += q.Quad[areaSelected]; } baseVertex /= quadTemp.Count; } double rs = 0.0; for (int i = 0; i < 4; i++) { if (i == areaSelected) { predicts.Add(baseVertex); } else { var yValue = new List() { baseVertex }; foreach (var q in quadTemp) { yValue.Add(q.Quad[i]); } var lr = LinerRegression1D.Fit(2, xValue.ToArray(), yValue.ToArray()); rs += lr.RSquared / 3; predicts.Add(lr.Predict(1)); } } SetScreenQuad(new QuadrilateralInCamera(predicts, new Vector(Size.x, Size.y))); Debug.Log($"[ScreenIdentification] 拟合成功,RSquared: {rs}, Quad: {Screen.QuadInCamera}"); //if (rs < 0.8) Screen.Quad = null; } OnLocateScreenEnd?.Invoke(); } } public bool Update(Texture2D cam) { //if (!bStartLocateScreen) return false; if (delay != 0) { //ScreenLocate.Main.CreateUVCTexture2DFocusSizeIfNeeded(1280, 720); delay--; if (delay == 0) { ScreenLocate.Main.SetCameraSize(new Vector(cam.width, cam.height)); // 记录当前的分辨率 Debug.Log("[ScreenIdentification] 采样纹理,记录采样分辨率: [" + Size.x + ", " + Size.y + "]"); } return true; } if (capture != 0) { //ScreenLocate.Main.CreateUVCTexture2DFocusSizeIfNeeded(1280, 720); CaptureStay(cam); capture--; if (capture == 0) { CaptureEnd(); NextScreen(); } return true; } return false; #region Old /* if (delay != 0) { delay--; return true; } if (capture != 0) { capture--; if (ScreenBlackTexture == null) ScreenBlackTexture = new Geometry.Vector[Size.x * Size.y]; var pixel = cam.GetPixels(); Parallel.For(0, Size.x * Size.y, i => { var ip = pixel[i]; ScreenBlackTexture[i] += new Geometry.Vector(ip.r, ip.g, ip.b); }); if (capture == 0) ScreenLocate.SetScreen(UnityEngine.Color.black); return true; } if (delay != 0) { delay--; return true; } if (capture != 0) { capture--; if (ScreenWhiteTexture == null) ScreenWhiteTexture = new Geometry.Vector[Size.x * Size.y]; var pixel = cam.GetPixels(); Parallel.For(0, Size.x * Size.y, i => { var ip = pixel[i]; ScreenWhiteTexture[i] += new Geometry.Vector(ip.r, ip.g, ip.b); }); if (capture == 0) ScreenLocate.SetScreen(UnityEngine.Color.black); return true; } if (delay != 0) { delay--; return true; } if (capture != 0) { capture--; var pixel = cam.GetPixels(); Parallel.For(0, Size.x * Size.y, i => { var ip = pixel[i]; ScreenWhiteTexture[i] -= new Geometry.Vector(ip.r, ip.g, ip.b); }); if (capture == 0) { ScreenLocate.SetScreen(null); UnityEngine.Color[] newPixel = new UnityEngine.Color[Size.x * Size.y]; Parallel.For(0, Size.x * Size.y, i => { var pi = ScreenWhiteTexture[i] /= capture; newPixel[i] = new UnityEngine.Color(pi.x, pi.y, pi.z); }); //读取数据 //{ // var fileName = "3.bin"; // ScreenLocateTexture = $"2023 04 16 厦门测试数据/{fileName}".FileReadByte[]>(); // Debug.Log($"Read {fileName}"); // Parallel.For(0, Size.x * Size.y, i => // { // var pi = ScreenLocateTexture[i]; // newPixel[i] = new UnityEngine.Color(pi.x, pi.y, pi.z); // }); //} var ScreenLocateTex = new Texture2D(Size.x, Size.y); ScreenLocateTex.SetPixels(newPixel); ScreenLocateTex.Apply(); //ScreenLocate.DebugTexture(2, ScreenLocateTex); var ScreenLocateTexLighted = ScreenLocateTex.AutoLight(10); //ScreenLocate.DebugTexture(2, ScreenLocateTexLighted); //var FileSavePath = Application.persistentDataPath + "/ScreenLocateTexture.bin"; bool Save = ScreenLocate.Main.SaveToggle.isOn; string time; if (Save) { time = DateTime.Now.ToString("yyyyMMdd_HHmmss"); var FileSavePath = $"屏幕定位数据{time}.bin"; FileSavePath.FileWriteByte(ScreenWhiteTexture); var bytes = ScreenLocateTexLighted.EncodeToPNG(); File.WriteAllBytes($"屏幕定位数据{time}.png", bytes); Debug.Log("ScreenLocateTexture Saved To: " + FileSavePath); } var ScreenLocateTexR = ScreenLocateTexLighted.ToRGB(ColorChannel.Red); var ScreenLocateTexG = ScreenLocateTexLighted.ToRGB(ColorChannel.Green); var ScreenLocateTexB = ScreenLocateTexLighted.ToRGB(ColorChannel.Blue); ScreenLocate.DebugTexture(2, ScreenLocateTexR); //ScreenLocate.DebugTexture(4, ScreenLocateTexG); //ScreenLocate.DebugTexture(5, ScreenLocateTexB); var watch = new System.Diagnostics.Stopwatch(); watch.Start(); var times = new List() { 0.0 }; var ScreenLocateTexLightedMat = ScreenLocateTexLighted.Too0Mat(); //var ScreenLocateTexLightedMat = texture.Too0Mat(); //var (edge, edgeDir) = ScreenLocateTexLightedMat.IdentifyEdge(); var (edge, edgeDir) = ScreenLocateTexLightedMat.zimIdentifyEdgeGradientAny(15); //ScreenLocate.DebugTexture(4, ScreenLocateTexLighted.Too0Mat().IdentifyEdgeGradient().ToTex()); //ScreenLocate.DebugTexture(4, edge.ToTex()); var quadLines = ScreenLocateTexLightedMat.IdentifyQuadLSD(edge, edgeDir, out List lightLines, 30); var drawLineMap = new MatrixF2D(edge..Size.x, edge.Size.y); int lineCount = 0; foreach (var l in quadLines) { if (l != null) { o0Extension.DrawLine(drawLineMap.DrawLine(l, (x, y) => 1, new Vector(0, 10)); lineCount++; } } if (lineCount == 4) { var a = quadLines[0].Intersect(quadLines[3], false).Value; var b = quadLines[0].Intersect(quadLines[1], false).Value; var c = quadLines[2].Intersect(quadLines[3], false).Value; var d = quadLines[1].Intersect(quadLines[2], false).Value; Quad = new Quadrilateral(a, b, c, d); if (!Quad.IsInScreen(ScreenLocate.Main.WebCamera.Size)) Quad = null; } ScreenLocate.Main.ShowScreen(Quad); //var lines = edge.IdentifyLineLSD(edgeDir, 100); ////var lines = ScreenLocateTexLightedMat.IdentifyLineLSD(); //var drawLineMap = new MatrixF2D(edge..Size.x, edge.Size.y); //var returnMaxLines = lines.Sub(0, 10); //foreach (var (line, sum, gradient) in returnMaxLines) // o0Extension.DrawLine(drawLineMap.DrawLine(line, (x, y) => 1, new Vector(0, 10)); ScreenLocate.DebugTexture(3, drawLineMap.ToTex()); //{ // var bytes = drawLineMap.ToTex().EncodeToPNG(); // File.WriteAllBytes($"屏幕定位数据DrawLineMap.png", bytes); //} times.Add(watch.ElapsedMilliseconds); Debug.Log("time: " + (times[times.Count - 1] - times[times.Count - 2])); //ScreenLocate.DebugTexture(5, edge.IdentifyLine(edgeDir).ToTex()); //ScreenLocate.DebugTexture(4, ScreenLocateTexLighted.Too0Mat().IdentifyEdgeGradientX().ToTex()); //ScreenLocate.DebugTexture(5, ScreenLocateTexLighted.Too0Mat().IdentifyEdgeGradientY().ToTex()); //var convolutionLighted2 = ScreenLocateTexLighted.Too0Mat().IdentifyEdgeVariance().ToTex(); // opecncv处理 // zim { //var cvLines = edge.cvHoughLinesP(); //ScreenLocate.DebugTexture(5, cvLines); //var myLines = Hough.Transform(edgeMat); //var cvLines = edge.cvLine(myLines); //ScreenLocate.DebugTexture(5, cvLines); } UnityEngine.Object.Destroy(ScreenLocateTex); //ScreenLocate.DebugTexture(4, convolutionLighted2); } return true; } /* var avg = new Geometry4D.Vector(); var pixel = texture.GetPixels(); foreach(var i in pixel.Index()) { var iP = pixel[i]; avg += new Geometry4D.Vector(iP.r, iP.g, iP.b, iP.a); } avg /= pixel.Count(); /* var (texLightedR, texLightedG, texLightedB) = ToRGB(newTex); ScreenLocate.DebugTexture(3, texLightedR); ScreenLocate.DebugTexture(4, texLightedG); ScreenLocate.DebugTexture(5, texLightedB); //Debug.Log(avg); return false; /**/ #endregion } float GetBrightness() { UnityEngine.Color[] differPixel = new UnityEngine.Color[(int)(Size.x * Size.y)]; Parallel.For(0, (int)(Size.x * Size.y), i => { var pi = ScreenWhiteTexture[i] - ScreenBlackTexture[i]; differPixel[i] = new UnityEngine.Color(pi.x, pi.y, pi.z); }); var sum = 0f; foreach (var i in differPixel) { sum += i.Brightness(); } sum /= differPixel.Length; //Debug.Log(sum); return sum; } // 转换成屏幕定位所需的纹理图像 Texture2D ToLocateTex(UnityEngine.Color[] pixels) { var ScreenLocateTex = new Texture2D(o0Size.x, o0Size.y); ScreenLocateTex.SetPixels(pixels); ScreenLocateTex.Apply(); //ScreenLocate.DebugTexture(2, ScreenLocateTex); return ScreenLocateTex.AutoLight(10); //ScreenLocate.DebugTexture(2, ScreenLocateTexLighted); //var ScreenLocateTexR = ToLocateTex.ToRGB(ColorChannel.Red); //var ScreenLocateTexG = ToLocateTex.ToRGB(ColorChannel.Green); //var ScreenLocateTexB = ToLocateTex.ToRGB(ColorChannel.Blue); //LocateLightedRedTex = ScreenLocateTexR; //ScreenLocate.DebugTexture(2, ScreenLocateTexR); //ScreenLocate.DebugTexture(4, ScreenLocateTexG); //ScreenLocate.DebugTexture(5, ScreenLocateTexB); //var ScreenLocateTexLightedMat = texture.Too0Mat(); } /// 识别的最小线段长度 /// 这个参数如果不为null且数量大于0,则执行debug操作 void QuadrilateralFit(List debugImages = null) { // 屏幕黑白差值,存放多批次的图像用于识别, 该List数量不能等于 0 List PixelsMultipleBatches = new List(); //var sw = new System.Diagnostics.Stopwatch(); //sw.Start(); //读取数据 int batchCount; if (debugImages != null && debugImages.Count != 0) { batchCount = debugImages.Count; var dSize = debugImages.First().Size(); foreach (var i in debugImages) { Debug.Log($"Debug {i.name}"); if (i.Size() != dSize) throw new InvalidOperationException("Multiple Debug textures have different sizes"); //PixelsMultipleBatches.Add(i.GetPixels()); } } else // 获得屏幕差值 { var differPixel = new UnityEngine.Color[(int)(Size.x * Size.y)]; var whitePixel = new UnityEngine.Color[(int)(Size.x * Size.y)]; Parallel.For(0, (int)Size.x, x => { for (int y = 0; y < (int)Size.y; y++) { var i = y * (int)Size.x + x; var d = ScreenWhiteTexture[i] - ScreenBlackTexture[i]; differPixel[i] = new UnityEngine.Color(d.x, d.y, d.z); whitePixel[i] = new UnityEngine.Color(ScreenWhiteTexture[i].x, ScreenWhiteTexture[i].y, ScreenWhiteTexture[i].z); } }); PixelsMultipleBatches.Add(differPixel); // 色差图 PixelsMultipleBatches.Add(whitePixel); // 原图 batchCount = PixelsMultipleBatches.Count; } int conSize = (int)Math.Ceiling(0.007f * Size.y) * 2 + 1; conSize = Math.Max(conSize, 7); // 设置最小为7 float minLength = conSize * 7.7f; minLength = locateIndex == -1 ? minLength : minLength * areaPercent; // minLength需要按areaPercent比例缩小 string log = $"[Log][ScreenLocate Auto] Size: ({Size.x},{Size.y}), 卷积核Size: {conSize}, 最小线段长度: {minLength}"; var allLines = new List(); List LocateTexTemp = new List(); List ScreenLocateMatList = new List(); for (int batch = 0; batch < batchCount; batch++) { Texture2D locateTex; if (debugImages != null && debugImages.Count != 0) locateTex = debugImages[batch]; else locateTex = ToLocateTex(PixelsMultipleBatches[batch]); LocateTexTemp.Add(locateTex); var ScreenLocateMat = locateTex.Too0Mat(); // 用于获取Lines的Matrix var lineCount = ZIMIdentifyQuadLSD( ref allLines, batch, ScreenLocateMat.zimIdentifyEdgeGradientAny(conSize), minLength, new Vector(minLength * 0.4f, conSize * 1.6f)); log += $"\r\n识别图片{batch}, 识别到的线段数量为: {lineCount}"; ScreenLocateMatList.Add(ScreenLocateMat); } Texture2D ScreenLocateTexture = LocateTexTemp[0]; // 色差图,for output // LSD计算得到的矩阵尺寸较小(因为卷积),这里必须进行位移 // 新增:根据阈值筛去梯度太低的线段 float minGradient = 0.08f; var offset = new Vector((conSize - 1) / 2, (conSize - 1) / 2); var tempList = new List(); for (int i = 0; i < allLines.Count; i++) { var l = allLines[i]; if (l.Gradient > minGradient * l.Line.Length) { l.Offset(offset); tempList.Add(l); } } allLines = tempList; log += $"\r\n根据梯度阈值筛选,最终线段数量为: {allLines.Count}"; // 如果有手动数据,刷新一下Size QuadManual?.ReSize(new Vector(Size.x, Size.y), ScreenMap.ViewAspectRatioSetting); // 估算屏幕中点,如果已有手动定位数据,根据现有数据取平均即可,否则从色差计算,ScreenLocateMatList[0]默认是屏幕的黑白色差 Vector AvgPoint = QuadManual != null ? QuadManual.Quad.Centroid : GetAvgPoint(ScreenLocateMatList[0]); // 过滤得到四边形的四条边, var (quadLinesSemiAuto, quadLinesAuto) = FilterLines( ScreenLocateMatList, allLines, AvgPoint, out LineIdentified[] manualLines, out List possibleLines, conSize, minLength); #region 全自动识别的结果 List LineIdentifiedAuto = new List(); // 线段顺序: 下、右、上、左 for (int i = 0; i < 4; i++) { if (quadLinesAuto[i] != null) LineIdentifiedAuto.Add(quadLinesAuto[i]); } if (LineIdentifiedAuto.Count == 4) // 判断识别的线段能否拼成屏幕,能拼成则记录 { var a = LineIdentifiedAuto[0].Line.Intersect(LineIdentifiedAuto[3].Line, false).Value; var b = LineIdentifiedAuto[0].Line.Intersect(LineIdentifiedAuto[1].Line, false).Value; var c = LineIdentifiedAuto[2].Line.Intersect(LineIdentifiedAuto[3].Line, false).Value; var d = LineIdentifiedAuto[1].Line.Intersect(LineIdentifiedAuto[2].Line, false).Value; QuadAuto = new QuadrilateralInCamera(a, b, c, d, new Vector(Size.x, Size.y)); if (!QuadAuto.IsQuadComplete()) QuadAuto = null; } #endregion #region 半自动识别 List LineIdentifiedSemiAuto = new List(); // 线段顺序: 下、右、上、左 LastQuadSemiAutoState = new bool[4] { true, true, true, true }; for (int i = 0; i < 4; i++) { if (quadLinesSemiAuto[i] != null) LineIdentifiedSemiAuto.Add(quadLinesSemiAuto[i]); else if (manualLines != null) { LineIdentifiedSemiAuto.Add(manualLines[i]); LastQuadSemiAutoState[i] = false; } } if (LineIdentifiedSemiAuto.Count == 4) // 判断识别的线段能否拼成屏幕,能拼成则记录 { var a = LineIdentifiedSemiAuto[0].Line.Intersect(LineIdentifiedSemiAuto[3].Line, false).Value; var b = LineIdentifiedSemiAuto[0].Line.Intersect(LineIdentifiedSemiAuto[1].Line, false).Value; var c = LineIdentifiedSemiAuto[2].Line.Intersect(LineIdentifiedSemiAuto[3].Line, false).Value; var d = LineIdentifiedSemiAuto[1].Line.Intersect(LineIdentifiedSemiAuto[2].Line, false).Value; QuadSemiAuto = new QuadrilateralInCamera(a, b, c, d, new Vector(Size.x, Size.y)); if (!QuadSemiAuto.IsQuadComplete()) QuadSemiAuto = null; } #endregion // 优先应用自动的结果(也可以在外部手动设置) if (QuadSemiAuto == null && QuadAuto == null && Screen.QuadInCamera != null) // 如果可能,回退到上一个screen { Debug.Log($"[ScreenIdentification] 本次识别失败,回退到上次的识别结果: {Screen.QuadInCamera}"); quadTemp.Add(Screen.QuadInCamera); } else if (QuadAuto != null) { Debug.Log($"[ScreenIdentification] 识别到四边形(全自动): {QuadAuto}"); quadTemp.Add(QuadAuto); } else if (QuadSemiAuto != null) { Debug.Log($"[ScreenIdentification] 识别到四边形(半自动): {QuadSemiAuto}"); quadTemp.Add(QuadSemiAuto); } #region 绘制 output texture // 绘制半自动 var ScreenQuadMap = new Matrix(o0Size, Tiling: true); // 识别的到的屏幕四边形(半自动和自动在一张图上) foreach (var i in LineIdentifiedSemiAuto.Index()) { if (LastQuadSemiAutoState[i]) o0Extension.DrawLine(ScreenQuadMap, LineIdentifiedSemiAuto[i].DrawLine, (x, y) => 5, new Vector(0, 10)); else o0Extension.DrawLine(ScreenQuadMap, LineIdentifiedSemiAuto[i].DrawLine, (x, y) => 3, new Vector(0, 6), true); } // 绘制全自动 foreach (var i in LineIdentifiedAuto.Index()) o0Extension.DrawLine(ScreenQuadMap, LineIdentifiedAuto[i].DrawLine, (x, y) => 4, new Vector(0, 4), true); Texture2D ScreenQuad = ScreenQuadMap.ToTexRGBA(FloatValueToColor); Texture2D ScreenQuadWithScreen = ScreenQuad.Overlay(ScreenLocateTexture); // 叠加屏幕色差图 // 绘制allLines var allLinesMap = new Matrix(o0Size, Tiling: true); foreach (var l in allLines) { if (l.DrawLine != null) o0Extension.DrawLine(allLinesMap, l.DrawLine, (x, y) => 1, new Vector(0, 2), true); } var allLinesTex = allLinesMap.ToTexRGBA(FloatValueToColor); ScreenLocate.DebugTexture(1, allLinesTex); // 还需要输出一张识别结果图,包含干扰线段 var ChoosableLineMap = new Matrix(o0Size, Tiling: true); foreach (var l in possibleLines) { if (l != null && !quadLinesSemiAuto.Contains(l) && !manualLines.Contains(l)) o0Extension.DrawLine(ChoosableLineMap, l.DrawLine, (x, y) => 1, new Vector(0, 2), true); // 其他的备选线段 } foreach (var l in LineIdentifiedSemiAuto) { if (l != null) o0Extension.DrawLine(ChoosableLineMap, l.DrawLine, (x, y) => 5, new Vector(0, 5)); // 识别的结果 } if (manualLines != null) { foreach (var l in manualLines) o0Extension.DrawLine(ChoosableLineMap, l.DrawLine, (x, y) => 3, new Vector(0, 2), true); // 旧的屏幕线段(例如上次手动识别的) } Texture2D ChoosableLineTex = ChoosableLineMap.ToTexRGBA(FloatValueToColor); #endregion log += $"\r\n屏幕四边形_手动识别{QuadManual != null}\r\n屏幕四边形_半自动识别{QuadSemiAuto != null}\r\n屏幕四边形_全自动识别{QuadAuto != null}"; Debug.Log(log); // 是否将图片保存到本地 if (ScreenLocate.Main.SaveToggle?.isOn ?? false && ScreenLocate.Main.DebugOnZIMDemo) { var FileDirectory = $"Debug_屏幕定位/"; SaveImages(FileDirectory, log, ScreenLocateTexture, allLinesTex, ChoosableLineTex, ScreenQuad); } //times.Add(watch.ElapsedMilliseconds); //Debug.Log("time: " + (times[times.Count - 1] - times[times.Count - 2])); // opecncv处理, zim { //var cvLines = edge.cvHoughLinesP(); //ScreenLocate.DebugTexture(5, cvLines); //var myLines = Hough.Transform(edgeMat); //var cvLines = edge.cvLine(myLines); //ScreenLocate.DebugTexture(5, cvLines); } { ScreenLocate.DebugTexture(2, ScreenLocateTexture); ScreenLocate.DebugTexture(3, ScreenQuad); ScreenLocate.DebugTexture(4, ScreenQuadWithScreen); ScreenLocate.DebugTexture(5, ChoosableLineTex); } foreach (var i in LocateTexTemp) { if (i != ScreenLocateTexture) // ScreenLocateTexture 由 ScreenLocate.DebugTexture 释放 GameObject.Destroy(i); } } Vector GetAvgPoint(Matrix screenLocateMat) { // 加权平均 Vector[] avgPointsColumn = new Vector[screenLocateMat.Size.x]; float[] valueSumsColumn = new float[screenLocateMat.Size.x]; Parallel.For(0, screenLocateMat.Size.x, i => { for (int j = 0; j < screenLocateMat.Size.y; j++) { var value = screenLocateMat[i, j]; valueSumsColumn[i] += value; avgPointsColumn[i] += new Vector(i, j) * value; } }); Vector avgPoint = Vector.Zero; var valueSum = 0f; for (int i = 0; i < screenLocateMat.Size.x; i++) { avgPoint += avgPointsColumn[i]; valueSum += valueSumsColumn[i]; } avgPoint /= valueSum; return avgPoint; } // 返回查找到的线段数量,0是查找失败 int ZIMIdentifyQuadLSD(ref List allLines, int batch, (Matrix edgeMat, Matrix edgeDirMat) edgeGradient, float minLength, Vector LineCaptureSize) { var l = edgeGradient.edgeMat.IdentifyLineLSD(edgeGradient.edgeDirMat, minLength, 25, LineCaptureSize); if (l == null || l.Count == 0) return 0; allLines.AddRange(l.Select((i) => new LineIdentified(batch, i))); return l.Count; } // 返回四边形的四条边(半自动、全自动),List长度一定是4 (如果没有识别到就是null),且线段顺序是: 下、右、上、左 (List, List) FilterLines(List screenLocateMatList, List allLines, Vector avgPoint, out LineIdentified[] manualLines, out List possibleLines, float gradientLength, float minLength = 100) { // 筛掉椭圆框外的线段(超出一半会筛掉) var innerLines = new List(); for (int i = 0; i < allLines.Count; i++) { List InArea = new List(); var dir = (allLines[i].Line.B - allLines[i].Line.A) / 4; var points = new Vector[5] { allLines[i].Line.A, allLines[i].Line.A + dir, allLines[i].Line.A + dir * 2f, allLines[i].Line.A + dir * 3f, allLines[i].Line.B }; // A点、中间的点、B点 for (int pI = 0; pI < points.Length; pI++) { if (ScreenLocate.Main.ScreenPixelCheaker != null && !ScreenLocate.Main.ScreenPixelCheaker.OutArea2D(points[pI], Size)) InArea.Add(points[pI]); } if (InArea.Count < 2) // 少于2个点在内部 continue; else if (InArea.Count < points.Length) // 不完全在内部 allLines[i].DrawLine = new Line(InArea.First(), InArea.Last()); // 将部分线条设置为drawline,用于下一步的绘制 else // 线段全部在椭圆内 allLines[i].DrawLine = allLines[i].Line; innerLines.Add(allLines[i]); } // 角度阈值,用来判断线段的梯度方向是否指向屏幕中心(avgPoint) var avaAngleHalf = 75f; // 评估屏幕内部的Line var interLineGuess = new InterLineGuess(screenLocateMatList, gradientLength * 2, minLength); #region 内部函数 float ScreenGrad(LineIdentified line) { var dir = (line.Line.B - line.Line.A).Normalized; var vertical = new Vector(-dir.y, dir.x) * (gradientLength / 2); int step = (int)(minLength / 5); var ll = line.Line.Length; var lg = new List(); for (int i = 0; i <= ll; i += step) { var point = line.Line.A + dir * i; var ga = point + vertical; var gb = point - vertical; lg.Add(screenLocateMatList[line.Batch][(int)ga.x, (int)ga.y] - screenLocateMatList[line.Batch][(int)gb.x, (int)gb.y]); } return Math.Abs(lg.Mean()); } // 沿直线计算综合梯度(梯度乘以长度系数,再乘以距离系数), distanceRatio是实际距离除以距离阈值 float estimateGradient(LineIdentified line, float distanceRatio) { var gM = ScreenGrad(line); if (line.Batch > 0) // 其他batch的图,梯度权重小 gM /= 3; float e = (float)Math.Sqrt(Math.Ceiling(line.Line.Length / minLength)); // 长度系数,筛选时梯度更大、长度更长的线段更优 float d = (10f - distanceRatio * distanceRatio) / 10f; // 距离系数,距离越近,系数越大 line.ZIMGradient = e * gM + d; // 记录一下综合梯度,新增的识别黑边功能会二次使用 return line.ZIMGradient; } // 根据线段梯度的角度,判断是不是屏幕的边,index代表是哪条边(顺序是: 下、右、上、左) void GetScreenLineIndex(LineIdentified line) { var a = (avgPoint - (line.Line.A + line.Line.B) / 2).DegreeToXAxis(); //Debug.Log(a + ", " + gradient + ", " + sum); var index = -1; if (Math.Abs(a - line.GradientDegree) < avaAngleHalf || Math.Abs(a - 360 - line.GradientDegree) < avaAngleHalf || Math.Abs(a + 360 - line.GradientDegree) < avaAngleHalf) { if (line.GradientDegree > 45 && line.GradientDegree < 135) // 下 index = 0; else if (line.GradientDegree > 135 && line.GradientDegree < 225) // 右 index = 1; else if (line.GradientDegree > 225 && line.GradientDegree < 315) // 上 index = 2; else index = 3; } line.ScreenLineIndex = index; } #endregion // 根据梯度方向,判断是哪条边 foreach (var l in innerLines) GetScreenLineIndex(l); // 下、右、上、左, 半自动和自动 var quadLinesSemiAuto = new List<(float, LineIdentified)>[4] { new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>() }; var quadLinesAuto = new List<(float, LineIdentified)>[4] { new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>() }; possibleLines = new List(); #region 半自动(利用手动数据) // 如果已有手动定位数据,根据现有数据筛选线条(半自动) manualLines = null; if (QuadManual != null) { Debug.Log($"[IdentifyLineLSD] 根据已有定位数据做筛选, QuadManual: {QuadManual}"); manualLines = QuadManual.GetLines().Select((i) => new LineIdentified(0, i, 0, 0, true)).ToArray(); var calibration = ScreenLocate.Main.ReDoLocateCalibrationRatio * Size.y; var distanceMeasure = 0.02f * Size.y; var avgPointCross = manualLines.Select((i) => i.Line.LineCrossWithPoint(avgPoint)).ToArray(); // 对于平均点的corss值 var avgPointPedal = manualLines.Select((i) => o0Extension.PointPedal(i.Line, avgPoint, out _)).ToArray(); // 当前定位的垂足,下、右、上、左 foreach (var line in innerLines) { // 筛选条件:1-梯度方向匹配,2-垂足的距离足够近, 3-线段的AB点均在旧线段外部, 4-新的线段的中点,到旧线段的垂足,要在旧线段内 if (line.ScreenLineIndex >= 0) { var distanceToOld = (o0Extension.PointPedal(line.Line, avgPoint, out _) - avgPointPedal[line.ScreenLineIndex]).Length; if (distanceToOld < calibration && manualLines[line.ScreenLineIndex].Line.LineCrossWithPoint(line.Line.A) * avgPointCross[line.ScreenLineIndex] <= 0 && manualLines[line.ScreenLineIndex].Line.LineCrossWithPoint(line.Line.B) * avgPointCross[line.ScreenLineIndex] <= 0) { var middleToOldLine = o0Extension.PointPedal(manualLines[line.ScreenLineIndex].Line, (line.Line.A + line.Line.B) / 2, out bool inLineSegment); if (inLineSegment) { quadLinesSemiAuto[line.ScreenLineIndex].Add((estimateGradient(line, (float)Math.Floor(distanceToOld / distanceMeasure)), line)); possibleLines.Add(line); } } } } } // 获得结果 var resultSemiAuto = new LineIdentified[4]; var resultSemiAutoPedal = new Vector[4]; // 用于找平行线 for (int i = 0; i < 4; i++) { resultSemiAuto[i] = quadLinesSemiAuto[i].Max((a, b) => a.Item1.CompareTo(b.Item1)).Item2; if (resultSemiAuto[i] != null) resultSemiAutoPedal[i] = o0Extension.PointPedal(resultSemiAuto[i].Line, avgPoint, out _); } // 新增功能(解决黑边问题):根据 result 再找平行线,判断是否替换(1-在 result 内部,且离中点最近,2-接近平行) UpdateResultlines(resultSemiAuto, FindInterLinePair( interLineGuess, GetInterSelectableLines(quadLinesSemiAuto, resultSemiAuto, resultSemiAutoPedal, avgPoint))); #endregion #region 全自动 // 全自动 foreach (var line in innerLines) { if (line.ScreenLineIndex >= 0 && line.Batch < 1) // 全自动只处理第一张图,默认是色差图 { quadLinesAuto[line.ScreenLineIndex].Add((estimateGradient(line, 1), line)); } } // 获得结果 var resultAuto = new LineIdentified[4]; var resultAutoPedal = new Vector[4]; // 用于找平行线 for (int i = 0; i < 4; i++) { resultAuto[i] = quadLinesAuto[i].Max((a, b) => a.Item1.CompareTo(b.Item1)).Item2; if (resultAuto[i] != null) resultAutoPedal[i] = o0Extension.PointPedal(resultAuto[i].Line, avgPoint, out _); } // 新增功能(解决黑边问题):根据 resultAuto 再找平行线,判断是否替换(1-在 result 内部,且离中点最近,2-接近平行,3-LineGuess判断是直线) UpdateResultlines(resultAuto, FindInterLinePair( interLineGuess, GetInterSelectableLines(quadLinesAuto, resultAuto, resultAutoPedal, avgPoint))); #endregion return (resultSemiAuto.ToList(), resultAuto.ToList()); } List GetInterSelectableLines(List<(float, LineIdentified)>[] quadLines, LineIdentified[] resultLines, Vector[] resultPedal, Vector avgPoint) { var result = new List(); foreach (var ql in quadLines) { foreach (var (_, line) in ql) { if (line != resultLines[line.ScreenLineIndex] && line.Batch < 1) // batch0才做黑边的内部线条检测 { var pedal = o0Extension.PointPedal(line.Line, avgPoint, out _); var a0 = pedal - avgPoint; var a0L = a0.Length; line.DistanceToMiddle = a0L; var a1 = resultPedal[line.ScreenLineIndex] - avgPoint; var a1L = a1.Length; if (a0L < a1L) { var dotN = a0.Dot(a1) / a0L / a1L; if (Math.Abs(dotN - 1) < 0.002) // 接近平行即可 result.Add(line); } } } } return result; } void UpdateResultlines(LineIdentified[] result, (LineIdentified a, LineIdentified b) inter) { if (inter.a != null) // 替换上一步筛选的结果中的部分边,得到最终的结果 result[inter.a.ScreenLineIndex] = inter.a; if (inter.b != null) result[inter.b.ScreenLineIndex] = inter.b; } (LineIdentified a, LineIdentified b) FindInterLinePair(InterLineGuess lineGuess, List interSelectable, int maxCountToSelect = 8) { Debug.Log("[ScreenIdentification] selectable inter line count: " + interSelectable.Count); interSelectable.Sort((a, b) => b.ZIMGradient.CompareTo(a.ZIMGradient)); int count = 0; LineIdentified[] selected = new LineIdentified[4]; foreach (var line in interSelectable) { if (line.GuessIsInterLine(lineGuess)) // 评价是不是Line, 并且找到离中心点最近的 { if (ScreenLocate.Main.DebugOnZIMDemo) Debug.Log($"[ScreenIdentification] {interSelectable.IndexOf(line)}, guess is line: (index)" + line.ScreenLineIndex); if (selected[line.ScreenLineIndex] == null || selected[line.ScreenLineIndex].DistanceToMiddle > line.DistanceToMiddle) selected[line.ScreenLineIndex] = line; } if (count++ >= maxCountToSelect) break; } var selectedList = new List(); foreach (var i in selected) { if (i != null) selectedList.Add(i); } if (selectedList.Count == 4) { if (selected[0].ZIMGradient + selected[2].ZIMGradient > selected[1].ZIMGradient + selected[3].ZIMGradient) return (selected[0], selected[2]); else return (selected[1], selected[3]); } else if (selected[0] != null && selected[2] != null) return (selected[0], selected[2]); else if (selected[1] != null && selected[3] != null) return (selected[1], selected[3]); else if (selectedList.Count == 2) return selectedList[0].ZIMGradient > selectedList[1].ZIMGradient ? (selectedList[0], null) : (selectedList[1], null); else if (selectedList.Count == 1) return (selectedList[0], null); else return (null, null); } void SaveImages(string FileDirectory, string log, Texture2D ScreenLocateTex, Texture2D allLinesTex, Texture2D ChoosableLineTex, Texture2D ScreenQuadTex) { if (!Directory.Exists(FileDirectory)) Directory.CreateDirectory(FileDirectory); var time = DateTime.Now.ToString("yyyyMMdd_HHmmss"); var pngData = ScreenLocate.Main.OutputTextures[7]?.EncodeToPNG(); if (pngData != null) File.WriteAllBytes($"{FileDirectory}{time}A屏幕原图.png", pngData); var pngData1 = ScreenLocateTex?.EncodeToPNG(); if (pngData1 != null) File.WriteAllBytes($"{FileDirectory}{time}B黑白色差.png", pngData1); var pngData2 = allLinesTex?.EncodeToPNG(); if (pngData2 != null) File.WriteAllBytes($"{FileDirectory}{time}C全部识别线段_半自动.png", pngData2); var pngData3 = ChoosableLineTex?.EncodeToPNG(); if (pngData3 != null) File.WriteAllBytes($"{FileDirectory}{time}D备选线段_半自动.png", pngData3); var pngData4 = ScreenQuadTex?.EncodeToPNG(); if (pngData4 != null) File.WriteAllBytes($"{FileDirectory}{time}E识别结果.png", pngData4); Debug.Log($"({time}) 屏幕识别图片保存至:程序根目录/{FileDirectory}"); log += $"\r\n屏幕原图保存{pngData != null}, " + $"\r\n黑白色差保存{pngData1 != null}, " + $"\r\n全部识别线段(半自动)保存{pngData2 != null}, " + $"\r\n备选线段(半自动)保存{pngData3 != null}, " + $"\r\n识别结果保存{pngData4 != null}"; File.WriteAllText($"{FileDirectory}{time}屏幕自动定位_日志.log", log); } } }