smart-bow-infrared/Assets/InfraredProject/WebCamera/Script/ZIM/InfraredLocate/ScreenIdentification.cs

#define ENABLE_LOG

using o0.Geometry2D.Float;
using o0.Num;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using UnityEngine;
using UnityEngine.UIElements;
using ZIM;
using ZIM.Unity;

namespace o0.Project
{
    public partial class ScreenIdentification
    {
        private const string TAG = "ScreenIdentification#";

        // LocateAreaData表示每次屏幕的色差变化的区域，可能有多次。通过设置LocateSingleStep可调整为仅识别一次色差
        static Rect[][] LocateAreaData = new Rect[][] {
             new Rect[] { new Rect(0f, 0f, 0.3f, 0.3f),     new Rect(0f, 0f, 0.4f, 0.4f),     new Rect(0f, 0f, 0.5f, 0.5f)     },
             new Rect[] { new Rect(0.7f, 0f, 0.3f, 0.3f),   new Rect(0.6f, 0f, 0.4f, 0.4f),   new Rect(0.5f, 0f, 0.5f, 0.5f)   },
             new Rect[] { new Rect(0f, 0.7f, 0.3f, 0.3f),   new Rect(0f, 0.6f, 0.4f, 0.4f),   new Rect(0f, 0.5f, 0.5f, 0.5f)   },
             new Rect[] { new Rect(0.7f, 0.7f, 0.3f, 0.3f), new Rect(0.6f, 0.6f, 0.4f, 0.4f), new Rect(0.5f, 0.5f, 0.5f, 0.5f) }
        };
        //static Rect[][] LocateAreaData = new Rect[][] {
        //     new Rect[] { new Rect(0f, 0f, 0.3f, 0.3f),     new Rect(0f, 0f, 0.4f, 0.4f),     new Rect(0f, 0f, 0.5f, 0.5f),     new Rect(0f, 0f, 0.6f, 0.6f) },
        //     new Rect[] { new Rect(0.7f, 0f, 0.3f, 0.3f),   new Rect(0.6f, 0f, 0.4f, 0.4f),   new Rect(0.5f, 0f, 0.5f, 0.5f),   new Rect(0.4f, 0f, 0.6f, 0.6f) },
        //     new Rect[] { new Rect(0f, 0.7f, 0.3f, 0.3f),   new Rect(0f, 0.6f, 0.4f, 0.4f),   new Rect(0f, 0.5f, 0.5f, 0.5f),   new Rect(0f, 0.4f, 0.6f, 0.6f) },
        //     new Rect[] { new Rect(0.7f, 0.7f, 0.3f, 0.3f), new Rect(0.6f, 0.6f, 0.4f, 0.4f), new Rect(0.5f, 0.5f, 0.5f, 0.5f), new Rect(0.4f, 0.4f, 0.6f, 0.6f) }
        //};

        //static bool LocateSingleStep = false;
        static bool LocateSingleStep = true;

        public Geometry2D.Vector<int> Size => ScreenLocate.Main.CameraSize;

        public QuadrilateralInCamera QuadManual;
        public QuadrilateralInCamera QuadAuto;      // 全自动，可以给用户选择（赋值给Screen.QuadInCamera即生效）
        public QuadrilateralInCamera QuadSemiAuto;  // 半自动，可以给用户选择（赋值给Screen.QuadInCamera即生效）

        public ScreenMap Screen;       // 识别到的屏幕，用于执行透视变换

        int capture = 0;
        int delay = 0;
        int maxCapture;
        int maxDelay;


        Geometry.Vector<float>[] ScreenBlackTexture;
        Geometry.Vector<float>[] ScreenWhiteTexture;
        int locateIndex = -1;
        readonly List<Rect> locateArea = new List<Rect> {
            new Rect(0f, 0f, 0.5f, 0.5f), new Rect(0.5f, 0f, 0.5f, 0.5f), new Rect(0f, 0.5f, 0.5f, 0.5f), new Rect(0.5f, 0.5f, 0.5f, 0.5f)
        };      // 屏幕显示白色的区域大小

        float areaPercent => locateArea[locateIndex].size.x;         // 当前白色区域的占比
        int areaSelected = -1;                                       // 选择哪个区域，顺序与Quadrilateral对应
        readonly List<float> sumTemp = new List<float>();
        readonly List<QuadrilateralInCamera> quadTemp = new List<QuadrilateralInCamera>();

        //public ScreenIdentification(WebCamTexture texture)
        //{
        //    Size = new Geometry2D.Vector<int>(texture.width, texture.height);
        //    Screen = new ScreenMap();
        //}
        public static UnityEngine.Color FloatValueToColor(float i)
        {
            return i switch
            {
                1 => UnityEngine.Color.green,
                2 => UnityEngine.Color.red,
                3 => UnityEngine.Color.yellow,
                4 => UnityEngine.Color.white,
                _ => UnityEngine.Color.black,
            };
        }

        public ScreenIdentification()
        {
            Screen = new ScreenMap();
            OnLocateScreenEnter += () => Application.targetFrameRate = 30;      // 固定识别的帧率，确保摄像机拍到正确的画面
            OnLocateScreenEnd += () => Application.targetFrameRate = 60;
        }

        public void SetScreenQuad(QuadrilateralInCamera quad) => Screen.QuadInCamera = quad;

        // 上一次半自动识别的情况，false代表这条边识别失败，线段顺序: 下、右、上、左
        public bool[] LastQuadSemiAutoState;

        public event Action OnLocateScreenEnter;
        public event Action OnLocateScreenEnd;
        public bool bStartLocateScreen { get; set; } = false;//是否进行捕获

        public bool SelectScreenAfterLocate(ScreenLocate.ScreenIdentificationTag tag)
        {
            QuadrilateralInCamera target = tag switch
            {
                ScreenLocate.ScreenIdentificationTag.Manual => QuadManual,
                ScreenLocate.ScreenIdentificationTag.SemiAuto => QuadSemiAuto,
                ScreenLocate.ScreenIdentificationTag.Auto => QuadAuto,
                _ => null
            };

            if (target == null)
                return false;
            Debug.Log($"<color=aqua>[ScreenIdentification] 选择已识别到的屏幕({Enum.GetName(typeof(ScreenLocate.ScreenIdentificationTag), tag)}), {target}</color>");
            SetScreenQuad(target);
            return true;
        }

        // 自动识别开始的入口
        public void LocateScreen(int Capture = 30, int Delay = 30)  //数值单位是frame
        {
            if (ScreenLocate.Main.DebugScreenImages.Count != 0 && ScreenLocate.Main.DebugOnZIMDemo)     // 这段仅用于测试图片
            {
                ScreenLocate.Main.CameraSize = new Geometry2D.Vector<int>(ScreenLocate.Main.DebugScreenImages[0].width, ScreenLocate.Main.DebugScreenImages[0].height);
                DebugImage(ScreenLocate.Main.DebugScreenImages);
                Screen.QuadInCamera = quadTemp[0];
                ScreenLocate.SetScreen(null);
                ScreenLocate.Main.ShowScreen(ScreenLocate.Main.ScreenQuad, Screen.QuadInCamera);
                delay = 0;
                capture = 0;
                ScreenWhiteTexture = null;
                ScreenBlackTexture = null;
                locateIndex = -1;
                areaSelected = -1;
                quadTemp.Clear();
                sumTemp.Clear();
                ScreenLocate.Main.DebugScreenImages.Clear();
                return;
            }

            delay = Math.Max(Delay, 5);
            capture = Math.Max(Capture, 5);
            maxDelay = Delay;
            maxCapture = Capture;

            ScreenLocate.SetScreen(new Rect(0f, 0f, 1f, 1f), UnityEngine.Color.black);
            //ScreenLocate.SetScreen(new Rect(0f, 0f, 0.6f, 0.6f), UnityEngine.Color.white);

            //bStartLocateScreen = false;

            OnLocateScreenEnter?.Invoke();
        }
        /// <summary>
        /// 开始进行捕获
        /// 初始化了两个数据 capture 和 delay
        /// </summary>
        /// <returns></returns>
        public bool isInitLocateScreen()
        {
            return capture != 0 && delay != 0;
        }

        void DebugImage(List<Texture2D> images)
        {
            QuadrilateralFit(images);

            //var watch = new System.Diagnostics.Stopwatch();
            //watch.Start();
            //var times = new List<double>() { 0.0 };
#if (!NDEBUG && DEBUG && ENABLE_LOG)
            Console.WriteLine($"{TAG} quadTemp.Count:{quadTemp.Count}");
#endif
            if (quadTemp.Count > 0)
            {
                ScreenLocate.Main.ShowScreen(ScreenLocate.Main.outputRawImages[4].transform.GetChild(0) as RectTransform, quadTemp[0]);

                // 透视变换
                //                var srcWidth = LocateLightedRedTex.width;
                //                var transformWidth = (int)((quad.B.x - quad.A.x + quad.D.x - quad.C.x) / 2);
                //                var transformHeight = (int)((quad.C.y - quad.A.y + quad.D.y - quad.B.y) / 2);
                //                var transformTex = new Texture2D(transformWidth, transformHeight);
                //                var pt = new ZIMPerspectiveTransform(new OrdinalQuadrilateral(new Vector(0, 0), new Vector(transformWidth, 0), new Vector(0, transformHeight), new Vector(transformWidth, transformHeight)), quad);
                //                var dstPixel = new UnityEngine.Color[transformWidth * transformHeight];
                //                var srcPixel = LocateLightedRedTex.GetPixels();
                //                Parallel.For(0, transformWidth, (x) =>
                //                {
                //                    for (int y = 0; y < transformHeight; y++)
                //                    {
                //                        var index = y * transformWidth + x;
                //                        var sampleCoord = pt.TransformRound(x, y);
                //                        dstPixel[index] = srcPixel[sampleCoord.y * srcWidth + sampleCoord.x];
                //                    }
                //                });
                //                transformTex.SetPixels(dstPixel);
                //                transformTex.Apply();
                //                //ScreenLocate.DebugTexture(1, transformTex);
                //#if (!NDEBUG && DEBUG && ENABLE_LOG)
                //                Console.WriteLine($"{TAG} ScreenLocate.DebugTexture 1:{transformTex.GetNativeTexturePtr()}");
                //#endif
            }

            //times.Add(watch.ElapsedMilliseconds);
            //Debug.Log("time: " + (times[times.Count - 1] - times[times.Count - 2]));
        }

        public void NextScreen()
        {
            // 只识别一次色差变化
            if (LocateSingleStep && areaSelected == -1)
            {
                LocateAreaData = new Rect[][] { new Rect[] { new Rect(0, 0, 1f, 1f) } };
                locateIndex = 3;
                areaSelected = 0;
                locateArea.AddRange(LocateAreaData[0]);
            }

            // index从-1开始
            locateIndex++;
            if (locateIndex < locateArea.Count)     // 依次点亮屏幕区域
            {
                ScreenLocate.SetScreen(locateArea[locateIndex], UnityEngine.Color.white);
                delay = maxDelay;
                capture = maxCapture;
            }
            else                // 退出屏幕黑白控制
            {
                ScreenLocate.SetScreen(null);
                ScreenLocate.Main.ShowScreen(ScreenLocate.Main.ScreenQuad, Screen.QuadInCamera);
                Reset();
            }
        }

        // 清除记录的屏幕识别数据（手动、自动等）
        public void ClearQuadCache()
        {
            SetScreenQuad(null);
            QuadManual = null;
            QuadSemiAuto = null;
            QuadAuto = null;
        }

        public void Reset()
        {
            // bStartLocateScreen = false;
            delay = 0;
            capture = 0;
            ScreenWhiteTexture = null;
            ScreenBlackTexture = null;
            locateIndex = -1;
            areaSelected = -1;
            if (locateArea.Count > 4)
                locateArea.RemoveRange(4, LocateAreaData[0].Length);
            quadTemp.Clear();
            sumTemp.Clear();
        }

        public void CaptureBlack(Texture2D cam)
        {
            if (ScreenBlackTexture == null)
                ScreenBlackTexture = new Geometry.Vector<float>[Size.x * Size.y];
            var pixel = cam.GetPixels();
            Parallel.For(0, Size.x * Size.y, i =>
            {
                var ip = pixel[i];
                ScreenBlackTexture[i] += new Geometry.Vector<float>(ip.r / maxCapture, ip.g / maxCapture, ip.b / maxCapture);
            });
        }

        public void CaptureWhite(Texture2D cam)
        {
            if (ScreenWhiteTexture == null)
                ScreenWhiteTexture = new Geometry.Vector<float>[Size.x * Size.y];
            var pixel = cam.GetPixels();
            Parallel.For(0, Size.x * Size.y, i =>
            {
                var ip = pixel[i];
                ScreenWhiteTexture[i] += new Geometry.Vector<float>(ip.r / maxCapture, ip.g / maxCapture, ip.b / maxCapture);
            });
        }

        public void CaptureStay(Texture2D cam)
        {
            if (locateIndex == -1)                      // 屏幕黑色
            {
                CaptureBlack(cam);
            }
            else                                        // 屏幕部分为白色
            {
                CaptureWhite(cam);
            }
        }

        public void CaptureEnd()
        {
            //Debug.Log("locateIndex: " + locateIndex + ", quad: " + quadTemp.Count);
            if (locateIndex == -1)
                return;

            if (locateIndex < 4)
            {
                sumTemp.Add(GetBrightness());
                ScreenWhiteTexture = null;

                // 选择亮度差最大的区域
                if (locateIndex == 3)
                {
                    areaSelected = sumTemp.MaxIndex();
                    locateArea.AddRange(LocateAreaData[areaSelected]);
                }
            }
            else if (locateIndex >= 4 && locateIndex < locateArea.Count - 1)
            {
                QuadrilateralFit();
                ScreenWhiteTexture = null;
            }
            else
            {
                QuadrilateralFit();

                if (quadTemp.Count != LocateAreaData[0].Length)
                {
                    Debug.Log($"<color=yellow>[ScreenIdentification] 拟合四边形失败, quadTemp.Count: {quadTemp.Count}</color>");
                }
                else if (quadTemp.Count == 1)
                {
                    SetScreenQuad(quadTemp[0]);
                    Debug.Log($"[ScreenIdentification] 拟合成功，识别数据: {Screen.QuadInCamera}");
                }
                else
                {
                    // Debug.Log($"拟合四边形 2 , quadTemp.Count: {quadTemp.Count}");
                    // 线性拟合
                    var xValue = new List<float>() { 0 };
                    var predicts = new List<Vector>();
                    foreach (var i in LocateAreaData[0])
                        xValue.Add(i.size.x);

                    Vector baseVertex = Vector.Zero;            // x==0 时的点
                    {
                        foreach (var q in quadTemp)
                        {
                            baseVertex += q.Quad[areaSelected];
                        }
                        baseVertex /= quadTemp.Count;
                    }
                    double rs = 0.0;
                    for (int i = 0; i < 4; i++)
                    {
                        if (i == areaSelected)
                        {
                            predicts.Add(baseVertex);
                        }
                        else
                        {
                            var yValue = new List<Vector>() { baseVertex };
                            foreach (var q in quadTemp)
                            {
                                yValue.Add(q.Quad[i]);
                            }
                            var lr = LinerRegression1D.Fit(2, xValue.ToArray(), yValue.ToArray());
                            rs += lr.RSquared / 3;
                            predicts.Add(lr.Predict<Vector>(1));
                        }
                    }
                    SetScreenQuad(new QuadrilateralInCamera(predicts, new Vector(Size.x, Size.y)));
                    Debug.Log($"[ScreenIdentification] 拟合成功，RSquared: {rs}, Quad: {Screen.QuadInCamera}");
                    //if (rs < 0.8) Screen.Quad = null;
                }

                OnLocateScreenEnd?.Invoke();
            }

        }

        public bool Update(Texture2D cam)
        {
            //if (!bStartLocateScreen) return false;

            if (delay != 0)
            {
                //ScreenLocate.Main.CreateUVCTexture2DFocusSizeIfNeeded(1280, 720);
                delay--;
                if (delay == 0)
                {
                    ScreenLocate.Main.CameraSize = new Geometry2D.Vector<int>(cam.width, cam.height);          // 记录当前的分辨率
                    Debug.Log("[ScreenIdentification] 采样纹理，记录采样分辨率: [" + Size.x + ", " + Size.y + "]");
                }
                return true;
            }
            if (capture != 0)
            {
                //ScreenLocate.Main.CreateUVCTexture2DFocusSizeIfNeeded(1280, 720);
                CaptureStay(cam);

                capture--;
                if (capture == 0)
                {
                    CaptureEnd();
                    NextScreen();
                }
                return true;
            }

            return false;

            #region Old
            /*


    if (delay != 0)
    {
        delay--;
        return true;
    }
    if (capture != 0)
    {
        capture--;
        if (ScreenBlackTexture == null)
            ScreenBlackTexture = new Geometry.Vector<float>[Size.x * Size.y];
        var pixel = cam.GetPixels();
        Parallel.For(0, Size.x * Size.y, i =>
        {
            var ip = pixel[i];
            ScreenBlackTexture[i] += new Geometry.Vector<float>(ip.r, ip.g, ip.b);
        });
        if (capture == 0)
            ScreenLocate.SetScreen(UnityEngine.Color.black);
        return true;
    }

    if (delay != 0)
    {
        delay--;
        return true;
    }
    if (capture != 0)
    {
        capture--;
        if (ScreenWhiteTexture == null)
            ScreenWhiteTexture = new Geometry.Vector<float>[Size.x * Size.y];
        var pixel = cam.GetPixels();
        Parallel.For(0, Size.x * Size.y, i =>
        {
            var ip = pixel[i];
            ScreenWhiteTexture[i] += new Geometry.Vector<float>(ip.r, ip.g, ip.b);
        });
        if (capture == 0)
            ScreenLocate.SetScreen(UnityEngine.Color.black);
        return true;
    }
    if (delay != 0)
    {
        delay--;
        return true;
    }
    if (capture != 0)
    {
        capture--;
        var pixel = cam.GetPixels();
        Parallel.For(0, Size.x * Size.y, i =>
        {
            var ip = pixel[i];
            ScreenWhiteTexture[i] -= new Geometry.Vector<float>(ip.r, ip.g, ip.b);
        });
        if (capture == 0)
        {
            ScreenLocate.SetScreen(null);
            UnityEngine.Color[] newPixel = new UnityEngine.Color[Size.x * Size.y];

            Parallel.For(0, Size.x * Size.y, i => {
                var pi = ScreenWhiteTexture[i] /= capture;
                newPixel[i] = new UnityEngine.Color(pi.x, pi.y, pi.z);
            });

            //读取数据
            //{
            //    var fileName = "3.bin";
            //    ScreenLocateTexture = $"2023 04 16 厦门测试数据/{fileName}".FileReadByte<Vector<float>[]>();
            //    Debug.Log($"Read {fileName}");
            //    Parallel.For(0, Size.x * Size.y, i =>
            //    {
            //        var pi = ScreenLocateTexture[i];
            //        newPixel[i] = new UnityEngine.Color(pi.x, pi.y, pi.z);
            //    });
            //}

            var ScreenLocateTex = new Texture2D(Size.x, Size.y);
            ScreenLocateTex.SetPixels(newPixel);
            ScreenLocateTex.Apply();
            //ScreenLocate.DebugTexture(2, ScreenLocateTex);
            var ScreenLocateTexLighted = ScreenLocateTex.AutoLight(10);

            //ScreenLocate.DebugTexture(2, ScreenLocateTexLighted);
            //var FileSavePath = Application.persistentDataPath + "/ScreenLocateTexture.bin";
            bool Save = ScreenLocate.Main.SaveToggle.isOn;
            string time;
            if (Save)
            {
                time = DateTime.Now.ToString("yyyyMMdd_HHmmss");
                var FileSavePath = $"屏幕定位数据{time}.bin";
                FileSavePath.FileWriteByte(ScreenWhiteTexture);

                var bytes = ScreenLocateTexLighted.EncodeToPNG();
                File.WriteAllBytes($"屏幕定位数据{time}.png", bytes);

                Debug.Log("ScreenLocateTexture Saved To: " + FileSavePath);
            }

            var ScreenLocateTexR = ScreenLocateTexLighted.ToRGB(ColorChannel.Red);
            var ScreenLocateTexG = ScreenLocateTexLighted.ToRGB(ColorChannel.Green);
            var ScreenLocateTexB = ScreenLocateTexLighted.ToRGB(ColorChannel.Blue);
            ScreenLocate.DebugTexture(2, ScreenLocateTexR);
            //ScreenLocate.DebugTexture(4, ScreenLocateTexG);
            //ScreenLocate.DebugTexture(5, ScreenLocateTexB);


            var watch = new System.Diagnostics.Stopwatch();
            watch.Start();
            var times = new List<double>() { 0.0 };



            var ScreenLocateTexLightedMat = ScreenLocateTexLighted.Too0Mat();
            //var ScreenLocateTexLightedMat = texture.Too0Mat();

            //var (edge, edgeDir) = ScreenLocateTexLightedMat.IdentifyEdge();
            var (edge, edgeDir) = ScreenLocateTexLightedMat.zimIdentifyEdgeGradientAny(15);
            //ScreenLocate.DebugTexture(4, ScreenLocateTexLighted.Too0Mat().IdentifyEdgeGradient().ToTex());
            //ScreenLocate.DebugTexture(4, edge.ToTex());

            var quadLines = ScreenLocateTexLightedMat.IdentifyQuadLSD(edge, edgeDir, out List<Line> lightLines, 30);
            var drawLineMap = new MatrixF2D(edge..Size.x, edge.Size.y);
            int lineCount = 0;
            foreach (var l in quadLines)
            {
                if (l != null)
                {
                    o0Extension.DrawLine(drawLineMap.DrawLine(l, (x, y) => 1, new Geometry2D.Float.Vector(0, 10));
                    lineCount++;
                }
            }
            if (lineCount == 4)
            {
                var a = quadLines[0].Intersect(quadLines[3], false).Value;
                var b = quadLines[0].Intersect(quadLines[1], false).Value;
                var c = quadLines[2].Intersect(quadLines[3], false).Value;
                var d = quadLines[1].Intersect(quadLines[2], false).Value;
                Quad = new Quadrilateral(a, b, c, d);
                if (!Quad.IsInScreen(ScreenLocate.Main.WebCamera.Size))
                    Quad = null;
            }
            ScreenLocate.Main.ShowScreen(Quad);

            //var lines = edge.IdentifyLineLSD(edgeDir, 100);
            ////var lines = ScreenLocateTexLightedMat.IdentifyLineLSD();

            //var drawLineMap = new MatrixF2D(edge..Size.x, edge.Size.y);
            //var returnMaxLines = lines.Sub(0, 10);
            //foreach (var (line, sum, gradient) in returnMaxLines)
            //    o0Extension.DrawLine(drawLineMap.DrawLine(line, (x, y) => 1, new Geometry2D.Float.Vector(0, 10));


            ScreenLocate.DebugTexture(3, drawLineMap.ToTex());
            //{
            //    var bytes = drawLineMap.ToTex().EncodeToPNG();
            //    File.WriteAllBytes($"屏幕定位数据DrawLineMap.png", bytes);
            //}


            times.Add(watch.ElapsedMilliseconds);
            Debug.Log("time: " + (times[times.Count - 1] - times[times.Count - 2]));

            //ScreenLocate.DebugTexture(5, edge.IdentifyLine(edgeDir).ToTex());
            //ScreenLocate.DebugTexture(4, ScreenLocateTexLighted.Too0Mat().IdentifyEdgeGradientX().ToTex());
            //ScreenLocate.DebugTexture(5, ScreenLocateTexLighted.Too0Mat().IdentifyEdgeGradientY().ToTex());

            //var convolutionLighted2 = ScreenLocateTexLighted.Too0Mat().IdentifyEdgeVariance().ToTex();

            // opecncv处理
            // zim
            {
                //var cvLines = edge.cvHoughLinesP();
                //ScreenLocate.DebugTexture(5, cvLines);

                //var myLines = Hough.Transform(edgeMat);
                //var cvLines = edge.cvLine(myLines);
                //ScreenLocate.DebugTexture(5, cvLines);
            }

            UnityEngine.Object.Destroy(ScreenLocateTex);


            //ScreenLocate.DebugTexture(4, convolutionLighted2);
        }

        return true;
    }

    /*
    var avg = new Geometry4D.Vector<float>();
    var pixel = texture.GetPixels();
    foreach(var i in pixel.Index())
    {
        var iP = pixel[i];
        avg += new Geometry4D.Vector<float>(iP.r, iP.g, iP.b, iP.a);
    }
    avg /= pixel.Count();

    /*
    var (texLightedR, texLightedG, texLightedB) = ToRGB(newTex);
    ScreenLocate.DebugTexture(3, texLightedR);
    ScreenLocate.DebugTexture(4, texLightedG);
    ScreenLocate.DebugTexture(5, texLightedB);
    //Debug.Log(avg);

    return false;

/**/
            #endregion


        }

        float GetBrightness()
        {
            UnityEngine.Color[] differPixel = new UnityEngine.Color[Size.x * Size.y];

            Parallel.For(0, Size.x * Size.y, i =>
            {
                var pi = ScreenWhiteTexture[i] - ScreenBlackTexture[i];
                differPixel[i] = new UnityEngine.Color(pi.x, pi.y, pi.z);
            });

            var sum = 0f;
            foreach (var i in differPixel)
            {
                sum += i.Brightness();
            }
            sum /= differPixel.Length;
            //Debug.Log(sum);
            return sum;
        }

        // 转换成屏幕定位所需的纹理图像
        Texture2D ToLocateTex(UnityEngine.Color[] pixels)
        {
            var ScreenLocateTex = new Texture2D(Size.x, Size.y);
            ScreenLocateTex.SetPixels(pixels);
            ScreenLocateTex.Apply();
            //ScreenLocate.DebugTexture(2, ScreenLocateTex);
            return ScreenLocateTex.AutoLight(10);

            //ScreenLocate.DebugTexture(2, ScreenLocateTexLighted);

            //var ScreenLocateTexR = ToLocateTex.ToRGB(ColorChannel.Red);
            //var ScreenLocateTexG = ToLocateTex.ToRGB(ColorChannel.Green);
            //var ScreenLocateTexB = ToLocateTex.ToRGB(ColorChannel.Blue);

            //LocateLightedRedTex = ScreenLocateTexR;
            //ScreenLocate.DebugTexture(2, ScreenLocateTexR);
            //ScreenLocate.DebugTexture(4, ScreenLocateTexG);
            //ScreenLocate.DebugTexture(5, ScreenLocateTexB);

            //var ScreenLocateTexLightedMat = texture.Too0Mat();
        }

        /// <param name="lineWidth">识别的最小线段长度</param>
        /// <param name="debugImages">这个参数如果不为null且数量大于0，则执行debug操作</param>
        void QuadrilateralFit(List<Texture2D> debugImages = null)
        {
            // 屏幕黑白差值，存放多批次的图像用于识别, 该List数量不能等于 0 
            List<UnityEngine.Color[]> PixelsMultipleBatches = new List<UnityEngine.Color[]>();

            var sw = new System.Diagnostics.Stopwatch();
            sw.Start();

            //读取数据
            if (debugImages != null && debugImages.Count != 0)
            {
                foreach (var i in debugImages)
                {
                    Debug.Log($"<color=aqua>Debug {i.name}</color>");
                    PixelsMultipleBatches.Add(i.GetPixels());
                }

            }
            else    // 获得屏幕差值
            {
                var maxWhite = 0f;
                foreach (var i in ScreenWhiteTexture)
                {
                    var m = i.x > i.y ? (i.x > i.z ? i.x : i.z) : (i.y > i.z ? i.y : i.z);
                    if (maxWhite < m)
                        maxWhite = m;
                }
                var scale = 1.0f / maxWhite;        // 放大对比度

                var differPixel = new UnityEngine.Color[Size.x * Size.y];
                //var whitePixel = new UnityEngine.Color[Size.x * Size.y];
                Parallel.For(0, Size.x, x =>
                {
                    for (int y = 0; y < Size.y; y++)
                    {
                        var i = y * Size.x + x;
                        var d = ScreenWhiteTexture[i] - ScreenBlackTexture[i];
                        differPixel[i] = new UnityEngine.Color(d.x, d.y, d.z) * scale;
                        //whitePixel[i] = new UnityEngine.Color(ScreenWhiteTexture[i].x, ScreenWhiteTexture[i].y, ScreenWhiteTexture[i].z) * scale;
                    }
                });
                PixelsMultipleBatches.Add(differPixel);     // 色差图
                //PixelsMultipleBatches.Add(whitePixel);    // 原图
            }

            int conSize = (int)Math.Ceiling(0.007f * Size.y) * 2 + 1;
            conSize = Math.Max(conSize, 7);     // 设置最小为7

            float minLength = conSize * 7.7f;
            minLength = locateIndex == -1 ? minLength : minLength * areaPercent;    // minLength需要按比例缩小
            string log = $"[Log][ScreenLocate Auto] Size: ({Size.x},{Size.y}), 卷积核Size: {conSize}, 最小线段长度: {minLength}";

            var allLines = new List<LineIdentified>();
            List<Texture2D> LocateTexTemp = new List<Texture2D>();
            List<Matrix> ScreenLocateMatList = new List<Matrix>();
            foreach (var batch in PixelsMultipleBatches.Index())
            {
                var locateTex = ToLocateTex(PixelsMultipleBatches[batch]);
                LocateTexTemp.Add(locateTex);
                var ScreenLocateMat = locateTex.Too0Mat();        // 用于获取Lines的Matrix
                var lineCount = ZIMIdentifyQuadLSD(ref allLines, batch, ScreenLocateMat.zimIdentifyEdgeGradientAny(conSize), minLength);
                log += $"\r\n识别图片{batch}, 识别到的线段数量为: {lineCount}";
                ScreenLocateMatList.Add(ScreenLocateMat);
            }
            Texture2D ScreenLocateTexture = LocateTexTemp[0];       // for output

            // 如果有手动数据，刷新一下Size
            QuadManual?.ReSize(new Vector(Size.x, Size.y), ScreenMap.ViewAspectRatioSetting);
            // 估算屏幕中点，如果已有手动定位数据，根据现有数据取平均即可，否则从色差计算，ScreenLocateMatList[0]默认是屏幕的黑白色差
            Vector AvgPoint = QuadManual != null ? QuadManual.Quad.Centroid : GetAvgPoint(ScreenLocateMatList[0]);
            // 过滤得到四边形的四条边,
            var (quadLinesSemiAuto, quadLinesAuto) = FilterLines(ScreenLocateMatList, allLines, AvgPoint,
                out LineIdentified[] manualLines, out List<LineIdentified> possibleLines,
                conSize, conSize, minLength);

            #region 全自动识别的结果
            List<LineIdentified> LineIdentifiedAuto = new List<LineIdentified>();               // 线段顺序: 下、右、上、左
            for (int i = 0; i < 4; i++)
            {
                if (quadLinesAuto[i] != null)
                    LineIdentifiedAuto.Add(quadLinesAuto[i]);
            }
            if (LineIdentifiedAuto.Count == 4)            // 判断识别的线段能否拼成屏幕，能拼成则记录
            {
                var a = LineIdentifiedAuto[0].Line.Intersect(LineIdentifiedAuto[3].Line, false).Value;
                var b = LineIdentifiedAuto[0].Line.Intersect(LineIdentifiedAuto[1].Line, false).Value;
                var c = LineIdentifiedAuto[2].Line.Intersect(LineIdentifiedAuto[3].Line, false).Value;
                var d = LineIdentifiedAuto[1].Line.Intersect(LineIdentifiedAuto[2].Line, false).Value;
                QuadAuto = new QuadrilateralInCamera(a, b, c, d, new Vector(Size.x, Size.y));
                if (!QuadAuto.IsQuadComplete())
                    QuadAuto = null;
            }
            #endregion

            #region 半自动识别
            List<LineIdentified> LineIdentifiedSemiAuto = new List<LineIdentified>();               // 线段顺序: 下、右、上、左
            LastQuadSemiAutoState = new bool[4] { true, true, true, true };
            for (int i = 0; i < 4; i++)
            {
                if (quadLinesSemiAuto[i] != null)
                    LineIdentifiedSemiAuto.Add(quadLinesSemiAuto[i]);
                else if (manualLines != null)
                {
                    LineIdentifiedSemiAuto.Add(manualLines[i]);
                    LastQuadSemiAutoState[i] = false;
                }
            }

            if (LineIdentifiedSemiAuto.Count == 4)            // 判断识别的线段能否拼成屏幕，能拼成则记录
            {
                var a = LineIdentifiedSemiAuto[0].Line.Intersect(LineIdentifiedSemiAuto[3].Line, false).Value;
                var b = LineIdentifiedSemiAuto[0].Line.Intersect(LineIdentifiedSemiAuto[1].Line, false).Value;
                var c = LineIdentifiedSemiAuto[2].Line.Intersect(LineIdentifiedSemiAuto[3].Line, false).Value;
                var d = LineIdentifiedSemiAuto[1].Line.Intersect(LineIdentifiedSemiAuto[2].Line, false).Value;
                QuadSemiAuto = new QuadrilateralInCamera(a, b, c, d, new Vector(Size.x, Size.y));
                if (!QuadSemiAuto.IsQuadComplete())
                    QuadSemiAuto = null;
            }
            #endregion

            // 优先应用半自动的结果（也可以在外部手动设置）
            if (QuadSemiAuto == null && QuadAuto == null && Screen.QuadInCamera != null)     // 如果可能，回退到上一个screen
            {
                Debug.Log($"<color=yellow>[ScreenIdentification] 本次识别失败，回退到上次的识别结果: {Screen.QuadInCamera}</color>");
                quadTemp.Add(Screen.QuadInCamera);
            }
            else if (QuadSemiAuto != null)
            {
                Debug.Log($"<color=aqua>[ScreenIdentification] 识别到四边形(半自动): {QuadSemiAuto}</color>");
                quadTemp.Add(QuadSemiAuto);
            }
            else if (QuadAuto != null)
            {
                Debug.Log($"<color=aqua>[ScreenIdentification] 识别到四边形(全自动): {QuadAuto}</color>");
                quadTemp.Add(QuadAuto);
            }

            #region 绘制 output texture
            // 绘制半自动
            var ScreenQuadMap = new Matrix(Size, Tiling: true);     // 识别的到的屏幕四边形（半自动和自动在一张图上） 
            foreach (var i in LineIdentifiedSemiAuto.Index())
            {
                if (LastQuadSemiAutoState[i])
                    o0Extension.DrawLine(ScreenQuadMap, LineIdentifiedSemiAuto[i].DrawLine, (x, y) => 2, new Geometry2D.Float.Vector(0, 10));
                else
                    o0Extension.DrawLine(ScreenQuadMap, LineIdentifiedSemiAuto[i].DrawLine, (x, y) => 1, new Geometry2D.Float.Vector(0, 6), true);
            }

            // 绘制全自动
            foreach (var i in LineIdentifiedAuto.Index())
                o0Extension.DrawLine(ScreenQuadMap, LineIdentifiedAuto[i].DrawLine, (x, y) => 4, new Geometry2D.Float.Vector(0, 4), true);

            Texture2D ScreenQuad = ScreenQuadMap.ToTexRGBA(FloatValueToColor);
            Texture2D ScreenQuadWithScreen = ScreenQuad.Overlay(ScreenLocateTexture);   // 叠加屏幕色差图

            // 绘制allLines
            var allLinesMap = new Matrix(Size, Tiling: true);
            foreach (var l in allLines)
            {
                if (l.DrawLine != null)
                    o0Extension.DrawLine(allLinesMap, l.DrawLine, (x, y) => 3, new Geometry2D.Float.Vector(0, 2), true);
            }
            var allLinesTex = allLinesMap.ToTexRGBA(FloatValueToColor);
            ScreenLocate.DebugTexture(1, allLinesTex);

            // 还需要输出一张识别结果图，包含干扰线段
            var ChoosableLineMap = new Matrix(Size, Tiling: true);
            foreach (var l in possibleLines)
            {
                if (l != null && !quadLinesSemiAuto.Contains(l) && !manualLines.Contains(l))
                    o0Extension.DrawLine(ChoosableLineMap, l.DrawLine, (x, y) => 3, new Geometry2D.Float.Vector(0, 2), true); // 其他的备选线段
            }
            foreach (var l in LineIdentifiedSemiAuto)
            {
                if (l != null)
                    o0Extension.DrawLine(ChoosableLineMap, l.DrawLine, (x, y) => 2, new Geometry2D.Float.Vector(0, 5)); // 识别的结果
            }
            if (manualLines != null)
            {
                foreach (var l in manualLines)
                    o0Extension.DrawLine(ChoosableLineMap, l.DrawLine, (x, y) => 1, new Geometry2D.Float.Vector(0, 2), true);     // 旧的屏幕线段(例如上次手动识别的)
            }
            Texture2D ChoosableLineTex = ChoosableLineMap.ToTexRGBA(FloatValueToColor);
            #endregion

            log += $"\r\n屏幕四边形_手动识别{QuadManual != null}\r\n屏幕四边形_半自动识别{QuadSemiAuto != null}\r\n屏幕四边形_全自动识别{QuadAuto != null}";
            Debug.Log(log);
            // 是否将图片保存到本地
            if (ScreenLocate.Main.SaveToggle?.isOn ?? false && ScreenLocate.Main.DebugOnZIMDemo)
            {
                var FileDirectory = $"Debug_屏幕定位/";
                SaveImages(FileDirectory, log, ScreenLocateTexture, allLinesTex, ChoosableLineTex, ScreenQuad);
            }

            //times.Add(watch.ElapsedMilliseconds);
            //Debug.Log("time: " + (times[times.Count - 1] - times[times.Count - 2]));

            // opecncv处理, zim
            {
                //var cvLines = edge.cvHoughLinesP();
                //ScreenLocate.DebugTexture(5, cvLines);

                //var myLines = Hough.Transform(edgeMat);
                //var cvLines = edge.cvLine(myLines);
                //ScreenLocate.DebugTexture(5, cvLines);
            }

            {
                ScreenLocate.DebugTexture(2, ScreenLocateTexture);
                ScreenLocate.DebugTexture(3, ScreenQuad);
                ScreenLocate.DebugTexture(4, ScreenQuadWithScreen);
                ScreenLocate.DebugTexture(5, ChoosableLineTex);
            }
            foreach (var i in LocateTexTemp)
            {
                if (i != ScreenLocateTexture)       // ScreenLocateTexture 由 ScreenLocate.DebugTexture 释放
                    GameObject.Destroy(i);
            }
        }

        Vector GetAvgPoint(Matrix screenLocateMat)
        {
            // 加权平均
            Vector[] avgPointsColumn = new Vector[screenLocateMat.Size.x];
            float[] valueSumsColumn = new float[screenLocateMat.Size.x];
            Parallel.For(0, screenLocateMat.Size.x, i =>
            {
                for (int j = 0; j < screenLocateMat.Size.y; j++)
                {
                    var value = screenLocateMat[i, j];
                    valueSumsColumn[i] += value;
                    avgPointsColumn[i] += new Vector(i, j) * value;
                }
            });

            Vector avgPoint = Vector.Zero;
            var valueSum = 0f;
            for (int i = 0; i < screenLocateMat.Size.x; i++)
            {
                avgPoint += avgPointsColumn[i];
                valueSum += valueSumsColumn[i];
            }
            avgPoint /= valueSum;
            return avgPoint;
        }

        // 返回查找到的线段数量，0是查找失败
        int ZIMIdentifyQuadLSD(ref List<LineIdentified> allLines, int batch, (Matrix edgeMat, Matrix edgeDirMat) edgeGradient,
            float minLength = 100)
        {
            var l = edgeGradient.edgeMat.IdentifyLineLSD(edgeGradient.edgeDirMat, minLength, 50, LineCaptureSize: new Vector(10, 6));
            if (l == null || l.Count == 0)
                return 0;
            allLines.AddRange(l.Select((i) => new LineIdentified(batch, i)));
            return l.Count;
        }

        // 返回四边形的四条边(半自动、全自动)，List长度一定是4 (如果没有识别到就是null)，且线段顺序是: 下、右、上、左
        (List<LineIdentified>, List<LineIdentified>) FilterLines(List<Matrix> screenLocateMatList, List<LineIdentified> allLines, Vector avgPoint,
            out LineIdentified[] manualLines, out List<LineIdentified> possibleLines, float conSize, float gradientLength, float minLength = 100)
        {
            //Debug.Log("[IdentifyLineLSD] lines.Count: " + lines.Count);
            var offset = new Vector((conSize - 1) / 2, (conSize - 1) / 2);
            // LSD计算得到的矩阵尺寸较小（因为卷积），这里必须进行位移
            for (int i = 0; i < allLines.Count; i++)
                allLines[i].Offset(offset);

            // 筛掉椭圆框外的线段（超出一半会筛掉）
            var innerLines = new List<LineIdentified>();
            for (int i = 0; i < allLines.Count; i++)
            {
                List<Vector> InArea = new List<Vector>();
                var dir = (allLines[i].Line.B - allLines[i].Line.A) / 4;
                var points = new Vector[5] { allLines[i].Line.A, allLines[i].Line.A + dir, allLines[i].Line.A + dir * 2f, allLines[i].Line.A + dir * 3f, allLines[i].Line.B };  // A点、中间的点、B点
                for (int pI = 0; pI < points.Length; pI++)
                {
                    if (!ScreenLocate.Main.ScreenPixelCheaker.OutArea2D(points[pI], Size))
                        InArea.Add(points[pI]);
                }

                if (InArea.Count < 2)   // 少于2个点在内部
                    continue;
                else if (InArea.Count < points.Length)  // 不完全在内部
                    allLines[i].DrawLine = new Line(InArea.First(), InArea.Last());        // 将部分线条设置为drawline，用于下一步的绘制
                else   // 线段全部在椭圆内
                    allLines[i].DrawLine = allLines[i].Line;

                innerLines.Add(allLines[i]);
            }

            // 角度阈值，用来判断线段的梯度方向是否指向屏幕中心(avgPoint)
            var avaAngleHalf = 75f;

            // 沿直线计算综合梯度（梯度乘以长度系数，再乘以距离系数）, distanceRatio是实际距离除以最大距离
            float estimateGradient(LineIdentified line, float distanceRatio)
            {
                var dir = (line.Line.B - line.Line.A).Normalized;
                var vertical = new Vector(-dir.y, dir.x) * (gradientLength / 2);
                var step = 2;
                var ll = line.Line.Length;
                var lg = new List<float>();
                for (int i = 0; i <= ll; i += step)
                {
                    var point = line.Line.A + dir * i;
                    var ga = point + vertical;
                    var gb = point - vertical;
                    lg.Add(screenLocateMatList[line.Batch][(int)ga.x, (int)ga.y] - screenLocateMatList[line.Batch][(int)gb.x, (int)gb.y]);
                }

                float e = (float)Math.Sqrt(Math.Ceiling(line.Line.Length / minLength));       // 长度系数，筛选时梯度更大、长度更长的线段更优
                float d = (3 - distanceRatio) / 2;            // 距离系数，距离越近，系数越大
                return e * d * Math.Abs(lg.Mean());
            }
            // 根据线段梯度的角度，判断是不是屏幕的边，out index代表是哪条边(顺序是: 下、右、上、左)
            bool isScreenLine(LineIdentified line, out int index)
            {
                var a = (avgPoint - (line.Line.A + line.Line.B) / 2).DegreeToXAxis();
                //Debug.Log(a + ", " + gradient + ", " + sum);
                index = -1;
                if (Math.Abs(a - line.GradientDegree) < avaAngleHalf || Math.Abs(a - 360 - line.GradientDegree) < avaAngleHalf || Math.Abs(a + 360 - line.GradientDegree) < avaAngleHalf)
                {
                    if (line.GradientDegree > 45 && line.GradientDegree < 135)     // 下
                        index = 0;
                    else if (line.GradientDegree > 135 && line.GradientDegree < 225) // 右
                        index = 1;
                    else if (line.GradientDegree > 225 && line.GradientDegree < 315)  // 上
                        index = 2;
                    else
                        index = 3;
                    return true;
                }
                return false;
            }


            // 下、右、上、左, 半自动和自动
            var quadLinesSemiAuto = new List<(float, LineIdentified)>[4] { new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>() };
            var quadLinesAuto = new List<(float, LineIdentified)>[4] { new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>() };
            possibleLines = new List<LineIdentified>();

            // 如果已有手动定位数据，根据现有数据筛选线条（半自动）
            manualLines = null;
            if (QuadManual != null)
            {
                Debug.Log($"[IdentifyLineLSD] 根据已有定位数据做筛选, QuadManual: {QuadManual}");
                manualLines = QuadManual.GetLines().Select((i) => new LineIdentified(0, i, 0, 0, true)).ToArray();
                var calibration = ScreenLocate.Main.ReDoLocateCalibrationRatio * Size.y;
                var avgPointCross = manualLines.Select((i) => i.Line.LineCrossWithPoint(avgPoint)).ToArray();     // 对于平均点的corss值
                var avgPointPedal = manualLines.Select((i) => o0Extension.PointPedal(i.Line, avgPoint, out _)).ToArray();     // 当前定位的垂足，下、右、上、左

                foreach (var line in innerLines)
                {
                    // 筛选条件：1-梯度方向匹配，2-垂足的距离足够近, 3-线段的AB点均在旧线段外部, 4-新的线段的中点，到旧线段的垂足，要在旧线段内
                    if (isScreenLine(line, out int index))
                    {
                        var distanceToOld = (o0Extension.PointPedal(line.Line, avgPoint, out _) - avgPointPedal[index]).Length;
                        if (distanceToOld < calibration &&
                            manualLines[index].Line.LineCrossWithPoint(line.Line.A) * avgPointCross[index] <= 0 &&
                            manualLines[index].Line.LineCrossWithPoint(line.Line.B) * avgPointCross[index] <= 0)
                        {
                            var middleToOldLine = o0Extension.PointPedal(manualLines[index].Line, (line.Line.A + line.Line.B) / 2, out bool inLineSegment);
                            if (inLineSegment)
                            {
                                quadLinesSemiAuto[index].Add((estimateGradient(line, distanceToOld / calibration), line));
                                possibleLines.Add(line);
                            }
                        }
                    }
                }
            }

            // 全自动
            foreach (var line in allLines)
            {
                if (isScreenLine(line, out int index))
                {
                    if (line.Batch < 1)     // 全自动只处理第一张图，默认是色差图
                    {
                        quadLinesAuto[index].Add((estimateGradient(line, 1), line));
                    }
                }
            }

            var resultSemiAuto = new LineIdentified[4];
            var resultAuto = new LineIdentified[4];
            for (int i = 0; i < 4; i++)
            {
                if (quadLinesSemiAuto[i].Count > 0)
                    resultSemiAuto[i] = quadLinesSemiAuto[i].Max((a, b) => a.Item1.CompareTo(b.Item1)).Item2;
                if (quadLinesAuto[i].Count > 0)
                    resultAuto[i] = quadLinesAuto[i].Max((a, b) => a.Item1.CompareTo(b.Item1)).Item2;
            }
            return (resultSemiAuto.ToList(), resultAuto.ToList());
        }

        void SaveImages(string FileDirectory, string log,
            Texture2D ScreenLocateTex, Texture2D allLinesTex, Texture2D ChoosableLineTex, Texture2D ScreenQuadTex)
        {
            if (!Directory.Exists(FileDirectory))
                Directory.CreateDirectory(FileDirectory);
            var time = DateTime.Now.ToString("yyyyMMdd_HHmmss");

            var pngData = ScreenLocate.Main.OutputTextures[7]?.EncodeToPNG();
            if (pngData != null)
                File.WriteAllBytes($"{FileDirectory}{time}A屏幕原图.png", pngData);
            var pngData1 = ScreenLocateTex?.EncodeToPNG();
            if (pngData1 != null)
                File.WriteAllBytes($"{FileDirectory}{time}B黑白色差.png", pngData1);
            var pngData2 = allLinesTex?.EncodeToPNG();
            if (pngData2 != null)
                File.WriteAllBytes($"{FileDirectory}{time}C全部识别线段_半自动.png", pngData2);
            var pngData3 = ChoosableLineTex?.EncodeToPNG();
            if (pngData3 != null)
                File.WriteAllBytes($"{FileDirectory}{time}D备选线段_半自动.png", pngData3);
            var pngData4 = ScreenQuadTex?.EncodeToPNG();
            if (pngData4 != null)
                File.WriteAllBytes($"{FileDirectory}{time}E识别结果.png", pngData4);


            Debug.Log($"<color=aqua>({time}) 屏幕识别图片保存至：程序根目录/{FileDirectory}</color>");
            log +=
                $"\r\n屏幕原图保存{pngData != null}, " +
                $"\r\n黑白色差保存{pngData1 != null}, " +
                $"\r\n全部识别线段（半自动）保存{pngData2 != null}, " +
                $"\r\n备选线段（半自动）保存{pngData3 != null}, " +
                $"\r\n识别结果保存{pngData4 != null}";
            File.WriteAllText($"{FileDirectory}{time}屏幕自动定位_日志.log", log);
        }

    }
}