smart-bow-infrared/Assets/InfraredProject/WebCamera/Script/ZIM/InfraredLocate/ScreenIdentification.cs

#define ENABLE_LOG

using o0.Geometry2D.Float;
using o0.Num;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using UnityEngine;
using ZIM;
using ZIM.Unity;

namespace o0.Project
{
    public partial class ScreenIdentification
    {
        private const string TAG = "ScreenIdentification#";

        //static Rect[][] LocateAreaData = new Rect[][] {
        //     new Rect[] { new Rect(0f, 0f, 0.3f, 0.3f),     new Rect(0f, 0f, 0.4f, 0.4f),     new Rect(0f, 0f, 0.5f, 0.5f),     new Rect(0f, 0f, 0.6f, 0.6f) },
        //     new Rect[] { new Rect(0.7f, 0f, 0.3f, 0.3f),   new Rect(0.6f, 0f, 0.4f, 0.4f),   new Rect(0.5f, 0f, 0.5f, 0.5f),   new Rect(0.4f, 0f, 0.6f, 0.6f) },
        //     new Rect[] { new Rect(0f, 0.7f, 0.3f, 0.3f),   new Rect(0f, 0.6f, 0.4f, 0.4f),   new Rect(0f, 0.5f, 0.5f, 0.5f),   new Rect(0f, 0.4f, 0.6f, 0.6f) },
        //     new Rect[] { new Rect(0.7f, 0.7f, 0.3f, 0.3f), new Rect(0.6f, 0.6f, 0.4f, 0.4f), new Rect(0.5f, 0.5f, 0.5f, 0.5f), new Rect(0.4f, 0.4f, 0.6f, 0.6f) }
        //};
        static Rect[][] LocateAreaData = new Rect[][] {
             new Rect[] { new Rect(0f, 0f, 0.3f, 0.3f),     new Rect(0f, 0f, 0.4f, 0.4f),     new Rect(0f, 0f, 0.5f, 0.5f)     },
             new Rect[] { new Rect(0.7f, 0f, 0.3f, 0.3f),   new Rect(0.6f, 0f, 0.4f, 0.4f),   new Rect(0.5f, 0f, 0.5f, 0.5f)   },
             new Rect[] { new Rect(0f, 0.7f, 0.3f, 0.3f),   new Rect(0f, 0.6f, 0.4f, 0.4f),   new Rect(0f, 0.5f, 0.5f, 0.5f)   },
             new Rect[] { new Rect(0.7f, 0.7f, 0.3f, 0.3f), new Rect(0.6f, 0.6f, 0.4f, 0.4f), new Rect(0.5f, 0.5f, 0.5f, 0.5f) }
        };
        //static bool LocateDebug = false;
        static bool LocateDebug = true;

        public Geometry2D.Vector<int> Size => ScreenLocate.Main.CameraSize;

        public ScreenMap Screen;       // 识别到的屏幕，用于执行透视变换

        int capture = 0;
        int delay = 0;
        int maxCapture;
        int maxDelay;


        Geometry.Vector<float>[] ScreenBlackTexture;
        Geometry.Vector<float>[] ScreenWhiteTexture;
        int locateIndex = -1;
        List<Rect> locateArea = new List<Rect> {
            new Rect(0f, 0f, 0.5f, 0.5f), new Rect(0.5f, 0f, 0.5f, 0.5f), new Rect(0f, 0.5f, 0.5f, 0.5f), new Rect(0.5f, 0.5f, 0.5f, 0.5f)
        };      // 屏幕显示白色的区域大小

        float areaPercent => locateArea[locateIndex].size.x;         // 当前白色区域的占比
        int areaSelected = -1;                                       // 选择哪个区域，顺序与Quadrilateral对应
        List<float> sumTemp = new List<float>();
        List<OrdinalQuadrilateral> quadTemp = new List<OrdinalQuadrilateral>();

        //public ScreenIdentification(WebCamTexture texture)
        //{
        //    Size = new Geometry2D.Vector<int>(texture.width, texture.height);
        //    Screen = new ScreenMap();
        //}
        public static UnityEngine.Color FloatValueToColor(float i)
        {
            switch (i)
            {
                case 1:
                    return UnityEngine.Color.green;
                case 2:
                    return UnityEngine.Color.red;
                case 3:
                    return UnityEngine.Color.yellow;
                default:
                    return UnityEngine.Color.black;
            }
        }

        public ScreenIdentification()
        {
            Screen = new ScreenMap();
            OnLocateScreenEnter += () => Application.targetFrameRate = 30;      // 固定识别的帧率，确保摄像机拍到正确的画面
            OnLocateScreenEnd += () => Application.targetFrameRate = 60;
        }

        public void SetScreenQuad(QuadrilateralInCamera quad) => Screen.QuadInCamera = quad;

        public event Action OnLocateScreenEnter;
        public event Action OnLocateScreenEnd;
        public bool bStartLocateScreen { get; set; } = false;//是否进行捕获

        // 自动识别开始的入口
        public void LocateScreen(int Capture = 30, int Delay = 30)  //数值单位是frame
        {
            if (ScreenLocate.Main.DebugScreenImages.Count != 0 && ScreenLocate.Main.DebugOnZIMDemo)     // 这段仅用于测试图片
            {
                ScreenLocate.Main.CameraSize = new Geometry2D.Vector<int>(ScreenLocate.Main.DebugScreenImages[0].width, ScreenLocate.Main.DebugScreenImages[0].height);
                DebugImage(ScreenLocate.Main.DebugScreenImages);
                Screen.QuadInCamera = new QuadrilateralInCamera(quadTemp[0], new Vector(ScreenLocate.Main.DebugScreenImages[0].width, ScreenLocate.Main.DebugScreenImages[0].height));
                ScreenLocate.SetScreen(null);
                ScreenLocate.Main.ShowScreen(ScreenLocate.Main.ScreenQuad, Screen.QuadInCamera);
                delay = 0;
                capture = 0;
                ScreenWhiteTexture = null;
                ScreenBlackTexture = null;
                locateIndex = -1;
                areaSelected = -1;
                quadTemp.Clear();
                sumTemp.Clear();
                ScreenLocate.Main.DebugScreenImages.Clear();
                return;
            }

            delay = Math.Max(Delay, 5);
            capture = Math.Max(Capture, 5);
            maxDelay = Delay;
            maxCapture = Capture;

            ScreenLocate.SetScreen(new Rect(0f, 0f, 1f, 1f), UnityEngine.Color.black);
            //ScreenLocate.SetScreen(new Rect(0f, 0f, 0.6f, 0.6f), UnityEngine.Color.white);

            //bStartLocateScreen = false;

            OnLocateScreenEnter?.Invoke();
        }
        /// <summary>
        /// 开始进行捕获
        /// 初始化了两个数据 capture 和 delay
        /// </summary>
        /// <returns></returns>
        public bool isInitLocateScreen() {
            return capture != 0 && delay != 0;
        }

        void DebugImage(List<Texture2D> images)
        {
            QuadrilateralFit(images, 5);

            //var watch = new System.Diagnostics.Stopwatch();
            //watch.Start();
            //var times = new List<double>() { 0.0 };
#if (!NDEBUG && DEBUG && ENABLE_LOG)
            Console.WriteLine($"{TAG} quadTemp.Count:{quadTemp.Count}");
#endif
            if (quadTemp.Count > 0)
            {
                var quad = quadTemp[0];
                ScreenLocate.Main.ShowScreen(ScreenLocate.Main.outputRawImages[4].transform.GetChild(0) as RectTransform,
                    new QuadrilateralInCamera(quad, images[0].Size().o0Vector()));

                // 透视变换
//                var srcWidth = LocateLightedRedTex.width;
//                var transformWidth = (int)((quad.B.x - quad.A.x + quad.D.x - quad.C.x) / 2);
//                var transformHeight = (int)((quad.C.y - quad.A.y + quad.D.y - quad.B.y) / 2);
//                var transformTex = new Texture2D(transformWidth, transformHeight);
//                var pt = new ZIMPerspectiveTransform(new OrdinalQuadrilateral(new Vector(0, 0), new Vector(transformWidth, 0), new Vector(0, transformHeight), new Vector(transformWidth, transformHeight)), quad);
//                var dstPixel = new UnityEngine.Color[transformWidth * transformHeight];
//                var srcPixel = LocateLightedRedTex.GetPixels();
//                Parallel.For(0, transformWidth, (x) =>
//                {
//                    for (int y = 0; y < transformHeight; y++)
//                    {
//                        var index = y * transformWidth + x;
//                        var sampleCoord = pt.TransformRound(x, y);
//                        dstPixel[index] = srcPixel[sampleCoord.y * srcWidth + sampleCoord.x];
//                    }
//                });
//                transformTex.SetPixels(dstPixel);
//                transformTex.Apply();
//                //ScreenLocate.DebugTexture(1, transformTex);
//#if (!NDEBUG && DEBUG && ENABLE_LOG)
//                Console.WriteLine($"{TAG} ScreenLocate.DebugTexture 1:{transformTex.GetNativeTexturePtr()}");
//#endif
            }

            //times.Add(watch.ElapsedMilliseconds);
            //UnityEngine.Debug.Log("time: " + (times[times.Count - 1] - times[times.Count - 2]));
        }

        public void NextScreen()
        {
            // 测试用
            if (LocateDebug && areaSelected == -1)
            {
                LocateAreaData = new Rect[][] { new Rect[] { new Rect(0, 0, 1f, 1f) } };
                locateIndex = 3;
                areaSelected = 0;
                locateArea.AddRange(LocateAreaData[0]);
            }

            // index从-1开始
            locateIndex++;
            if (locateIndex < locateArea.Count)     // 依次点亮屏幕区域
            {
                ScreenLocate.SetScreen(locateArea[locateIndex], UnityEngine.Color.white);
                delay = maxDelay;
                capture = maxCapture;
            }
            else                // 退出屏幕黑白控制
            {
                ScreenLocate.SetScreen(null);
                ScreenLocate.Main.ShowScreen(ScreenLocate.Main.ScreenQuad, Screen.QuadInCamera);
                Reset();
            }
        }

        void Reset()
        {
            // bStartLocateScreen = false;
            delay = 0;
            capture = 0;
            ScreenWhiteTexture = null;
            ScreenBlackTexture = null;
            locateIndex = -1;
            areaSelected = -1;
            locateArea.RemoveRange(4, LocateAreaData[0].Length);
            quadTemp.Clear();
            sumTemp.Clear();
        }

        public void CaptureBlack(Texture2D cam)
        {
            if (ScreenBlackTexture == null)
                ScreenBlackTexture = new Geometry.Vector<float>[Size.x * Size.y];
            var pixel = cam.GetPixels();
            Parallel.For(0, Size.x * Size.y, i =>
            {
                var ip = pixel[i];
                ScreenBlackTexture[i] += new Geometry.Vector<float>(ip.r / maxCapture, ip.g / maxCapture, ip.b / maxCapture);
            });
        }

        public void CaptureWhite(Texture2D cam)
        {
            if (ScreenWhiteTexture == null)
                ScreenWhiteTexture = new Geometry.Vector<float>[Size.x * Size.y];
            var pixel = cam.GetPixels();
            Parallel.For(0, Size.x * Size.y, i =>
            {
                var ip = pixel[i];
                ScreenWhiteTexture[i] += new Geometry.Vector<float>(ip.r / maxCapture, ip.g / maxCapture, ip.b / maxCapture);
            });
        }

        public void CaptureStay(Texture2D cam)
        {
            if (locateIndex == -1)                      // 屏幕黑色
            {
                CaptureBlack(cam);
            }
            else                                        // 屏幕部分为白色
            {
                CaptureWhite(cam);
            }
        }

        public void CaptureEnd()
        {
            //Debug.Log("locateIndex: " + locateIndex + ", quad: " + quadTemp.Count);
            if (locateIndex == -1)
                return;

            if (locateIndex < 4)
            {
                sumTemp.Add(GetBrightness());
                ScreenWhiteTexture = null;

                // 选择亮度差最大的区域
                if (locateIndex == 3)
                {
                    areaSelected = sumTemp.MaxIndex();
                    locateArea.AddRange(LocateAreaData[areaSelected]);
                }
            }
            else if (locateIndex >= 4 && locateIndex < locateArea.Count - 1)
            {
                QuadrilateralFit();
                ScreenWhiteTexture = null;
            }
            else
            {
                QuadrilateralFit();

                if (quadTemp.Count != LocateAreaData[0].Length)
                {
                    Debug.Log($"<color=yellow>[ScreenIdentification] 拟合四边形失败, quadTemp.Count: {quadTemp.Count}</color>");
                }
                else if (quadTemp.Count == 1)
                {
                    Screen.QuadInCamera = new QuadrilateralInCamera(quadTemp[0], new Vector(Size.x, Size.y));
                    Debug.Log($"<color=aqua>[ScreenIdentification] 拟合成功，Quad: {Screen.QuadInCamera.QuadString}____{Screen.QuadInCamera.SizeString}</color>");
                }
                else
                {
                    // Debug.Log($"拟合四边形 2 , quadTemp.Count: {quadTemp.Count}");
                    // 线性拟合
                    var xValue = new List<float>() { 0 };
                    var predicts = new List<Vector>();
                    foreach (var i in LocateAreaData[0])
                        xValue.Add(i.size.x);

                    Vector baseVertex = Vector.Zero;            // x==0 时的点
                    {
                        foreach (var q in quadTemp)
                        {
                            baseVertex += q[areaSelected];
                        }
                        baseVertex /= quadTemp.Count;
                    }
                    double rs = 0.0;
                    for (int i = 0; i < 4; i++)
                    {
                        if (i == areaSelected)
                        {
                            predicts.Add(baseVertex);
                        }
                        else
                        {
                            var yValue = new List<Vector>() { baseVertex };
                            foreach (var q in quadTemp)
                            {
                                yValue.Add(q[i]);
                            }
                            var lr = LinerRegression1D.Fit(2, xValue.ToArray(), yValue.ToArray());
                            rs += lr.RSquared / 3;
                            predicts.Add(lr.Predict<Vector>(1));
                        }
                    }
                    Screen.QuadInCamera = new QuadrilateralInCamera(predicts, new Vector(Size.x, Size.y));
                    Debug.Log($"<color=aqua>[ScreenIdentification] 拟合成功，RSquared: {rs}, Quad: {Screen.QuadInCamera.QuadString}____{Screen.QuadInCamera.SizeString}</color>");
                    //if (rs < 0.8) Screen.Quad = null;
                }

                OnLocateScreenEnd?.Invoke();
            }

        }

        public bool Update(Texture2D cam)
        {
            //if (!bStartLocateScreen) return false;

            if (delay != 0)
            {
                //ScreenLocate.Main.CreateUVCTexture2DFocusSizeIfNeeded(1280, 720);
                delay--;
                if (delay == 0)
                {
                    ScreenLocate.Main.CameraSize = new Geometry2D.Vector<int>(cam.width, cam.height);          // 记录当前的分辨率
                    Debug.Log("[ScreenIdentification] 采样纹理，分辨率: [" + Size.x + ", " + Size.y + "]");
                }
                return true;
            }
            if (capture != 0)
            {
                //ScreenLocate.Main.CreateUVCTexture2DFocusSizeIfNeeded(1280, 720);
                CaptureStay(cam);

                capture--;
                if (capture == 0)
                {
                    CaptureEnd();
                    NextScreen();
                }
                return true;
            }

            return false;

            #region Old
            /*


    if (delay != 0)
    {
        delay--;
        return true;
    }
    if (capture != 0)
    {
        capture--;
        if (ScreenBlackTexture == null)
            ScreenBlackTexture = new Geometry.Vector<float>[Size.x * Size.y];
        var pixel = cam.GetPixels();
        Parallel.For(0, Size.x * Size.y, i =>
        {
            var ip = pixel[i];
            ScreenBlackTexture[i] += new Geometry.Vector<float>(ip.r, ip.g, ip.b);
        });
        if (capture == 0)
            ScreenLocate.SetScreen(UnityEngine.Color.black);
        return true;
    }

    if (delay != 0)
    {
        delay--;
        return true;
    }
    if (capture != 0)
    {
        capture--;
        if (ScreenWhiteTexture == null)
            ScreenWhiteTexture = new Geometry.Vector<float>[Size.x * Size.y];
        var pixel = cam.GetPixels();
        Parallel.For(0, Size.x * Size.y, i =>
        {
            var ip = pixel[i];
            ScreenWhiteTexture[i] += new Geometry.Vector<float>(ip.r, ip.g, ip.b);
        });
        if (capture == 0)
            ScreenLocate.SetScreen(UnityEngine.Color.black);
        return true;
    }
    if (delay != 0)
    {
        delay--;
        return true;
    }
    if (capture != 0)
    {
        capture--;
        var pixel = cam.GetPixels();
        Parallel.For(0, Size.x * Size.y, i =>
        {
            var ip = pixel[i];
            ScreenWhiteTexture[i] -= new Geometry.Vector<float>(ip.r, ip.g, ip.b);
        });
        if (capture == 0)
        {
            ScreenLocate.SetScreen(null);
            UnityEngine.Color[] newPixel = new UnityEngine.Color[Size.x * Size.y];

            Parallel.For(0, Size.x * Size.y, i => {
                var pi = ScreenWhiteTexture[i] /= capture;
                newPixel[i] = new UnityEngine.Color(pi.x, pi.y, pi.z);
            });

            //读取数据
            //{
            //    var fileName = "3.bin";
            //    ScreenLocateTexture = $"2023 04 16 厦门测试数据/{fileName}".FileReadByte<Vector<float>[]>();
            //    Debug.Log($"Read {fileName}");
            //    Parallel.For(0, Size.x * Size.y, i =>
            //    {
            //        var pi = ScreenLocateTexture[i];
            //        newPixel[i] = new UnityEngine.Color(pi.x, pi.y, pi.z);
            //    });
            //}

            var ScreenLocateTex = new Texture2D(Size.x, Size.y);
            ScreenLocateTex.SetPixels(newPixel);
            ScreenLocateTex.Apply();
            //ScreenLocate.DebugTexture(2, ScreenLocateTex);
            var ScreenLocateTexLighted = ScreenLocateTex.AutoLight(10);

            //ScreenLocate.DebugTexture(2, ScreenLocateTexLighted);
            //var FileSavePath = Application.persistentDataPath + "/ScreenLocateTexture.bin";
            bool Save = ScreenLocate.Main.SaveToggle.isOn;
            string time;
            if (Save)
            {
                time = DateTime.Now.ToString("yyyyMMdd_HHmmss");
                var FileSavePath = $"屏幕定位数据{time}.bin";
                FileSavePath.FileWriteByte(ScreenWhiteTexture);

                var bytes = ScreenLocateTexLighted.EncodeToPNG();
                File.WriteAllBytes($"屏幕定位数据{time}.png", bytes);

                Debug.Log("ScreenLocateTexture Saved To: " + FileSavePath);
            }

            var ScreenLocateTexR = ScreenLocateTexLighted.ToRGB(ColorChannel.Red);
            var ScreenLocateTexG = ScreenLocateTexLighted.ToRGB(ColorChannel.Green);
            var ScreenLocateTexB = ScreenLocateTexLighted.ToRGB(ColorChannel.Blue);
            ScreenLocate.DebugTexture(2, ScreenLocateTexR);
            //ScreenLocate.DebugTexture(4, ScreenLocateTexG);
            //ScreenLocate.DebugTexture(5, ScreenLocateTexB);


            var watch = new System.Diagnostics.Stopwatch();
            watch.Start();
            var times = new List<double>() { 0.0 };



            var ScreenLocateTexLightedMat = ScreenLocateTexLighted.Too0Mat();
            //var ScreenLocateTexLightedMat = texture.Too0Mat();

            //var (edge, edgeDir) = ScreenLocateTexLightedMat.IdentifyEdge();
            var (edge, edgeDir) = ScreenLocateTexLightedMat.zimIdentifyEdgeGradientAny(15);
            //ScreenLocate.DebugTexture(4, ScreenLocateTexLighted.Too0Mat().IdentifyEdgeGradient().ToTex());
            //ScreenLocate.DebugTexture(4, edge.ToTex());

            var quadLines = ScreenLocateTexLightedMat.IdentifyQuadLSD(edge, edgeDir, out List<Line> lightLines, 30);
            var drawLineMap = new MatrixF2D(edge..Size.x, edge.Size.y);
            int lineCount = 0;
            foreach (var l in quadLines)
            {
                if (l != null)
                {
                    o0Extension.DrawLine(drawLineMap.DrawLine(l, (x, y) => 1, new Geometry2D.Float.Vector(0, 10));
                    lineCount++;
                }
            }
            if (lineCount == 4)
            {
                var a = quadLines[0].Intersect(quadLines[3], false).Value;
                var b = quadLines[0].Intersect(quadLines[1], false).Value;
                var c = quadLines[2].Intersect(quadLines[3], false).Value;
                var d = quadLines[1].Intersect(quadLines[2], false).Value;
                Quad = new Quadrilateral(a, b, c, d);
                if (!Quad.IsInScreen(ScreenLocate.Main.WebCamera.Size))
                    Quad = null;
            }
            ScreenLocate.Main.ShowScreen(Quad);

            //var lines = edge.IdentifyLineLSD(edgeDir, 100);
            ////var lines = ScreenLocateTexLightedMat.IdentifyLineLSD();

            //var drawLineMap = new MatrixF2D(edge..Size.x, edge.Size.y);
            //var returnMaxLines = lines.Sub(0, 10);
            //foreach (var (line, sum, gradient) in returnMaxLines)
            //    o0Extension.DrawLine(drawLineMap.DrawLine(line, (x, y) => 1, new Geometry2D.Float.Vector(0, 10));


            ScreenLocate.DebugTexture(3, drawLineMap.ToTex());
            //{
            //    var bytes = drawLineMap.ToTex().EncodeToPNG();
            //    File.WriteAllBytes($"屏幕定位数据DrawLineMap.png", bytes);
            //}


            times.Add(watch.ElapsedMilliseconds);
            UnityEngine.Debug.Log("time: " + (times[times.Count - 1] - times[times.Count - 2]));

            //ScreenLocate.DebugTexture(5, edge.IdentifyLine(edgeDir).ToTex());
            //ScreenLocate.DebugTexture(4, ScreenLocateTexLighted.Too0Mat().IdentifyEdgeGradientX().ToTex());
            //ScreenLocate.DebugTexture(5, ScreenLocateTexLighted.Too0Mat().IdentifyEdgeGradientY().ToTex());

            //var convolutionLighted2 = ScreenLocateTexLighted.Too0Mat().IdentifyEdgeVariance().ToTex();

            // opecncv处理
            // zim
            {
                //var cvLines = edge.cvHoughLinesP();
                //ScreenLocate.DebugTexture(5, cvLines);

                //var myLines = Hough.Transform(edgeMat);
                //var cvLines = edge.cvLine(myLines);
                //ScreenLocate.DebugTexture(5, cvLines);
            }

            UnityEngine.Object.Destroy(ScreenLocateTex);


            //ScreenLocate.DebugTexture(4, convolutionLighted2);
        }

        return true;
    }

    /*
    var avg = new Geometry4D.Vector<float>();
    var pixel = texture.GetPixels();
    foreach(var i in pixel.Index())
    {
        var iP = pixel[i];
        avg += new Geometry4D.Vector<float>(iP.r, iP.g, iP.b, iP.a);
    }
    avg /= pixel.Count();

    /*
    var (texLightedR, texLightedG, texLightedB) = ToRGB(newTex);
    ScreenLocate.DebugTexture(3, texLightedR);
    ScreenLocate.DebugTexture(4, texLightedG);
    ScreenLocate.DebugTexture(5, texLightedB);
    //Debug.Log(avg);

    return false;

/**/
            #endregion


        }

        float GetBrightness()
        {
            UnityEngine.Color[] differPixel = new UnityEngine.Color[Size.x * Size.y];

            Parallel.For(0, Size.x * Size.y, i =>
            {
                var pi = ScreenWhiteTexture[i] - ScreenBlackTexture[i];
                differPixel[i] = new UnityEngine.Color(pi.x, pi.y, pi.z);
            });

            var sum = 0f;
            foreach (var i in differPixel)
            {
                sum += i.Brightness();
            }
            sum /= differPixel.Length;
            //Debug.Log(sum);
            return sum;
        }

        // 转换成屏幕定位所需的纹理图像
        Texture2D ToLocateTex(UnityEngine.Color[] pixels)
        {
            var ScreenLocateTex = new Texture2D(Size.x, Size.y);
            ScreenLocateTex.SetPixels(pixels);
            ScreenLocateTex.Apply();
            //ScreenLocate.DebugTexture(2, ScreenLocateTex);
            return ScreenLocateTex.AutoLight(10);

            //ScreenLocate.DebugTexture(2, ScreenLocateTexLighted);

            //var ScreenLocateTexR = ToLocateTex.ToRGB(ColorChannel.Red);
            //var ScreenLocateTexG = ToLocateTex.ToRGB(ColorChannel.Green);
            //var ScreenLocateTexB = ToLocateTex.ToRGB(ColorChannel.Blue);

            //LocateLightedRedTex = ScreenLocateTexR;
            //ScreenLocate.DebugTexture(2, ScreenLocateTexR);
            //ScreenLocate.DebugTexture(4, ScreenLocateTexG);
            //ScreenLocate.DebugTexture(5, ScreenLocateTexB);


            //var watch = new System.Diagnostics.Stopwatch();
            //watch.Start();
            //var times = new List<double>() { 0.0 };

            //var ScreenLocateTexLightedMat = texture.Too0Mat();
        }

        /// <param name="lineWidth">识别的最小线段长度</param>
        /// <param name="debugImages">这个参数如果不为null且数量大于0，则执行debug操作</param>
        void QuadrilateralFit(List<Texture2D> debugImages = null, float lineWidth = 10)
        {
            // 屏幕黑白差值，存放多批次的图像用于识别, 该List数量不能等于 0 
            List<UnityEngine.Color[]> PixelsMultipleBatches = new List<UnityEngine.Color[]>();

            //读取数据
            if (debugImages != null && debugImages.Count != 0) 
            {
                foreach (var i in debugImages)
                {
                    Debug.Log($"<color=aqua>Debug {i.name}</color>");
                    PixelsMultipleBatches.Add(i.GetPixels());
                }

            }
            else    // 获得屏幕差值
            {
                var maxWhite = 0f;
                foreach (var i in ScreenWhiteTexture)
                {
                    var m = i.x > i.y ? (i.x > i.z ? i.x : i.z) : (i.y > i.z ? i.y : i.z);
                    if (maxWhite < m)
                        maxWhite = m;
                }
                var scale = 1.0f / maxWhite;        // 放大对比度

                var differPixel = new UnityEngine.Color[Size.x * Size.y];
                Parallel.For(0, Size.x * Size.y, i =>
                {
                    var pi = ScreenWhiteTexture[i] - ScreenBlackTexture[i];
                    differPixel[i] = new UnityEngine.Color(pi.x, pi.y, pi.z) * scale;
                });
                PixelsMultipleBatches.Add(differPixel);
                PixelsMultipleBatches.Add(ScreenWhiteTexture.Select((i) => new UnityEngine.Color(i.x, i.y, i.z) * scale).ToArray());
            }

            int conSize = (int)Math.Ceiling(0.007f * Size.y) * 2 + 1;
            conSize = Math.Max(conSize, 7);     // 设置最小为7

            float minLength = conSize * 7.7f;
            minLength = locateIndex == -1 ? minLength : minLength * areaPercent;    // minLength需要按比例缩小
            string log = $"[ScreenLocate Auto] Size: ({Size.x},{Size.y}), 卷积核Size: {conSize}, 最小线段长度: {minLength}";

            var allLines = new List<LineIdentified>();
            List<Texture2D> LocateTexTemp = new List<Texture2D>();
            List<Matrix> ScreenLocateMatList = new List<Matrix>();
            foreach (var batch in PixelsMultipleBatches.Index())
            {
                var locateTex = ToLocateTex(PixelsMultipleBatches[batch]);
                LocateTexTemp.Add(locateTex);
                var ScreenLocateMat = locateTex.Too0Mat();        // 用于获取Lines的Matrix
                var lineCount = ZIMIdentifyQuadLSD(ref allLines, batch, ScreenLocateMat.zimIdentifyEdgeGradientAny(conSize), minLength);
                log += $"\r\n识别图片{batch}, 识别到的线段数量为: {lineCount}";
                ScreenLocateMatList.Add(ScreenLocateMat);
            }
            Texture2D ScreenLocateTexture = LocateTexTemp[0];       // for output

            // 过滤得到四边形的四条边, ScreenLocateMatList[0]默认是屏幕的黑白色差
            var quadLines = FilterLines(ScreenLocateMatList, allLines, GetAvgPoint(ScreenLocateMatList[0]),
                out Line[] oldLines, out List<Line> possibleLines,
                Screen, conSize, conSize, minLength);


            // 将 allLines 输出一张图片
            var allLinesMap = new Matrix(Size, Tiling: true);
            foreach (var l in allLines)
            {
                if (l.Line != null)
                    o0Extension.DrawLine(allLinesMap, l.Line, (x, y) => 3, new Geometry2D.Float.Vector(0, 2), true);
            }
            var allLinesTex = allLinesMap.ToTexRGBA(FloatValueToColor);
            ScreenLocate.DebugTexture(1, allLinesTex);



            // 将识别到的边画出来，并判断能否拼成屏幕，能拼成则设置ScreenMap
            // 线段顺序: 下、右、上、左
            List<Line> LineIdentified = new List<Line>();
            for (int i = 0; i < 4; i++)
            {
                if (quadLines[i] != null)
                    LineIdentified.Add(quadLines[i]);
                else if (oldLines != null)
                    LineIdentified.Add(oldLines[i]);
            }

            var drawScreenMap = new Matrix(Size, Tiling: true);
            foreach (var l in LineIdentified)
                o0Extension.DrawLine(drawScreenMap, l, (x, y) => 1, new Geometry2D.Float.Vector(0, lineWidth));
            Texture2D ScreenQuadTex = drawScreenMap.ToTex();      // out ScreenQuadTex

            QuadrilateralInCamera screenQuad = null;
            if (LineIdentified.Count == 4)
            {
                var a = LineIdentified[0].Intersect(LineIdentified[3], false).Value;
                var b = LineIdentified[0].Intersect(LineIdentified[1], false).Value;
                var c = LineIdentified[2].Intersect(LineIdentified[3], false).Value;
                var d = LineIdentified[1].Intersect(LineIdentified[2], false).Value;
                screenQuad = new QuadrilateralInCamera(a, b, c, d, new Vector(Size.x, Size.y));
                if (!screenQuad.IsQuadComplete())
                    screenQuad = null;
            }
            if (screenQuad == null && Screen.QuadInCamera != null)     // 如果可能，回退到上一个screen
            {
                Debug.Log("<color=aqua>[ScreenIdentification] 本次识别失败，回退到上次的识别结果</color>");
                quadTemp.Add(Screen.QuadInCamera.Quad);
            }
            else if (screenQuad != null)
            {
                Debug.Log("<color=aqua>[ScreenIdentification] 识别到四边形</color>");
                quadTemp.Add(screenQuad.Quad);
            }



            // 还需要输出一张识别结果图，包含干扰线段
            var LSDLineMap = new Matrix(Size, Tiling: true);
            foreach (var l in possibleLines)
            {
                if (l != null && !quadLines.Contains(l))
                    o0Extension.DrawLine(LSDLineMap, l, (x, y) => 3, new Geometry2D.Float.Vector(0, 2), true); // 其他的备选线段
            }
            foreach (var l in quadLines)
            {
                if (l != null)
                    o0Extension.DrawLine(LSDLineMap, l, (x, y) => 2, new Geometry2D.Float.Vector(0, 4)); // 这次识别到的线段
            }
            if (oldLines != null)
            {
                foreach (var l in oldLines)
                    o0Extension.DrawLine(LSDLineMap, l, (x, y) => 1, new Geometry2D.Float.Vector(0, 2), true);     // 旧的屏幕线段(例如上次手动识别的)
            }
            Texture2D ChoosableLineTex = LSDLineMap.ToTexRGBA(FloatValueToColor);


            Debug.Log(log);
            // 是否将图片保存到本地
            if (ScreenLocate.Main.SaveToggle.isOn && ScreenLocate.Main.DebugOnZIMDemo)
            {
                var FileDirectory = $"Debug_屏幕定位/";
                SaveImages(FileDirectory, log, ScreenLocateTexture, allLinesTex, ChoosableLineTex, ScreenQuadTex);
            }

            //times.Add(watch.ElapsedMilliseconds);
            //UnityEngine.Debug.Log("time: " + (times[times.Count - 1] - times[times.Count - 2]));

            // opecncv处理, zim
            {
                //var cvLines = edge.cvHoughLinesP();
                //ScreenLocate.DebugTexture(5, cvLines);

                //var myLines = Hough.Transform(edgeMat);
                //var cvLines = edge.cvLine(myLines);
                //ScreenLocate.DebugTexture(5, cvLines);
            }

            {
                ScreenLocate.DebugTexture(2, ScreenLocateTexture);
                ScreenLocate.DebugTexture(3, ScreenQuadTex);
                // 融合线段和原图
                ScreenLocate.DebugTexture(4, ScreenLocateTexture.Merge(ScreenQuadTex));
                ScreenLocate.DebugTexture(5, ChoosableLineTex);
            }
            foreach (var i in LocateTexTemp)
            {
                if (i != ScreenLocateTexture)       // ScreenLocateTexture 由 ScreenLocate.DebugTexture 释放
                    GameObject.Destroy(i);
            }
        }

        Vector GetAvgPoint(Matrix screenLocateMat)
        {
            // 加权平均
            Vector[] avgPointsColumn = new Vector[screenLocateMat.Size.x];
            float[] valueSumsColumn = new float[screenLocateMat.Size.x];
            Parallel.For(0, screenLocateMat.Size.x, i =>
            {
                for (int j = 0; j < screenLocateMat.Size.y; j++)
                {
                    var value = screenLocateMat[i, j];
                    valueSumsColumn[i] += value;
                    avgPointsColumn[i] += new Vector(i, j) * value;
                }
            });

            Vector avgPoint = Vector.Zero;
            var valueSum = 0f;
            for (int i = 0; i < screenLocateMat.Size.x; i++)
            {
                avgPoint += avgPointsColumn[i];
                valueSum += valueSumsColumn[i];
            }
            avgPoint /= valueSum;
            return avgPoint;
        }

        // 返回查找到的线段数量，0是查找失败
        int ZIMIdentifyQuadLSD(ref List<LineIdentified> allLines, int batch, (Matrix edgeMat, Matrix edgeDirMat) edgeGradient,
            float minLength = 100)
        {
            var l = edgeGradient.edgeMat.IdentifyLineLSD(edgeGradient.edgeDirMat, minLength, 50, LineCaptureSize: new Vector(10, 6));
            if (l == null || l.Count == 0)
                return 0;
            allLines.AddRange(l.Select((i) => new LineIdentified(batch, i)));
            return l.Count;
        }

        // 返回四边形的四条边，List长度一定是4 (如果没有识别到就是null)，且线段顺序是: 下、右、上、左
        List<Line> FilterLines(List<Matrix> screenLocateMatList, List<LineIdentified> allLines, Vector avgPoint,
            out Line[] oldLines, out List<Line> possibleLines, 
            ScreenMap screen, float conSize, float gradientLength, float minLength = 100)
        {
            //Debug.Log("[IdentifyLineLSD] lines.Count: " + lines.Count);
            // LSD计算得到的矩阵尺寸较小（因为卷积），这里必须进行位移
            var offset = new Vector((conSize - 1) / 2, (conSize - 1) / 2);
            for (int i = 0; i < allLines.Count; i++)
                allLines[i].Offset(offset);

            // 沿直线计算综合梯度（梯度乘以长度系数，再乘以距离系数）, distanceRatio是实际距离除以最大距离
            float estimateGradient(LineIdentified line, float distanceRatio)
            {
                var dir = (line.Line.B - line.Line.A).Normalized;
                var vertical = new Vector(-dir.y, dir.x) * (gradientLength / 2);
                var step = 2;
                var ll = line.Line.Length;
                var lg = new List<float>();
                for (int i = 0; i <= ll; i += step)
                {
                    var point = line.Line.A + dir * i;
                    var ga = point + vertical;
                    var gb = point - vertical;
                    lg.Add(screenLocateMatList[line.Batch][(int)ga.x, (int)ga.y] - screenLocateMatList[line.Batch][(int)gb.x, (int)gb.y]);
                }

                float e = (float)Math.Pow(Math.Ceiling(line.Line.Length / minLength), 0.2f);       // 长度系数，筛选时梯度更大、长度更长的线段更优
                float d = (3 - distanceRatio) / 2;            // 距离系数，距离越近，系数越大
                return e * d * Math.Abs(lg.Mean());
            }

            // 下、右、上、左
            var quadLines = new List<(float, Line)>[4] { new List<(float, Line)>(), new List<(float, Line)>(), new List<(float, Line)>(), new List<(float, Line)>() };
            possibleLines = new List<Line>();
            oldLines = null;

            // 如果已有定位数据，根据现有数据筛选线条
            if (screen.QuadInCamera != null)
            {
                Debug.Log("[IdentifyLineLSD] 根据已有定位数据做筛选");
                screen.RefreshCameraSize(new Vector2(Size.x, Size.y));

                var calibration = ScreenLocate.Main.ReDoLocateCalibrationRatio * Size.y;
                oldLines = screen.QuadInCamera.GetLines();

                var pedals = oldLines.Select((i) => o0Extension.PointPedal(i, avgPoint, out _)).ToArray();     // 当前定位的垂足，下、右、上、左

                foreach (var i in allLines)
                {
                    float minDistance = float.MaxValue;
                    int index = -1;
                    foreach (var j in pedals.Index())
                    {
                        var d = (o0Extension.PointPedal(i.Line, avgPoint, out _) - pedals[j]).Length;
                        if (d < minDistance)
                        {
                            minDistance = d;
                            index = j;
                        }
                    }
                    //Debug.Log(minDistance +", -----------"+ calibration);
                    if (minDistance < calibration)      // 垂足的距离足够近
                    {
                        // 另外满足，新的线段的中点，到旧线段的垂足，要在旧线段内
                        var middleToOldLine = o0Extension.PointPedal(oldLines[index], (i.Line.A + i.Line.B) / 2, out bool inLineSegment);
                        if (inLineSegment)
                        {
                            quadLines[index].Add((estimateGradient(i, minDistance / calibration), i.Line));
                            possibleLines.Add(i.Line);
                        }
                    }
                }
            }
            else
            {
                var avaAngleHalf = 75f;
                foreach (var line in allLines)
                {
                    possibleLines.Add(line.Line);

                    var a = (avgPoint - (line.Line.A + line.Line.B) / 2).DegreeToXAxis();
                    //Debug.Log(a + ", " + gradient + ", " + sum);
                    int index = -1;
                    if (Math.Abs(a - line.GradientDegree) < avaAngleHalf || Math.Abs(a - 360 - line.GradientDegree) < avaAngleHalf || Math.Abs(a + 360 - line.GradientDegree) < avaAngleHalf)
                    {
                        if (line.GradientDegree > 45 && line.GradientDegree < 135)     // 下
                            index = 0;
                        else if (line.GradientDegree > 135 && line.GradientDegree < 225) // 右
                            index = 1;
                        else if (line.GradientDegree > 225 && line.GradientDegree < 315)  // 上
                            index = 2;
                        else
                            index = 3;

                        //var g = Math.Abs(lg.Mean());
                        //Debug.Log(gradient + ", " + g);

                        //List<float> lp1 = new List<float>(), lp2 = new List<float>();    // 线两侧的值
                        //for (float i = 0; i <= ll; i += step)
                        //{
                        //    var point = line.A + dir * i;
                        //    var ga = point + vertical;
                        //    var gb = point - vertical;
                        //    lp1.Add(screenLocateMat[(int)ga.x, (int)ga.y]);
                        //    lp2.Add(screenLocateMat[(int)gb.x, (int)gb.y]);
                        //}

                        //var avg1 = lp1.Mean();
                        //var avg2 = lp2.Mean();
                        //var v1 = lp1.Variance();
                        //var v2 = lp2.Variance();

                        //var lineGradient = Math.Abs(avg1 - avg2) / (v1 + v2 + 0.2f);       // 方差越小，梯度的价值越高
                        ////var g = Math.Abs(lg.Mean());
                        ////Debug.Log(gradient + ", " + g);
                        //Debug.Log(v1 + ", " + v2 + ", " + lineGradient);

                        //quadLines[index].Add((lineGradient, line));

                        quadLines[index].Add((estimateGradient(line, 1), line.Line));
                    }
                }
            }

            var result = new Line[4];
            for (int i = 0; i < 4; i++)
            {
                if (quadLines[i].Count > 0)
                    result[i] = quadLines[i].Max((a, b) => a.Item1.CompareTo(b.Item1)).Item2;
            }
            return result.ToList();
        }

        void SaveImages(string FileDirectory, string log, Texture2D ScreenLocateTex, Texture2D allLinesTex, Texture2D ChoosableLineTex, Texture2D ScreenQuadTex)
        {
            if (!Directory.Exists(FileDirectory))
                Directory.CreateDirectory(FileDirectory);
            var time = DateTime.Now.ToString("yyyyMMdd_HHmmss");

            var pngData = (ScreenLocate.Main.outputTexture2D[7] as Texture2D)?.EncodeToPNG();
            if (pngData != null)
                File.WriteAllBytes($"{FileDirectory}{time}A屏幕原图.png", pngData);
            var pngData1 = ScreenLocateTex.EncodeToPNG();
            if (pngData1 != null)
                File.WriteAllBytes($"{FileDirectory}{time}B黑白色差.png", pngData1);
            var pngData2 = allLinesTex.EncodeToPNG();
            if (pngData2 != null)
                File.WriteAllBytes($"{FileDirectory}{time}C全部识别线段.png", pngData2);
            var pngData3 = ChoosableLineTex.EncodeToPNG();
            if (pngData3 != null)
                File.WriteAllBytes($"{FileDirectory}{time}D备选线段.png", pngData3);
            var pngData4 = ScreenQuadTex.EncodeToPNG();
            if (pngData4 != null)
                File.WriteAllBytes($"{FileDirectory}{time}E识别结果.png", pngData4);


            Debug.Log($"<color=aqua>({time}) 屏幕识别图片保存至：程序根目录/{FileDirectory}</color>");
            log += 
                $"\r\n屏幕原图保存{pngData != null}, " +
                $"\r\n黑白色差保存{pngData1 != null}, " +
                $"\r\n全部识别线段保存{pngData2 != null}, " +
                $"\r\n备选线段保存{pngData3 != null}, " +
                $"\r\n识别结果保存{pngData4 != null}, ";
            File.WriteAllText($"{FileDirectory}{time}屏幕自动定位_日志.log", log);
        }

    }
}