ScreenIdentification.cs 50 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110
  1. #define ENABLE_LOG
  2. using o0.Geometry2D.Float;
  3. using o0.Num;
  4. using System;
  5. using System.Collections.Generic;
  6. using System.IO;
  7. using System.Linq;
  8. using System.Threading.Tasks;
  9. using UnityEngine;
  10. using UnityEngine.UIElements;
  11. using ZIM;
  12. using ZIM.Unity;
  13. namespace o0.Project
  14. {
  15. public partial class ScreenIdentification
  16. {
  17. private const string TAG = "ScreenIdentification#";
  18. // LocateAreaData表示每次屏幕的色差变化的区域,可能有多次。通过设置LocateSingleStep可调整为仅识别一次色差
  19. static Rect[][] LocateAreaData = new Rect[][] {
  20. new Rect[] { new Rect(0f, 0f, 0.3f, 0.3f), new Rect(0f, 0f, 0.4f, 0.4f), new Rect(0f, 0f, 0.5f, 0.5f) },
  21. new Rect[] { new Rect(0.7f, 0f, 0.3f, 0.3f), new Rect(0.6f, 0f, 0.4f, 0.4f), new Rect(0.5f, 0f, 0.5f, 0.5f) },
  22. new Rect[] { new Rect(0f, 0.7f, 0.3f, 0.3f), new Rect(0f, 0.6f, 0.4f, 0.4f), new Rect(0f, 0.5f, 0.5f, 0.5f) },
  23. new Rect[] { new Rect(0.7f, 0.7f, 0.3f, 0.3f), new Rect(0.6f, 0.6f, 0.4f, 0.4f), new Rect(0.5f, 0.5f, 0.5f, 0.5f) }
  24. };
  25. //static Rect[][] LocateAreaData = new Rect[][] {
  26. // new Rect[] { new Rect(0f, 0f, 0.3f, 0.3f), new Rect(0f, 0f, 0.4f, 0.4f), new Rect(0f, 0f, 0.5f, 0.5f), new Rect(0f, 0f, 0.6f, 0.6f) },
  27. // new Rect[] { new Rect(0.7f, 0f, 0.3f, 0.3f), new Rect(0.6f, 0f, 0.4f, 0.4f), new Rect(0.5f, 0f, 0.5f, 0.5f), new Rect(0.4f, 0f, 0.6f, 0.6f) },
  28. // new Rect[] { new Rect(0f, 0.7f, 0.3f, 0.3f), new Rect(0f, 0.6f, 0.4f, 0.4f), new Rect(0f, 0.5f, 0.5f, 0.5f), new Rect(0f, 0.4f, 0.6f, 0.6f) },
  29. // new Rect[] { new Rect(0.7f, 0.7f, 0.3f, 0.3f), new Rect(0.6f, 0.6f, 0.4f, 0.4f), new Rect(0.5f, 0.5f, 0.5f, 0.5f), new Rect(0.4f, 0.4f, 0.6f, 0.6f) }
  30. //};
  31. //static bool LocateSingleStep = false;
  32. static bool LocateSingleStep = true;
  33. public Geometry2D.Vector<int> Size => ScreenLocate.Main.CameraSize;
  34. public QuadrilateralInCamera QuadManual;
  35. public QuadrilateralInCamera QuadAuto; // 全自动,可以给用户选择(赋值给Screen.QuadInCamera即生效)
  36. public QuadrilateralInCamera QuadSemiAuto; // 半自动,可以给用户选择(赋值给Screen.QuadInCamera即生效)
  37. public ScreenMap Screen; // 识别到的屏幕,用于执行透视变换
  38. int capture = 0;
  39. int delay = 0;
  40. int maxCapture;
  41. int maxDelay;
  42. Geometry.Vector<float>[] ScreenBlackTexture;
  43. Geometry.Vector<float>[] ScreenWhiteTexture;
  44. int locateIndex = -1;
  45. readonly List<Rect> locateArea = new List<Rect> {
  46. new Rect(0f, 0f, 0.5f, 0.5f), new Rect(0.5f, 0f, 0.5f, 0.5f), new Rect(0f, 0.5f, 0.5f, 0.5f), new Rect(0.5f, 0.5f, 0.5f, 0.5f)
  47. }; // 屏幕显示白色的区域大小
  48. float areaPercent => locateArea[locateIndex].size.x; // 当前白色区域的占比
  49. int areaSelected = -1; // 选择哪个区域,顺序与Quadrilateral对应
  50. readonly List<float> sumTemp = new List<float>();
  51. readonly List<QuadrilateralInCamera> quadTemp = new List<QuadrilateralInCamera>();
  52. //public ScreenIdentification(WebCamTexture texture)
  53. //{
  54. // Size = new Geometry2D.Vector<int>(texture.width, texture.height);
  55. // Screen = new ScreenMap();
  56. //}
  57. public static UnityEngine.Color FloatValueToColor(float i)
  58. {
  59. return i switch
  60. {
  61. 1 => UnityEngine.Color.yellow,
  62. 2 => new UnityEngine.Color(0,1,1,1),
  63. 3 => UnityEngine.Color.green,
  64. 4 => UnityEngine.Color.white,
  65. 5 => UnityEngine.Color.red,
  66. _ => UnityEngine.Color.black,
  67. };
  68. }
  69. public ScreenIdentification()
  70. {
  71. Screen = new ScreenMap();
  72. //OnLocateScreenEnter += () => Application.targetFrameRate = 30; // 固定识别的帧率,确保摄像机拍到正确的画面
  73. //OnLocateScreenEnd += () => Application.targetFrameRate = 60;
  74. }
  75. public void SetScreenQuad(QuadrilateralInCamera quad) => Screen.QuadInCamera = quad;
  76. // 上一次半自动识别的情况,false代表这条边识别失败,线段顺序: 下、右、上、左
  77. public bool[] LastQuadSemiAutoState;
  78. public event Action OnLocateScreenEnter;
  79. public event Action OnLocateScreenEnd;
  80. public bool bStartLocateScreen { get; set; } = false;//是否进行捕获
  81. public bool SelectScreenAfterLocate(ScreenLocate.ScreenIdentificationTag tag)
  82. {
  83. QuadrilateralInCamera target = tag switch
  84. {
  85. ScreenLocate.ScreenIdentificationTag.Manual => QuadManual,
  86. ScreenLocate.ScreenIdentificationTag.SemiAuto => QuadSemiAuto,
  87. ScreenLocate.ScreenIdentificationTag.Auto => QuadAuto,
  88. _ => null
  89. };
  90. if (target == null)
  91. return false;
  92. Debug.Log($"<color=aqua>[ScreenIdentification] 选择已识别到的屏幕({Enum.GetName(typeof(ScreenLocate.ScreenIdentificationTag), tag)}), {target}</color>");
  93. SetScreenQuad(target);
  94. return true;
  95. }
  96. // 自动识别开始的入口
  97. public void LocateScreen(int Capture = 30, int Delay = 30) //数值单位是frame
  98. {
  99. if (ScreenLocate.Main.DebugScreenImages.Count != 0 && ScreenLocate.Main.DebugOnZIMDemo) // 这段仅用于测试图片
  100. {
  101. ScreenLocate.Main.CameraSize = new Geometry2D.Vector<int>(ScreenLocate.Main.DebugScreenImages[0].width, ScreenLocate.Main.DebugScreenImages[0].height);
  102. DebugImage(ScreenLocate.Main.DebugScreenImages);
  103. Screen.QuadInCamera = quadTemp[0];
  104. ScreenLocate.SetScreen(null);
  105. ScreenLocate.Main.ShowScreen(ScreenLocate.Main.ScreenQuad, Screen.QuadInCamera);
  106. delay = 0;
  107. capture = 0;
  108. ScreenWhiteTexture = null;
  109. ScreenBlackTexture = null;
  110. locateIndex = -1;
  111. areaSelected = -1;
  112. quadTemp.Clear();
  113. sumTemp.Clear();
  114. ScreenLocate.Main.DebugScreenImages.Clear();
  115. return;
  116. }
  117. delay = Math.Max(Delay, 5);
  118. capture = Math.Max(Capture, 5);
  119. maxDelay = Delay;
  120. maxCapture = Capture;
  121. ScreenLocate.SetScreen(new Rect(0f, 0f, 1f, 1f), UnityEngine.Color.black);
  122. //ScreenLocate.SetScreen(new Rect(0f, 0f, 0.6f, 0.6f), UnityEngine.Color.white);
  123. //bStartLocateScreen = false;
  124. ScreenWhiteTexture = null;
  125. ScreenBlackTexture = null;
  126. OnLocateScreenEnter?.Invoke();
  127. }
  128. /// <summary>
  129. /// 开始进行捕获
  130. /// 初始化了两个数据 capture 和 delay
  131. /// </summary>
  132. /// <returns></returns>
  133. public bool isInitLocateScreen()
  134. {
  135. return capture != 0 && delay != 0;
  136. }
  137. void DebugImage(List<Texture2D> images)
  138. {
  139. QuadrilateralFit(images);
  140. //var watch = new System.Diagnostics.Stopwatch();
  141. //watch.Start();
  142. //var times = new List<double>() { 0.0 };
  143. #if (!NDEBUG && DEBUG && ENABLE_LOG)
  144. Console.WriteLine($"{TAG} quadTemp.Count:{quadTemp.Count}");
  145. #endif
  146. if (quadTemp.Count > 0)
  147. {
  148. ScreenLocate.Main.ShowScreen(ScreenLocate.Main.outputRawImages[4].transform.GetChild(0) as RectTransform, quadTemp[0]);
  149. // 透视变换
  150. // var srcWidth = LocateLightedRedTex.width;
  151. // var transformWidth = (int)((quad.B.x - quad.A.x + quad.D.x - quad.C.x) / 2);
  152. // var transformHeight = (int)((quad.C.y - quad.A.y + quad.D.y - quad.B.y) / 2);
  153. // var transformTex = new Texture2D(transformWidth, transformHeight);
  154. // var pt = new ZIMPerspectiveTransform(new OrdinalQuadrilateral(new Vector(0, 0), new Vector(transformWidth, 0), new Vector(0, transformHeight), new Vector(transformWidth, transformHeight)), quad);
  155. // var dstPixel = new UnityEngine.Color[transformWidth * transformHeight];
  156. // var srcPixel = LocateLightedRedTex.GetPixels();
  157. // Parallel.For(0, transformWidth, (x) =>
  158. // {
  159. // for (int y = 0; y < transformHeight; y++)
  160. // {
  161. // var index = y * transformWidth + x;
  162. // var sampleCoord = pt.TransformRound(x, y);
  163. // dstPixel[index] = srcPixel[sampleCoord.y * srcWidth + sampleCoord.x];
  164. // }
  165. // });
  166. // transformTex.SetPixels(dstPixel);
  167. // transformTex.Apply();
  168. // //ScreenLocate.DebugTexture(1, transformTex);
  169. //#if (!NDEBUG && DEBUG && ENABLE_LOG)
  170. // Console.WriteLine($"{TAG} ScreenLocate.DebugTexture 1:{transformTex.GetNativeTexturePtr()}");
  171. //#endif
  172. }
  173. //times.Add(watch.ElapsedMilliseconds);
  174. //Debug.Log("time: " + (times[times.Count - 1] - times[times.Count - 2]));
  175. }
  176. public void NextScreen()
  177. {
  178. // 只识别一次色差变化
  179. if (LocateSingleStep && areaSelected == -1)
  180. {
  181. LocateAreaData = new Rect[][] { new Rect[] { new Rect(0, 0, 1f, 1f) } };
  182. locateIndex = 3;
  183. areaSelected = 0;
  184. locateArea.AddRange(LocateAreaData[0]);
  185. }
  186. // index从-1开始
  187. locateIndex++;
  188. if (locateIndex < locateArea.Count) // 依次点亮屏幕区域
  189. {
  190. ScreenLocate.SetScreen(locateArea[locateIndex], UnityEngine.Color.white);
  191. delay = maxDelay;
  192. capture = maxCapture;
  193. }
  194. else // 退出屏幕黑白控制
  195. {
  196. ScreenLocate.SetScreen(null);
  197. ScreenLocate.Main.ShowScreen(ScreenLocate.Main.ScreenQuad, Screen.QuadInCamera);
  198. Reset();
  199. }
  200. }
  201. // 清除记录的屏幕识别数据(手动、自动等)
  202. public void ClearQuadCache()
  203. {
  204. SetScreenQuad(null);
  205. QuadManual = null;
  206. QuadSemiAuto = null;
  207. QuadAuto = null;
  208. }
  209. public void Reset()
  210. {
  211. // bStartLocateScreen = false;
  212. delay = 0;
  213. capture = 0;
  214. ScreenWhiteTexture = null;
  215. ScreenBlackTexture = null;
  216. locateIndex = -1;
  217. areaSelected = -1;
  218. if (locateArea.Count > 4)
  219. locateArea.RemoveRange(4, LocateAreaData[0].Length);
  220. quadTemp.Clear();
  221. sumTemp.Clear();
  222. }
  223. public void CaptureBlack(Texture2D cam)
  224. {
  225. if (ScreenBlackTexture == null)
  226. ScreenBlackTexture = new Geometry.Vector<float>[Size.x * Size.y];
  227. Debug.Log($"采样屏幕黑色,Texture2D size: {cam.Size()}, ScreenLocate size: ({Size.x}, {Size.y})");
  228. Debug.Log($"ScreenBlackTexture.Length: ({ScreenBlackTexture.Length})");
  229. var pixel = cam.GetPixels();
  230. Parallel.For(0, Size.x * Size.y, i =>
  231. {
  232. var ip = pixel[i];
  233. ScreenBlackTexture[i] += new Geometry.Vector<float>(ip.r / maxCapture, ip.g / maxCapture, ip.b / maxCapture);
  234. });
  235. }
  236. public void CaptureWhite(Texture2D cam)
  237. {
  238. if (ScreenWhiteTexture == null)
  239. ScreenWhiteTexture = new Geometry.Vector<float>[Size.x * Size.y];
  240. Debug.Log($"采样屏幕白色,Texture2D size: {cam.Size()}, ScreenLocate size: ({Size.x}, {Size.y})");
  241. Debug.Log($"ScreenWhiteTexture.Length: ({ScreenWhiteTexture.Length})");
  242. var pixel = cam.GetPixels();
  243. Parallel.For(0, Size.x * Size.y, i =>
  244. {
  245. var ip = pixel[i];
  246. ScreenWhiteTexture[i] += new Geometry.Vector<float>(ip.r / maxCapture, ip.g / maxCapture, ip.b / maxCapture);
  247. });
  248. }
  249. public void CaptureStay(Texture2D cam)
  250. {
  251. if (locateIndex == -1) // 屏幕黑色
  252. {
  253. CaptureBlack(cam);
  254. }
  255. else // 屏幕部分为白色
  256. {
  257. CaptureWhite(cam);
  258. }
  259. }
  260. public void CaptureEnd()
  261. {
  262. //Debug.Log("locateIndex: " + locateIndex + ", quad: " + quadTemp.Count);
  263. if (locateIndex == -1)
  264. return;
  265. if (locateIndex < 4)
  266. {
  267. sumTemp.Add(GetBrightness());
  268. ScreenWhiteTexture = null;
  269. // 选择亮度差最大的区域
  270. if (locateIndex == 3)
  271. {
  272. areaSelected = sumTemp.MaxIndex();
  273. locateArea.AddRange(LocateAreaData[areaSelected]);
  274. }
  275. }
  276. else if (locateIndex >= 4 && locateIndex < locateArea.Count - 1)
  277. {
  278. QuadrilateralFit();
  279. ScreenWhiteTexture = null;
  280. }
  281. else
  282. {
  283. QuadrilateralFit();
  284. if (quadTemp.Count != LocateAreaData[0].Length)
  285. {
  286. Debug.Log($"<color=yellow>[ScreenIdentification] 拟合四边形失败, quadTemp.Count: {quadTemp.Count}</color>");
  287. }
  288. else if (quadTemp.Count == 1)
  289. {
  290. SetScreenQuad(quadTemp[0]);
  291. Debug.Log($"[ScreenIdentification] 拟合成功,识别数据: {Screen.QuadInCamera}");
  292. }
  293. else
  294. {
  295. // Debug.Log($"拟合四边形 2 , quadTemp.Count: {quadTemp.Count}");
  296. // 线性拟合
  297. var xValue = new List<float>() { 0 };
  298. var predicts = new List<Vector>();
  299. foreach (var i in LocateAreaData[0])
  300. xValue.Add(i.size.x);
  301. Vector baseVertex = Vector.Zero; // x==0 时的点
  302. {
  303. foreach (var q in quadTemp)
  304. {
  305. baseVertex += q.Quad[areaSelected];
  306. }
  307. baseVertex /= quadTemp.Count;
  308. }
  309. double rs = 0.0;
  310. for (int i = 0; i < 4; i++)
  311. {
  312. if (i == areaSelected)
  313. {
  314. predicts.Add(baseVertex);
  315. }
  316. else
  317. {
  318. var yValue = new List<Vector>() { baseVertex };
  319. foreach (var q in quadTemp)
  320. {
  321. yValue.Add(q.Quad[i]);
  322. }
  323. var lr = LinerRegression1D.Fit(2, xValue.ToArray(), yValue.ToArray());
  324. rs += lr.RSquared / 3;
  325. predicts.Add(lr.Predict<Vector>(1));
  326. }
  327. }
  328. SetScreenQuad(new QuadrilateralInCamera(predicts, new Vector(Size.x, Size.y)));
  329. Debug.Log($"[ScreenIdentification] 拟合成功,RSquared: {rs}, Quad: {Screen.QuadInCamera}");
  330. //if (rs < 0.8) Screen.Quad = null;
  331. }
  332. OnLocateScreenEnd?.Invoke();
  333. }
  334. }
  335. public bool Update(Texture2D cam)
  336. {
  337. //if (!bStartLocateScreen) return false;
  338. if (delay != 0)
  339. {
  340. //ScreenLocate.Main.CreateUVCTexture2DFocusSizeIfNeeded(1280, 720);
  341. delay--;
  342. if (delay == 0)
  343. {
  344. ScreenLocate.Main.CameraSize = new Geometry2D.Vector<int>(cam.width, cam.height); // 记录当前的分辨率
  345. Debug.Log("[ScreenIdentification] 采样纹理,记录采样分辨率: [" + Size.x + ", " + Size.y + "]");
  346. }
  347. return true;
  348. }
  349. if (capture != 0)
  350. {
  351. //ScreenLocate.Main.CreateUVCTexture2DFocusSizeIfNeeded(1280, 720);
  352. CaptureStay(cam);
  353. capture--;
  354. if (capture == 0)
  355. {
  356. CaptureEnd();
  357. NextScreen();
  358. }
  359. return true;
  360. }
  361. return false;
  362. #region Old
  363. /*
  364. if (delay != 0)
  365. {
  366. delay--;
  367. return true;
  368. }
  369. if (capture != 0)
  370. {
  371. capture--;
  372. if (ScreenBlackTexture == null)
  373. ScreenBlackTexture = new Geometry.Vector<float>[Size.x * Size.y];
  374. var pixel = cam.GetPixels();
  375. Parallel.For(0, Size.x * Size.y, i =>
  376. {
  377. var ip = pixel[i];
  378. ScreenBlackTexture[i] += new Geometry.Vector<float>(ip.r, ip.g, ip.b);
  379. });
  380. if (capture == 0)
  381. ScreenLocate.SetScreen(UnityEngine.Color.black);
  382. return true;
  383. }
  384. if (delay != 0)
  385. {
  386. delay--;
  387. return true;
  388. }
  389. if (capture != 0)
  390. {
  391. capture--;
  392. if (ScreenWhiteTexture == null)
  393. ScreenWhiteTexture = new Geometry.Vector<float>[Size.x * Size.y];
  394. var pixel = cam.GetPixels();
  395. Parallel.For(0, Size.x * Size.y, i =>
  396. {
  397. var ip = pixel[i];
  398. ScreenWhiteTexture[i] += new Geometry.Vector<float>(ip.r, ip.g, ip.b);
  399. });
  400. if (capture == 0)
  401. ScreenLocate.SetScreen(UnityEngine.Color.black);
  402. return true;
  403. }
  404. if (delay != 0)
  405. {
  406. delay--;
  407. return true;
  408. }
  409. if (capture != 0)
  410. {
  411. capture--;
  412. var pixel = cam.GetPixels();
  413. Parallel.For(0, Size.x * Size.y, i =>
  414. {
  415. var ip = pixel[i];
  416. ScreenWhiteTexture[i] -= new Geometry.Vector<float>(ip.r, ip.g, ip.b);
  417. });
  418. if (capture == 0)
  419. {
  420. ScreenLocate.SetScreen(null);
  421. UnityEngine.Color[] newPixel = new UnityEngine.Color[Size.x * Size.y];
  422. Parallel.For(0, Size.x * Size.y, i => {
  423. var pi = ScreenWhiteTexture[i] /= capture;
  424. newPixel[i] = new UnityEngine.Color(pi.x, pi.y, pi.z);
  425. });
  426. //读取数据
  427. //{
  428. // var fileName = "3.bin";
  429. // ScreenLocateTexture = $"2023 04 16 厦门测试数据/{fileName}".FileReadByte<Vector<float>[]>();
  430. // Debug.Log($"Read {fileName}");
  431. // Parallel.For(0, Size.x * Size.y, i =>
  432. // {
  433. // var pi = ScreenLocateTexture[i];
  434. // newPixel[i] = new UnityEngine.Color(pi.x, pi.y, pi.z);
  435. // });
  436. //}
  437. var ScreenLocateTex = new Texture2D(Size.x, Size.y);
  438. ScreenLocateTex.SetPixels(newPixel);
  439. ScreenLocateTex.Apply();
  440. //ScreenLocate.DebugTexture(2, ScreenLocateTex);
  441. var ScreenLocateTexLighted = ScreenLocateTex.AutoLight(10);
  442. //ScreenLocate.DebugTexture(2, ScreenLocateTexLighted);
  443. //var FileSavePath = Application.persistentDataPath + "/ScreenLocateTexture.bin";
  444. bool Save = ScreenLocate.Main.SaveToggle.isOn;
  445. string time;
  446. if (Save)
  447. {
  448. time = DateTime.Now.ToString("yyyyMMdd_HHmmss");
  449. var FileSavePath = $"屏幕定位数据{time}.bin";
  450. FileSavePath.FileWriteByte(ScreenWhiteTexture);
  451. var bytes = ScreenLocateTexLighted.EncodeToPNG();
  452. File.WriteAllBytes($"屏幕定位数据{time}.png", bytes);
  453. Debug.Log("ScreenLocateTexture Saved To: " + FileSavePath);
  454. }
  455. var ScreenLocateTexR = ScreenLocateTexLighted.ToRGB(ColorChannel.Red);
  456. var ScreenLocateTexG = ScreenLocateTexLighted.ToRGB(ColorChannel.Green);
  457. var ScreenLocateTexB = ScreenLocateTexLighted.ToRGB(ColorChannel.Blue);
  458. ScreenLocate.DebugTexture(2, ScreenLocateTexR);
  459. //ScreenLocate.DebugTexture(4, ScreenLocateTexG);
  460. //ScreenLocate.DebugTexture(5, ScreenLocateTexB);
  461. var watch = new System.Diagnostics.Stopwatch();
  462. watch.Start();
  463. var times = new List<double>() { 0.0 };
  464. var ScreenLocateTexLightedMat = ScreenLocateTexLighted.Too0Mat();
  465. //var ScreenLocateTexLightedMat = texture.Too0Mat();
  466. //var (edge, edgeDir) = ScreenLocateTexLightedMat.IdentifyEdge();
  467. var (edge, edgeDir) = ScreenLocateTexLightedMat.zimIdentifyEdgeGradientAny(15);
  468. //ScreenLocate.DebugTexture(4, ScreenLocateTexLighted.Too0Mat().IdentifyEdgeGradient().ToTex());
  469. //ScreenLocate.DebugTexture(4, edge.ToTex());
  470. var quadLines = ScreenLocateTexLightedMat.IdentifyQuadLSD(edge, edgeDir, out List<Line> lightLines, 30);
  471. var drawLineMap = new MatrixF2D(edge..Size.x, edge.Size.y);
  472. int lineCount = 0;
  473. foreach (var l in quadLines)
  474. {
  475. if (l != null)
  476. {
  477. o0Extension.DrawLine(drawLineMap.DrawLine(l, (x, y) => 1, new Geometry2D.Float.Vector(0, 10));
  478. lineCount++;
  479. }
  480. }
  481. if (lineCount == 4)
  482. {
  483. var a = quadLines[0].Intersect(quadLines[3], false).Value;
  484. var b = quadLines[0].Intersect(quadLines[1], false).Value;
  485. var c = quadLines[2].Intersect(quadLines[3], false).Value;
  486. var d = quadLines[1].Intersect(quadLines[2], false).Value;
  487. Quad = new Quadrilateral(a, b, c, d);
  488. if (!Quad.IsInScreen(ScreenLocate.Main.WebCamera.Size))
  489. Quad = null;
  490. }
  491. ScreenLocate.Main.ShowScreen(Quad);
  492. //var lines = edge.IdentifyLineLSD(edgeDir, 100);
  493. ////var lines = ScreenLocateTexLightedMat.IdentifyLineLSD();
  494. //var drawLineMap = new MatrixF2D(edge..Size.x, edge.Size.y);
  495. //var returnMaxLines = lines.Sub(0, 10);
  496. //foreach (var (line, sum, gradient) in returnMaxLines)
  497. // o0Extension.DrawLine(drawLineMap.DrawLine(line, (x, y) => 1, new Geometry2D.Float.Vector(0, 10));
  498. ScreenLocate.DebugTexture(3, drawLineMap.ToTex());
  499. //{
  500. // var bytes = drawLineMap.ToTex().EncodeToPNG();
  501. // File.WriteAllBytes($"屏幕定位数据DrawLineMap.png", bytes);
  502. //}
  503. times.Add(watch.ElapsedMilliseconds);
  504. Debug.Log("time: " + (times[times.Count - 1] - times[times.Count - 2]));
  505. //ScreenLocate.DebugTexture(5, edge.IdentifyLine(edgeDir).ToTex());
  506. //ScreenLocate.DebugTexture(4, ScreenLocateTexLighted.Too0Mat().IdentifyEdgeGradientX().ToTex());
  507. //ScreenLocate.DebugTexture(5, ScreenLocateTexLighted.Too0Mat().IdentifyEdgeGradientY().ToTex());
  508. //var convolutionLighted2 = ScreenLocateTexLighted.Too0Mat().IdentifyEdgeVariance().ToTex();
  509. // opecncv处理
  510. // zim
  511. {
  512. //var cvLines = edge.cvHoughLinesP();
  513. //ScreenLocate.DebugTexture(5, cvLines);
  514. //var myLines = Hough.Transform(edgeMat);
  515. //var cvLines = edge.cvLine(myLines);
  516. //ScreenLocate.DebugTexture(5, cvLines);
  517. }
  518. UnityEngine.Object.Destroy(ScreenLocateTex);
  519. //ScreenLocate.DebugTexture(4, convolutionLighted2);
  520. }
  521. return true;
  522. }
  523. /*
  524. var avg = new Geometry4D.Vector<float>();
  525. var pixel = texture.GetPixels();
  526. foreach(var i in pixel.Index())
  527. {
  528. var iP = pixel[i];
  529. avg += new Geometry4D.Vector<float>(iP.r, iP.g, iP.b, iP.a);
  530. }
  531. avg /= pixel.Count();
  532. /*
  533. var (texLightedR, texLightedG, texLightedB) = ToRGB(newTex);
  534. ScreenLocate.DebugTexture(3, texLightedR);
  535. ScreenLocate.DebugTexture(4, texLightedG);
  536. ScreenLocate.DebugTexture(5, texLightedB);
  537. //Debug.Log(avg);
  538. return false;
  539. /**/
  540. #endregion
  541. }
  542. float GetBrightness()
  543. {
  544. UnityEngine.Color[] differPixel = new UnityEngine.Color[Size.x * Size.y];
  545. Parallel.For(0, Size.x * Size.y, i =>
  546. {
  547. var pi = ScreenWhiteTexture[i] - ScreenBlackTexture[i];
  548. differPixel[i] = new UnityEngine.Color(pi.x, pi.y, pi.z);
  549. });
  550. var sum = 0f;
  551. foreach (var i in differPixel)
  552. {
  553. sum += i.Brightness();
  554. }
  555. sum /= differPixel.Length;
  556. //Debug.Log(sum);
  557. return sum;
  558. }
  559. // 转换成屏幕定位所需的纹理图像
  560. Texture2D ToLocateTex(UnityEngine.Color[] pixels)
  561. {
  562. var ScreenLocateTex = new Texture2D(Size.x, Size.y);
  563. ScreenLocateTex.SetPixels(pixels);
  564. ScreenLocateTex.Apply();
  565. //ScreenLocate.DebugTexture(2, ScreenLocateTex);
  566. return ScreenLocateTex.AutoLight(10);
  567. //ScreenLocate.DebugTexture(2, ScreenLocateTexLighted);
  568. //var ScreenLocateTexR = ToLocateTex.ToRGB(ColorChannel.Red);
  569. //var ScreenLocateTexG = ToLocateTex.ToRGB(ColorChannel.Green);
  570. //var ScreenLocateTexB = ToLocateTex.ToRGB(ColorChannel.Blue);
  571. //LocateLightedRedTex = ScreenLocateTexR;
  572. //ScreenLocate.DebugTexture(2, ScreenLocateTexR);
  573. //ScreenLocate.DebugTexture(4, ScreenLocateTexG);
  574. //ScreenLocate.DebugTexture(5, ScreenLocateTexB);
  575. //var ScreenLocateTexLightedMat = texture.Too0Mat();
  576. }
  577. /// <param name="lineWidth">识别的最小线段长度</param>
  578. /// <param name="debugImages">这个参数如果不为null且数量大于0,则执行debug操作</param>
  579. void QuadrilateralFit(List<Texture2D> debugImages = null)
  580. {
  581. // 屏幕黑白差值,存放多批次的图像用于识别, 该List数量不能等于 0
  582. List<UnityEngine.Color[]> PixelsMultipleBatches = new List<UnityEngine.Color[]>();
  583. //var sw = new System.Diagnostics.Stopwatch();
  584. //sw.Start();
  585. //读取数据
  586. if (debugImages != null && debugImages.Count != 0)
  587. {
  588. foreach (var i in debugImages)
  589. {
  590. Debug.Log($"<color=aqua>Debug {i.name}</color>");
  591. PixelsMultipleBatches.Add(i.GetPixels());
  592. }
  593. }
  594. else // 获得屏幕差值
  595. {
  596. var maxWhite = 0f;
  597. foreach (var i in ScreenWhiteTexture)
  598. {
  599. var m = i.x > i.y ? (i.x > i.z ? i.x : i.z) : (i.y > i.z ? i.y : i.z);
  600. if (maxWhite < m)
  601. maxWhite = m;
  602. }
  603. //var scale = 1.0f / maxWhite; // 放大对比度
  604. //Debug.Log("[ScreenIdentification] 提高亮度倍数: " + scale);
  605. var differPixel = new UnityEngine.Color[Size.x * Size.y];
  606. var whitePixel = new UnityEngine.Color[Size.x * Size.y];
  607. Parallel.For(0, Size.x, x =>
  608. {
  609. for (int y = 0; y < Size.y; y++)
  610. {
  611. var i = y * Size.x + x;
  612. var d = ScreenWhiteTexture[i] - ScreenBlackTexture[i];
  613. differPixel[i] = new UnityEngine.Color(d.x, d.y, d.z);
  614. whitePixel[i] = new UnityEngine.Color(ScreenWhiteTexture[i].x, ScreenWhiteTexture[i].y, ScreenWhiteTexture[i].z);
  615. }
  616. });
  617. PixelsMultipleBatches.Add(differPixel); // 色差图
  618. PixelsMultipleBatches.Add(whitePixel); // 原图
  619. }
  620. int conSize = (int)Math.Ceiling(0.007f * Size.y) * 2 + 1;
  621. conSize = Math.Max(conSize, 7); // 设置最小为7
  622. float minLength = conSize * 4f;
  623. minLength = locateIndex == -1 ? minLength : minLength * areaPercent; // minLength需要按areaPercent比例缩小
  624. string log = $"[Log][ScreenLocate Auto] Size: ({Size.x},{Size.y}), 卷积核Size: {conSize}, 最小线段长度: {minLength}";
  625. var allLines = new List<LineIdentified>();
  626. List<Texture2D> LocateTexTemp = new List<Texture2D>();
  627. List<Matrix> ScreenLocateMatList = new List<Matrix>();
  628. var LineCaptureSizes = new Vector[2] { Vector.One * minLength, new Vector(10, 6) };
  629. foreach (var batch in PixelsMultipleBatches.Index())
  630. {
  631. var locateTex = ToLocateTex(PixelsMultipleBatches[batch]);
  632. LocateTexTemp.Add(locateTex);
  633. var ScreenLocateMat = locateTex.Too0Mat(); // 用于获取Lines的Matrix
  634. var lineCount = ZIMIdentifyQuadLSD(
  635. ref allLines,
  636. batch,
  637. ScreenLocateMat.zimIdentifyEdgeGradientAny(conSize),
  638. minLength,
  639. LineCaptureSizes[batch]);
  640. log += $"\r\n识别图片{batch}, 识别到的线段数量为: {lineCount}";
  641. ScreenLocateMatList.Add(ScreenLocateMat);
  642. }
  643. Texture2D ScreenLocateTexture = LocateTexTemp[0]; // for output
  644. // 如果有手动数据,刷新一下Size
  645. QuadManual?.ReSize(new Vector(Size.x, Size.y), ScreenMap.ViewAspectRatioSetting);
  646. // 估算屏幕中点,如果已有手动定位数据,根据现有数据取平均即可,否则从色差计算,ScreenLocateMatList[0]默认是屏幕的黑白色差
  647. Vector AvgPoint = QuadManual != null ? QuadManual.Quad.Centroid : GetAvgPoint(ScreenLocateMatList[0]);
  648. // 过滤得到四边形的四条边,
  649. var (quadLinesSemiAuto, quadLinesAuto) = FilterLines(ScreenLocateMatList, allLines, AvgPoint,
  650. out LineIdentified[] manualLines, out List<LineIdentified> possibleLines,
  651. conSize, conSize, minLength);
  652. #region 全自动识别的结果
  653. List<LineIdentified> LineIdentifiedAuto = new List<LineIdentified>(); // 线段顺序: 下、右、上、左
  654. for (int i = 0; i < 4; i++)
  655. {
  656. if (quadLinesAuto[i] != null)
  657. LineIdentifiedAuto.Add(quadLinesAuto[i]);
  658. }
  659. if (LineIdentifiedAuto.Count == 4) // 判断识别的线段能否拼成屏幕,能拼成则记录
  660. {
  661. var a = LineIdentifiedAuto[0].Line.Intersect(LineIdentifiedAuto[3].Line, false).Value;
  662. var b = LineIdentifiedAuto[0].Line.Intersect(LineIdentifiedAuto[1].Line, false).Value;
  663. var c = LineIdentifiedAuto[2].Line.Intersect(LineIdentifiedAuto[3].Line, false).Value;
  664. var d = LineIdentifiedAuto[1].Line.Intersect(LineIdentifiedAuto[2].Line, false).Value;
  665. QuadAuto = new QuadrilateralInCamera(a, b, c, d, new Vector(Size.x, Size.y));
  666. if (!QuadAuto.IsQuadComplete())
  667. QuadAuto = null;
  668. }
  669. #endregion
  670. #region 半自动识别
  671. List<LineIdentified> LineIdentifiedSemiAuto = new List<LineIdentified>(); // 线段顺序: 下、右、上、左
  672. LastQuadSemiAutoState = new bool[4] { true, true, true, true };
  673. for (int i = 0; i < 4; i++)
  674. {
  675. if (quadLinesSemiAuto[i] != null)
  676. LineIdentifiedSemiAuto.Add(quadLinesSemiAuto[i]);
  677. else if (manualLines != null)
  678. {
  679. LineIdentifiedSemiAuto.Add(manualLines[i]);
  680. LastQuadSemiAutoState[i] = false;
  681. }
  682. }
  683. if (LineIdentifiedSemiAuto.Count == 4) // 判断识别的线段能否拼成屏幕,能拼成则记录
  684. {
  685. var a = LineIdentifiedSemiAuto[0].Line.Intersect(LineIdentifiedSemiAuto[3].Line, false).Value;
  686. var b = LineIdentifiedSemiAuto[0].Line.Intersect(LineIdentifiedSemiAuto[1].Line, false).Value;
  687. var c = LineIdentifiedSemiAuto[2].Line.Intersect(LineIdentifiedSemiAuto[3].Line, false).Value;
  688. var d = LineIdentifiedSemiAuto[1].Line.Intersect(LineIdentifiedSemiAuto[2].Line, false).Value;
  689. QuadSemiAuto = new QuadrilateralInCamera(a, b, c, d, new Vector(Size.x, Size.y));
  690. if (!QuadSemiAuto.IsQuadComplete())
  691. QuadSemiAuto = null;
  692. }
  693. #endregion
  694. // 优先应用自动的结果(也可以在外部手动设置)
  695. if (QuadSemiAuto == null && QuadAuto == null && Screen.QuadInCamera != null) // 如果可能,回退到上一个screen
  696. {
  697. Debug.Log($"<color=yellow>[ScreenIdentification] 本次识别失败,回退到上次的识别结果: {Screen.QuadInCamera}</color>");
  698. quadTemp.Add(Screen.QuadInCamera);
  699. }
  700. else if (QuadAuto != null)
  701. {
  702. Debug.Log($"<color=aqua>[ScreenIdentification] 识别到四边形(全自动): {QuadAuto}</color>");
  703. quadTemp.Add(QuadAuto);
  704. }
  705. else if (QuadSemiAuto != null)
  706. {
  707. Debug.Log($"<color=aqua>[ScreenIdentification] 识别到四边形(半自动): {QuadSemiAuto}</color>");
  708. quadTemp.Add(QuadSemiAuto);
  709. }
  710. #region 绘制 output texture
  711. // 绘制半自动
  712. var ScreenQuadMap = new Matrix(Size, Tiling: true); // 识别的到的屏幕四边形(半自动和自动在一张图上)
  713. foreach (var i in LineIdentifiedSemiAuto.Index())
  714. {
  715. if (LastQuadSemiAutoState[i])
  716. o0Extension.DrawLine(ScreenQuadMap, LineIdentifiedSemiAuto[i].DrawLine, (x, y) => 5, new Geometry2D.Float.Vector(0, 10));
  717. else
  718. o0Extension.DrawLine(ScreenQuadMap, LineIdentifiedSemiAuto[i].DrawLine, (x, y) => 3, new Geometry2D.Float.Vector(0, 6), true);
  719. }
  720. // 绘制全自动
  721. foreach (var i in LineIdentifiedAuto.Index())
  722. o0Extension.DrawLine(ScreenQuadMap, LineIdentifiedAuto[i].DrawLine, (x, y) => 4, new Geometry2D.Float.Vector(0, 4), true);
  723. Texture2D ScreenQuad = ScreenQuadMap.ToTexRGBA(FloatValueToColor);
  724. Texture2D ScreenQuadWithScreen = ScreenQuad.Overlay(ScreenLocateTexture); // 叠加屏幕色差图
  725. // 绘制allLines
  726. var allLinesMap = new Matrix(Size, Tiling: true);
  727. foreach (var l in allLines)
  728. {
  729. if (l.DrawLine != null)
  730. o0Extension.DrawLine(allLinesMap, l.DrawLine, (x, y) => 1, new Geometry2D.Float.Vector(0, 2), true);
  731. }
  732. var allLinesTex = allLinesMap.ToTexRGBA(FloatValueToColor);
  733. ScreenLocate.DebugTexture(1, allLinesTex);
  734. // 还需要输出一张识别结果图,包含干扰线段
  735. var ChoosableLineMap = new Matrix(Size, Tiling: true);
  736. foreach (var l in possibleLines)
  737. {
  738. if (l != null && !quadLinesSemiAuto.Contains(l) && !manualLines.Contains(l))
  739. o0Extension.DrawLine(ChoosableLineMap, l.DrawLine, (x, y) => 1, new Geometry2D.Float.Vector(0, 2), true); // 其他的备选线段
  740. }
  741. foreach (var l in LineIdentifiedSemiAuto)
  742. {
  743. if (l != null)
  744. o0Extension.DrawLine(ChoosableLineMap, l.DrawLine, (x, y) => 5, new Geometry2D.Float.Vector(0, 5)); // 识别的结果
  745. }
  746. if (manualLines != null)
  747. {
  748. foreach (var l in manualLines)
  749. o0Extension.DrawLine(ChoosableLineMap, l.DrawLine, (x, y) => 3, new Geometry2D.Float.Vector(0, 2), true); // 旧的屏幕线段(例如上次手动识别的)
  750. }
  751. Texture2D ChoosableLineTex = ChoosableLineMap.ToTexRGBA(FloatValueToColor);
  752. #endregion
  753. log += $"\r\n屏幕四边形_手动识别{QuadManual != null}\r\n屏幕四边形_半自动识别{QuadSemiAuto != null}\r\n屏幕四边形_全自动识别{QuadAuto != null}";
  754. Debug.Log(log);
  755. // 是否将图片保存到本地
  756. if (ScreenLocate.Main.SaveToggle?.isOn ?? false && ScreenLocate.Main.DebugOnZIMDemo)
  757. {
  758. var FileDirectory = $"Debug_屏幕定位/";
  759. SaveImages(FileDirectory, log, ScreenLocateTexture, allLinesTex, ChoosableLineTex, ScreenQuad);
  760. }
  761. //times.Add(watch.ElapsedMilliseconds);
  762. //Debug.Log("time: " + (times[times.Count - 1] - times[times.Count - 2]));
  763. // opecncv处理, zim
  764. {
  765. //var cvLines = edge.cvHoughLinesP();
  766. //ScreenLocate.DebugTexture(5, cvLines);
  767. //var myLines = Hough.Transform(edgeMat);
  768. //var cvLines = edge.cvLine(myLines);
  769. //ScreenLocate.DebugTexture(5, cvLines);
  770. }
  771. {
  772. ScreenLocate.DebugTexture(2, ScreenLocateTexture);
  773. ScreenLocate.DebugTexture(3, ScreenQuad);
  774. ScreenLocate.DebugTexture(4, ScreenQuadWithScreen);
  775. ScreenLocate.DebugTexture(5, ChoosableLineTex);
  776. }
  777. foreach (var i in LocateTexTemp)
  778. {
  779. if (i != ScreenLocateTexture) // ScreenLocateTexture 由 ScreenLocate.DebugTexture 释放
  780. GameObject.Destroy(i);
  781. }
  782. }
  783. Vector GetAvgPoint(Matrix screenLocateMat)
  784. {
  785. // 加权平均
  786. Vector[] avgPointsColumn = new Vector[screenLocateMat.Size.x];
  787. float[] valueSumsColumn = new float[screenLocateMat.Size.x];
  788. Parallel.For(0, screenLocateMat.Size.x, i =>
  789. {
  790. for (int j = 0; j < screenLocateMat.Size.y; j++)
  791. {
  792. var value = screenLocateMat[i, j];
  793. valueSumsColumn[i] += value;
  794. avgPointsColumn[i] += new Vector(i, j) * value;
  795. }
  796. });
  797. Vector avgPoint = Vector.Zero;
  798. var valueSum = 0f;
  799. for (int i = 0; i < screenLocateMat.Size.x; i++)
  800. {
  801. avgPoint += avgPointsColumn[i];
  802. valueSum += valueSumsColumn[i];
  803. }
  804. avgPoint /= valueSum;
  805. return avgPoint;
  806. }
  807. // 返回查找到的线段数量,0是查找失败
  808. int ZIMIdentifyQuadLSD(ref List<LineIdentified> allLines, int batch, (Matrix edgeMat, Matrix edgeDirMat) edgeGradient,
  809. float minLength, Vector LineCaptureSize)
  810. {
  811. var l = edgeGradient.edgeMat.IdentifyLineLSD(edgeGradient.edgeDirMat, minLength, 50, LineCaptureSize);
  812. if (l == null || l.Count == 0)
  813. return 0;
  814. allLines.AddRange(l.Select((i) => new LineIdentified(batch, i)));
  815. return l.Count;
  816. }
  817. // 返回四边形的四条边(半自动、全自动),List长度一定是4 (如果没有识别到就是null),且线段顺序是: 下、右、上、左
  818. (List<LineIdentified>, List<LineIdentified>) FilterLines(List<Matrix> screenLocateMatList, List<LineIdentified> allLines, Vector avgPoint,
  819. out LineIdentified[] manualLines, out List<LineIdentified> possibleLines, float conSize, float gradientLength, float minLength = 100)
  820. {
  821. //Debug.Log("[IdentifyLineLSD] lines.Count: " + lines.Count);
  822. var offset = new Vector((conSize - 1) / 2, (conSize - 1) / 2);
  823. // LSD计算得到的矩阵尺寸较小(因为卷积),这里必须进行位移
  824. for (int i = 0; i < allLines.Count; i++)
  825. allLines[i].Offset(offset);
  826. // 筛掉椭圆框外的线段(超出一半会筛掉)
  827. var innerLines = new List<LineIdentified>();
  828. for (int i = 0; i < allLines.Count; i++)
  829. {
  830. List<Vector> InArea = new List<Vector>();
  831. var dir = (allLines[i].Line.B - allLines[i].Line.A) / 4;
  832. var points = new Vector[5] { allLines[i].Line.A, allLines[i].Line.A + dir, allLines[i].Line.A + dir * 2f, allLines[i].Line.A + dir * 3f, allLines[i].Line.B }; // A点、中间的点、B点
  833. for (int pI = 0; pI < points.Length; pI++)
  834. {
  835. if (!ScreenLocate.Main.ScreenPixelCheaker.OutArea2D(points[pI], Size))
  836. InArea.Add(points[pI]);
  837. }
  838. if (InArea.Count < 2) // 少于2个点在内部
  839. continue;
  840. else if (InArea.Count < points.Length) // 不完全在内部
  841. allLines[i].DrawLine = new Line(InArea.First(), InArea.Last()); // 将部分线条设置为drawline,用于下一步的绘制
  842. else // 线段全部在椭圆内
  843. allLines[i].DrawLine = allLines[i].Line;
  844. innerLines.Add(allLines[i]);
  845. }
  846. // 角度阈值,用来判断线段的梯度方向是否指向屏幕中心(avgPoint)
  847. var avaAngleHalf = 75f;
  848. // 沿直线计算综合梯度(梯度乘以长度系数,再乘以距离系数), distanceRatio是实际距离除以最大距离
  849. float estimateGradient(LineIdentified line, float distanceRatio)
  850. {
  851. var dir = (line.Line.B - line.Line.A).Normalized;
  852. var vertical = new Vector(-dir.y, dir.x) * (gradientLength / 2);
  853. var step = 2;
  854. var ll = line.Line.Length;
  855. var lg = new List<float>();
  856. for (int i = 0; i <= ll; i += step)
  857. {
  858. var point = line.Line.A + dir * i;
  859. var ga = point + vertical;
  860. var gb = point - vertical;
  861. lg.Add(screenLocateMatList[line.Batch][(int)ga.x, (int)ga.y] - screenLocateMatList[line.Batch][(int)gb.x, (int)gb.y]);
  862. }
  863. //float e = (float)Math.Sqrt(Math.Ceiling(line.Line.Length / minLength)); // 长度系数,筛选时梯度更大、长度更长的线段更优
  864. float d = (1.5f - distanceRatio) * 2; // 距离系数,距离越近,系数越大
  865. return d * Math.Abs(lg.Mean());
  866. }
  867. // 根据线段梯度的角度,判断是不是屏幕的边,out index代表是哪条边(顺序是: 下、右、上、左)
  868. bool isScreenLine(LineIdentified line, out int index)
  869. {
  870. var a = (avgPoint - (line.Line.A + line.Line.B) / 2).DegreeToXAxis();
  871. //Debug.Log(a + ", " + gradient + ", " + sum);
  872. index = -1;
  873. if (Math.Abs(a - line.GradientDegree) < avaAngleHalf || Math.Abs(a - 360 - line.GradientDegree) < avaAngleHalf || Math.Abs(a + 360 - line.GradientDegree) < avaAngleHalf)
  874. {
  875. if (line.GradientDegree > 45 && line.GradientDegree < 135) // 下
  876. index = 0;
  877. else if (line.GradientDegree > 135 && line.GradientDegree < 225) // 右
  878. index = 1;
  879. else if (line.GradientDegree > 225 && line.GradientDegree < 315) // 上
  880. index = 2;
  881. else
  882. index = 3;
  883. return true;
  884. }
  885. return false;
  886. }
  887. // 下、右、上、左, 半自动和自动
  888. var quadLinesSemiAuto = new List<(float, LineIdentified)>[4] { new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>() };
  889. var quadLinesAuto = new List<(float, LineIdentified)>[4] { new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>() };
  890. possibleLines = new List<LineIdentified>();
  891. // 如果已有手动定位数据,根据现有数据筛选线条(半自动)
  892. manualLines = null;
  893. if (QuadManual != null)
  894. {
  895. Debug.Log($"[IdentifyLineLSD] 根据已有定位数据做筛选, QuadManual: {QuadManual}");
  896. manualLines = QuadManual.GetLines().Select((i) => new LineIdentified(0, i, 0, 0, true)).ToArray();
  897. var calibration = ScreenLocate.Main.ReDoLocateCalibrationRatio * Size.y;
  898. var avgPointCross = manualLines.Select((i) => i.Line.LineCrossWithPoint(avgPoint)).ToArray(); // 对于平均点的corss值
  899. var avgPointPedal = manualLines.Select((i) => o0Extension.PointPedal(i.Line, avgPoint, out _)).ToArray(); // 当前定位的垂足,下、右、上、左
  900. foreach (var line in innerLines)
  901. {
  902. // 筛选条件:1-梯度方向匹配,2-垂足的距离足够近, 3-线段的AB点均在旧线段外部, 4-新的线段的中点,到旧线段的垂足,要在旧线段内
  903. if (isScreenLine(line, out int index))
  904. {
  905. var distanceToOld = (o0Extension.PointPedal(line.Line, avgPoint, out _) - avgPointPedal[index]).Length;
  906. if (distanceToOld < calibration &&
  907. manualLines[index].Line.LineCrossWithPoint(line.Line.A) * avgPointCross[index] <= 0 &&
  908. manualLines[index].Line.LineCrossWithPoint(line.Line.B) * avgPointCross[index] <= 0)
  909. {
  910. var middleToOldLine = o0Extension.PointPedal(manualLines[index].Line, (line.Line.A + line.Line.B) / 2, out bool inLineSegment);
  911. if (inLineSegment)
  912. {
  913. quadLinesSemiAuto[index].Add((estimateGradient(line, distanceToOld / calibration), line));
  914. possibleLines.Add(line);
  915. }
  916. }
  917. }
  918. }
  919. }
  920. // 全自动
  921. foreach (var line in allLines)
  922. {
  923. if (isScreenLine(line, out int index))
  924. {
  925. if (line.Batch < 1) // 全自动只处理第一张图,默认是色差图
  926. {
  927. quadLinesAuto[index].Add((estimateGradient(line, 1), line));
  928. }
  929. }
  930. }
  931. var resultSemiAuto = new LineIdentified[4];
  932. var resultAuto = new LineIdentified[4];
  933. for (int i = 0; i < 4; i++)
  934. {
  935. if (quadLinesSemiAuto[i].Count > 0)
  936. resultSemiAuto[i] = quadLinesSemiAuto[i].Max((a, b) => a.Item1.CompareTo(b.Item1)).Item2;
  937. if (quadLinesAuto[i].Count > 0)
  938. resultAuto[i] = quadLinesAuto[i].Max((a, b) => a.Item1.CompareTo(b.Item1)).Item2;
  939. }
  940. return (resultSemiAuto.ToList(), resultAuto.ToList());
  941. }
  942. void SaveImages(string FileDirectory, string log,
  943. Texture2D ScreenLocateTex, Texture2D allLinesTex, Texture2D ChoosableLineTex, Texture2D ScreenQuadTex)
  944. {
  945. if (!Directory.Exists(FileDirectory))
  946. Directory.CreateDirectory(FileDirectory);
  947. var time = DateTime.Now.ToString("yyyyMMdd_HHmmss");
  948. var pngData = ScreenLocate.Main.OutputTextures[7]?.EncodeToPNG();
  949. if (pngData != null)
  950. File.WriteAllBytes($"{FileDirectory}{time}A屏幕原图.png", pngData);
  951. var pngData1 = ScreenLocateTex?.EncodeToPNG();
  952. if (pngData1 != null)
  953. File.WriteAllBytes($"{FileDirectory}{time}B黑白色差.png", pngData1);
  954. var pngData2 = allLinesTex?.EncodeToPNG();
  955. if (pngData2 != null)
  956. File.WriteAllBytes($"{FileDirectory}{time}C全部识别线段_半自动.png", pngData2);
  957. var pngData3 = ChoosableLineTex?.EncodeToPNG();
  958. if (pngData3 != null)
  959. File.WriteAllBytes($"{FileDirectory}{time}D备选线段_半自动.png", pngData3);
  960. var pngData4 = ScreenQuadTex?.EncodeToPNG();
  961. if (pngData4 != null)
  962. File.WriteAllBytes($"{FileDirectory}{time}E识别结果.png", pngData4);
  963. Debug.Log($"<color=aqua>({time}) 屏幕识别图片保存至:程序根目录/{FileDirectory}</color>");
  964. log +=
  965. $"\r\n屏幕原图保存{pngData != null}, " +
  966. $"\r\n黑白色差保存{pngData1 != null}, " +
  967. $"\r\n全部识别线段(半自动)保存{pngData2 != null}, " +
  968. $"\r\n备选线段(半自动)保存{pngData3 != null}, " +
  969. $"\r\n识别结果保存{pngData4 != null}";
  970. File.WriteAllText($"{FileDirectory}{time}屏幕自动定位_日志.log", log);
  971. }
  972. }
  973. }