ScreenIdentification.cs 49 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107
  1. #define ENABLE_LOG
  2. using o0.Geometry2D.Float;
  3. using o0.Num;
  4. using System;
  5. using System.Collections.Generic;
  6. using System.IO;
  7. using System.Linq;
  8. using System.Threading.Tasks;
  9. using UnityEngine;
  10. using UnityEngine.UIElements;
  11. using ZIM;
  12. using ZIM.Unity;
  13. namespace o0.Project
  14. {
  15. public partial class ScreenIdentification
  16. {
  17. private const string TAG = "ScreenIdentification#";
  18. // LocateAreaData表示每次屏幕的色差变化的区域,可能有多次。通过设置LocateSingleStep可调整为仅识别一次色差
  19. static Rect[][] LocateAreaData = new Rect[][] {
  20. new Rect[] { new Rect(0f, 0f, 0.3f, 0.3f), new Rect(0f, 0f, 0.4f, 0.4f), new Rect(0f, 0f, 0.5f, 0.5f) },
  21. new Rect[] { new Rect(0.7f, 0f, 0.3f, 0.3f), new Rect(0.6f, 0f, 0.4f, 0.4f), new Rect(0.5f, 0f, 0.5f, 0.5f) },
  22. new Rect[] { new Rect(0f, 0.7f, 0.3f, 0.3f), new Rect(0f, 0.6f, 0.4f, 0.4f), new Rect(0f, 0.5f, 0.5f, 0.5f) },
  23. new Rect[] { new Rect(0.7f, 0.7f, 0.3f, 0.3f), new Rect(0.6f, 0.6f, 0.4f, 0.4f), new Rect(0.5f, 0.5f, 0.5f, 0.5f) }
  24. };
  25. //static Rect[][] LocateAreaData = new Rect[][] {
  26. // new Rect[] { new Rect(0f, 0f, 0.3f, 0.3f), new Rect(0f, 0f, 0.4f, 0.4f), new Rect(0f, 0f, 0.5f, 0.5f), new Rect(0f, 0f, 0.6f, 0.6f) },
  27. // new Rect[] { new Rect(0.7f, 0f, 0.3f, 0.3f), new Rect(0.6f, 0f, 0.4f, 0.4f), new Rect(0.5f, 0f, 0.5f, 0.5f), new Rect(0.4f, 0f, 0.6f, 0.6f) },
  28. // new Rect[] { new Rect(0f, 0.7f, 0.3f, 0.3f), new Rect(0f, 0.6f, 0.4f, 0.4f), new Rect(0f, 0.5f, 0.5f, 0.5f), new Rect(0f, 0.4f, 0.6f, 0.6f) },
  29. // new Rect[] { new Rect(0.7f, 0.7f, 0.3f, 0.3f), new Rect(0.6f, 0.6f, 0.4f, 0.4f), new Rect(0.5f, 0.5f, 0.5f, 0.5f), new Rect(0.4f, 0.4f, 0.6f, 0.6f) }
  30. //};
  31. //static bool LocateSingleStep = false;
  32. static bool LocateSingleStep = true;
  33. public Geometry2D.Vector<int> Size => ScreenLocate.Main.CameraSize;
  34. public QuadrilateralInCamera QuadManual;
  35. public QuadrilateralInCamera QuadAuto; // 全自动,可以给用户选择(赋值给Screen.QuadInCamera即生效)
  36. public QuadrilateralInCamera QuadSemiAuto; // 半自动,可以给用户选择(赋值给Screen.QuadInCamera即生效)
  37. public ScreenMap Screen; // 识别到的屏幕,用于执行透视变换
  38. int capture = 0;
  39. int delay = 0;
  40. int maxCapture;
  41. int maxDelay;
  42. Geometry.Vector<float>[] ScreenBlackTexture;
  43. Geometry.Vector<float>[] ScreenWhiteTexture;
  44. int locateIndex = -1;
  45. readonly List<Rect> locateArea = new List<Rect> {
  46. new Rect(0f, 0f, 0.5f, 0.5f), new Rect(0.5f, 0f, 0.5f, 0.5f), new Rect(0f, 0.5f, 0.5f, 0.5f), new Rect(0.5f, 0.5f, 0.5f, 0.5f)
  47. }; // 屏幕显示白色的区域大小
  48. float areaPercent => locateArea[locateIndex].size.x; // 当前白色区域的占比
  49. int areaSelected = -1; // 选择哪个区域,顺序与Quadrilateral对应
  50. readonly List<float> sumTemp = new List<float>();
  51. readonly List<QuadrilateralInCamera> quadTemp = new List<QuadrilateralInCamera>();
  52. //public ScreenIdentification(WebCamTexture texture)
  53. //{
  54. // Size = new Geometry2D.Vector<int>(texture.width, texture.height);
  55. // Screen = new ScreenMap();
  56. //}
  57. public static UnityEngine.Color FloatValueToColor(float i)
  58. {
  59. return i switch
  60. {
  61. 1 => UnityEngine.Color.yellow,
  62. 2 => new UnityEngine.Color(0,1,1,1),
  63. 3 => UnityEngine.Color.green,
  64. 4 => UnityEngine.Color.white,
  65. 5 => UnityEngine.Color.red,
  66. _ => UnityEngine.Color.black,
  67. };
  68. }
  69. public ScreenIdentification()
  70. {
  71. Screen = new ScreenMap();
  72. //OnLocateScreenEnter += () => Application.targetFrameRate = 30; // 固定识别的帧率,确保摄像机拍到正确的画面
  73. //OnLocateScreenEnd += () => Application.targetFrameRate = 60;
  74. }
  75. public void SetScreenQuad(QuadrilateralInCamera quad) => Screen.QuadInCamera = quad;
  76. // 上一次半自动识别的情况,false代表这条边识别失败,线段顺序: 下、右、上、左
  77. public bool[] LastQuadSemiAutoState;
  78. public event Action OnLocateScreenEnter;
  79. public event Action OnLocateScreenEnd;
  80. public bool bStartLocateScreen { get; set; } = false;//是否进行捕获
  81. public bool SelectScreenAfterLocate(ScreenLocate.ScreenIdentificationTag tag)
  82. {
  83. QuadrilateralInCamera target = GetScreenAfterLocate(tag);
  84. if (target == null)
  85. return false;
  86. Debug.Log($"<color=aqua>[ScreenIdentification] 选择已识别到的屏幕({Enum.GetName(typeof(ScreenLocate.ScreenIdentificationTag), tag)}), {target}</color>");
  87. SetScreenQuad(target);
  88. return true;
  89. }
  90. public QuadrilateralInCamera GetScreenAfterLocate(ScreenLocate.ScreenIdentificationTag tag)
  91. {
  92. return tag switch
  93. {
  94. ScreenLocate.ScreenIdentificationTag.Manual => QuadManual,
  95. ScreenLocate.ScreenIdentificationTag.SemiAuto => QuadSemiAuto,
  96. ScreenLocate.ScreenIdentificationTag.Auto => QuadAuto,
  97. _ => null
  98. };
  99. }
  100. // 自动识别开始的入口
  101. public void LocateScreen(int Capture = 30, int Delay = 30) //数值单位是frame
  102. {
  103. if (ScreenLocate.Main.DebugScreenImages.Count != 0 && ScreenLocate.Main.DebugOnZIMDemo) // 这段仅用于测试图片
  104. {
  105. ScreenLocate.Main.CameraSize = new Geometry2D.Vector<int>(ScreenLocate.Main.DebugScreenImages[0].width, ScreenLocate.Main.DebugScreenImages[0].height);
  106. DebugImage(ScreenLocate.Main.DebugScreenImages);
  107. Screen.QuadInCamera = quadTemp[0];
  108. ScreenLocate.SetScreen(null);
  109. ScreenLocate.Main.ShowScreen(ScreenLocate.Main.ScreenQuad, Screen.QuadInCamera);
  110. delay = 0;
  111. capture = 0;
  112. ScreenWhiteTexture = null;
  113. ScreenBlackTexture = null;
  114. locateIndex = -1;
  115. areaSelected = -1;
  116. quadTemp.Clear();
  117. sumTemp.Clear();
  118. ScreenLocate.Main.DebugScreenImages.Clear();
  119. return;
  120. }
  121. delay = Math.Max(Delay, 5);
  122. capture = Math.Max(Capture, 5);
  123. maxDelay = Delay;
  124. maxCapture = Capture;
  125. ScreenLocate.SetScreen(new Rect(0f, 0f, 1f, 1f), UnityEngine.Color.black);
  126. //ScreenLocate.SetScreen(new Rect(0f, 0f, 0.6f, 0.6f), UnityEngine.Color.white);
  127. //bStartLocateScreen = false;
  128. OnLocateScreenEnter?.Invoke();
  129. }
  130. /// <summary>
  131. /// 开始进行捕获
  132. /// 初始化了两个数据 capture 和 delay
  133. /// </summary>
  134. /// <returns></returns>
  135. public bool isInitLocateScreen()
  136. {
  137. return capture != 0 && delay != 0;
  138. }
  139. void DebugImage(List<Texture2D> images)
  140. {
  141. QuadrilateralFit(images);
  142. //var watch = new System.Diagnostics.Stopwatch();
  143. //watch.Start();
  144. //var times = new List<double>() { 0.0 };
  145. #if (!NDEBUG && DEBUG && ENABLE_LOG)
  146. Console.WriteLine($"{TAG} quadTemp.Count:{quadTemp.Count}");
  147. #endif
  148. if (quadTemp.Count > 0)
  149. {
  150. ScreenLocate.Main.ShowScreen(ScreenLocate.Main.outputRawImages[4].transform.GetChild(0) as RectTransform, quadTemp[0]);
  151. // 透视变换
  152. // var srcWidth = LocateLightedRedTex.width;
  153. // var transformWidth = (int)((quad.B.x - quad.A.x + quad.D.x - quad.C.x) / 2);
  154. // var transformHeight = (int)((quad.C.y - quad.A.y + quad.D.y - quad.B.y) / 2);
  155. // var transformTex = new Texture2D(transformWidth, transformHeight);
  156. // var pt = new ZIMPerspectiveTransform(new OrdinalQuadrilateral(new Vector(0, 0), new Vector(transformWidth, 0), new Vector(0, transformHeight), new Vector(transformWidth, transformHeight)), quad);
  157. // var dstPixel = new UnityEngine.Color[transformWidth * transformHeight];
  158. // var srcPixel = LocateLightedRedTex.GetPixels();
  159. // Parallel.For(0, transformWidth, (x) =>
  160. // {
  161. // for (int y = 0; y < transformHeight; y++)
  162. // {
  163. // var index = y * transformWidth + x;
  164. // var sampleCoord = pt.TransformRound(x, y);
  165. // dstPixel[index] = srcPixel[sampleCoord.y * srcWidth + sampleCoord.x];
  166. // }
  167. // });
  168. // transformTex.SetPixels(dstPixel);
  169. // transformTex.Apply();
  170. // //ScreenLocate.DebugTexture(1, transformTex);
  171. //#if (!NDEBUG && DEBUG && ENABLE_LOG)
  172. // Console.WriteLine($"{TAG} ScreenLocate.DebugTexture 1:{transformTex.GetNativeTexturePtr()}");
  173. //#endif
  174. }
  175. //times.Add(watch.ElapsedMilliseconds);
  176. //Debug.Log("time: " + (times[times.Count - 1] - times[times.Count - 2]));
  177. }
  178. public void NextScreen()
  179. {
  180. // 只识别一次色差变化
  181. if (LocateSingleStep && areaSelected == -1)
  182. {
  183. LocateAreaData = new Rect[][] { new Rect[] { new Rect(0, 0, 1f, 1f) } };
  184. locateIndex = 3;
  185. areaSelected = 0;
  186. locateArea.AddRange(LocateAreaData[0]);
  187. }
  188. // index从-1开始
  189. locateIndex++;
  190. if (locateIndex < locateArea.Count) // 依次点亮屏幕区域
  191. {
  192. ScreenLocate.SetScreen(locateArea[locateIndex], UnityEngine.Color.white);
  193. delay = maxDelay;
  194. capture = maxCapture;
  195. }
  196. else // 退出屏幕黑白控制
  197. {
  198. ScreenLocate.SetScreen(null);
  199. ScreenLocate.Main.ShowScreen(ScreenLocate.Main.ScreenQuad, Screen.QuadInCamera);
  200. Reset();
  201. }
  202. }
  203. // 清除记录的屏幕识别数据(手动、自动等)
  204. public void ClearQuadCache()
  205. {
  206. SetScreenQuad(null);
  207. QuadManual = null;
  208. QuadSemiAuto = null;
  209. QuadAuto = null;
  210. }
  211. public void Reset()
  212. {
  213. // bStartLocateScreen = false;
  214. delay = 0;
  215. capture = 0;
  216. ScreenWhiteTexture = null;
  217. ScreenBlackTexture = null;
  218. locateIndex = -1;
  219. areaSelected = -1;
  220. if (locateArea.Count > 4)
  221. locateArea.RemoveRange(4, LocateAreaData[0].Length);
  222. quadTemp.Clear();
  223. sumTemp.Clear();
  224. }
  225. public void CaptureBlack(Texture2D cam)
  226. {
  227. if (ScreenBlackTexture == null)
  228. ScreenBlackTexture = new Geometry.Vector<float>[Size.x * Size.y];
  229. var pixel = cam.GetPixels();
  230. Parallel.For(0, Size.x * Size.y, i =>
  231. {
  232. var ip = pixel[i];
  233. ScreenBlackTexture[i] += new Geometry.Vector<float>(ip.r / maxCapture, ip.g / maxCapture, ip.b / maxCapture);
  234. });
  235. }
  236. public void CaptureWhite(Texture2D cam)
  237. {
  238. if (ScreenWhiteTexture == null)
  239. ScreenWhiteTexture = new Geometry.Vector<float>[Size.x * Size.y];
  240. var pixel = cam.GetPixels();
  241. Parallel.For(0, Size.x * Size.y, i =>
  242. {
  243. var ip = pixel[i];
  244. ScreenWhiteTexture[i] += new Geometry.Vector<float>(ip.r / maxCapture, ip.g / maxCapture, ip.b / maxCapture);
  245. });
  246. }
  247. public void CaptureStay(Texture2D cam)
  248. {
  249. if (locateIndex == -1) // 屏幕黑色
  250. {
  251. CaptureBlack(cam);
  252. }
  253. else // 屏幕部分为白色
  254. {
  255. CaptureWhite(cam);
  256. }
  257. }
  258. public void CaptureEnd()
  259. {
  260. //Debug.Log("locateIndex: " + locateIndex + ", quad: " + quadTemp.Count);
  261. if (locateIndex == -1)
  262. return;
  263. if (locateIndex < 4)
  264. {
  265. sumTemp.Add(GetBrightness());
  266. ScreenWhiteTexture = null;
  267. // 选择亮度差最大的区域
  268. if (locateIndex == 3)
  269. {
  270. areaSelected = sumTemp.MaxIndex();
  271. locateArea.AddRange(LocateAreaData[areaSelected]);
  272. }
  273. }
  274. else if (locateIndex >= 4 && locateIndex < locateArea.Count - 1)
  275. {
  276. QuadrilateralFit();
  277. ScreenWhiteTexture = null;
  278. }
  279. else
  280. {
  281. QuadrilateralFit();
  282. if (quadTemp.Count != LocateAreaData[0].Length)
  283. {
  284. Debug.Log($"<color=yellow>[ScreenIdentification] 拟合四边形失败, quadTemp.Count: {quadTemp.Count}</color>");
  285. }
  286. else if (quadTemp.Count == 1)
  287. {
  288. SetScreenQuad(quadTemp[0]);
  289. Debug.Log($"[ScreenIdentification] 拟合成功,识别数据: {Screen.QuadInCamera}");
  290. }
  291. else
  292. {
  293. // Debug.Log($"拟合四边形 2 , quadTemp.Count: {quadTemp.Count}");
  294. // 线性拟合
  295. var xValue = new List<float>() { 0 };
  296. var predicts = new List<Vector>();
  297. foreach (var i in LocateAreaData[0])
  298. xValue.Add(i.size.x);
  299. Vector baseVertex = Vector.Zero; // x==0 时的点
  300. {
  301. foreach (var q in quadTemp)
  302. {
  303. baseVertex += q.Quad[areaSelected];
  304. }
  305. baseVertex /= quadTemp.Count;
  306. }
  307. double rs = 0.0;
  308. for (int i = 0; i < 4; i++)
  309. {
  310. if (i == areaSelected)
  311. {
  312. predicts.Add(baseVertex);
  313. }
  314. else
  315. {
  316. var yValue = new List<Vector>() { baseVertex };
  317. foreach (var q in quadTemp)
  318. {
  319. yValue.Add(q.Quad[i]);
  320. }
  321. var lr = LinerRegression1D.Fit(2, xValue.ToArray(), yValue.ToArray());
  322. rs += lr.RSquared / 3;
  323. predicts.Add(lr.Predict<Vector>(1));
  324. }
  325. }
  326. SetScreenQuad(new QuadrilateralInCamera(predicts, new Vector(Size.x, Size.y)));
  327. Debug.Log($"[ScreenIdentification] 拟合成功,RSquared: {rs}, Quad: {Screen.QuadInCamera}");
  328. //if (rs < 0.8) Screen.Quad = null;
  329. }
  330. OnLocateScreenEnd?.Invoke();
  331. }
  332. }
  333. public bool Update(Texture2D cam)
  334. {
  335. //if (!bStartLocateScreen) return false;
  336. if (delay != 0)
  337. {
  338. //ScreenLocate.Main.CreateUVCTexture2DFocusSizeIfNeeded(1280, 720);
  339. delay--;
  340. if (delay == 0)
  341. {
  342. ScreenLocate.Main.CameraSize = new Geometry2D.Vector<int>(cam.width, cam.height); // 记录当前的分辨率
  343. Debug.Log("[ScreenIdentification] 采样纹理,记录采样分辨率: [" + Size.x + ", " + Size.y + "]");
  344. }
  345. return true;
  346. }
  347. if (capture != 0)
  348. {
  349. //ScreenLocate.Main.CreateUVCTexture2DFocusSizeIfNeeded(1280, 720);
  350. CaptureStay(cam);
  351. capture--;
  352. if (capture == 0)
  353. {
  354. CaptureEnd();
  355. NextScreen();
  356. }
  357. return true;
  358. }
  359. return false;
  360. #region Old
  361. /*
  362. if (delay != 0)
  363. {
  364. delay--;
  365. return true;
  366. }
  367. if (capture != 0)
  368. {
  369. capture--;
  370. if (ScreenBlackTexture == null)
  371. ScreenBlackTexture = new Geometry.Vector<float>[Size.x * Size.y];
  372. var pixel = cam.GetPixels();
  373. Parallel.For(0, Size.x * Size.y, i =>
  374. {
  375. var ip = pixel[i];
  376. ScreenBlackTexture[i] += new Geometry.Vector<float>(ip.r, ip.g, ip.b);
  377. });
  378. if (capture == 0)
  379. ScreenLocate.SetScreen(UnityEngine.Color.black);
  380. return true;
  381. }
  382. if (delay != 0)
  383. {
  384. delay--;
  385. return true;
  386. }
  387. if (capture != 0)
  388. {
  389. capture--;
  390. if (ScreenWhiteTexture == null)
  391. ScreenWhiteTexture = new Geometry.Vector<float>[Size.x * Size.y];
  392. var pixel = cam.GetPixels();
  393. Parallel.For(0, Size.x * Size.y, i =>
  394. {
  395. var ip = pixel[i];
  396. ScreenWhiteTexture[i] += new Geometry.Vector<float>(ip.r, ip.g, ip.b);
  397. });
  398. if (capture == 0)
  399. ScreenLocate.SetScreen(UnityEngine.Color.black);
  400. return true;
  401. }
  402. if (delay != 0)
  403. {
  404. delay--;
  405. return true;
  406. }
  407. if (capture != 0)
  408. {
  409. capture--;
  410. var pixel = cam.GetPixels();
  411. Parallel.For(0, Size.x * Size.y, i =>
  412. {
  413. var ip = pixel[i];
  414. ScreenWhiteTexture[i] -= new Geometry.Vector<float>(ip.r, ip.g, ip.b);
  415. });
  416. if (capture == 0)
  417. {
  418. ScreenLocate.SetScreen(null);
  419. UnityEngine.Color[] newPixel = new UnityEngine.Color[Size.x * Size.y];
  420. Parallel.For(0, Size.x * Size.y, i => {
  421. var pi = ScreenWhiteTexture[i] /= capture;
  422. newPixel[i] = new UnityEngine.Color(pi.x, pi.y, pi.z);
  423. });
  424. //读取数据
  425. //{
  426. // var fileName = "3.bin";
  427. // ScreenLocateTexture = $"2023 04 16 厦门测试数据/{fileName}".FileReadByte<Vector<float>[]>();
  428. // Debug.Log($"Read {fileName}");
  429. // Parallel.For(0, Size.x * Size.y, i =>
  430. // {
  431. // var pi = ScreenLocateTexture[i];
  432. // newPixel[i] = new UnityEngine.Color(pi.x, pi.y, pi.z);
  433. // });
  434. //}
  435. var ScreenLocateTex = new Texture2D(Size.x, Size.y);
  436. ScreenLocateTex.SetPixels(newPixel);
  437. ScreenLocateTex.Apply();
  438. //ScreenLocate.DebugTexture(2, ScreenLocateTex);
  439. var ScreenLocateTexLighted = ScreenLocateTex.AutoLight(10);
  440. //ScreenLocate.DebugTexture(2, ScreenLocateTexLighted);
  441. //var FileSavePath = Application.persistentDataPath + "/ScreenLocateTexture.bin";
  442. bool Save = ScreenLocate.Main.SaveToggle.isOn;
  443. string time;
  444. if (Save)
  445. {
  446. time = DateTime.Now.ToString("yyyyMMdd_HHmmss");
  447. var FileSavePath = $"屏幕定位数据{time}.bin";
  448. FileSavePath.FileWriteByte(ScreenWhiteTexture);
  449. var bytes = ScreenLocateTexLighted.EncodeToPNG();
  450. File.WriteAllBytes($"屏幕定位数据{time}.png", bytes);
  451. Debug.Log("ScreenLocateTexture Saved To: " + FileSavePath);
  452. }
  453. var ScreenLocateTexR = ScreenLocateTexLighted.ToRGB(ColorChannel.Red);
  454. var ScreenLocateTexG = ScreenLocateTexLighted.ToRGB(ColorChannel.Green);
  455. var ScreenLocateTexB = ScreenLocateTexLighted.ToRGB(ColorChannel.Blue);
  456. ScreenLocate.DebugTexture(2, ScreenLocateTexR);
  457. //ScreenLocate.DebugTexture(4, ScreenLocateTexG);
  458. //ScreenLocate.DebugTexture(5, ScreenLocateTexB);
  459. var watch = new System.Diagnostics.Stopwatch();
  460. watch.Start();
  461. var times = new List<double>() { 0.0 };
  462. var ScreenLocateTexLightedMat = ScreenLocateTexLighted.Too0Mat();
  463. //var ScreenLocateTexLightedMat = texture.Too0Mat();
  464. //var (edge, edgeDir) = ScreenLocateTexLightedMat.IdentifyEdge();
  465. var (edge, edgeDir) = ScreenLocateTexLightedMat.zimIdentifyEdgeGradientAny(15);
  466. //ScreenLocate.DebugTexture(4, ScreenLocateTexLighted.Too0Mat().IdentifyEdgeGradient().ToTex());
  467. //ScreenLocate.DebugTexture(4, edge.ToTex());
  468. var quadLines = ScreenLocateTexLightedMat.IdentifyQuadLSD(edge, edgeDir, out List<Line> lightLines, 30);
  469. var drawLineMap = new MatrixF2D(edge..Size.x, edge.Size.y);
  470. int lineCount = 0;
  471. foreach (var l in quadLines)
  472. {
  473. if (l != null)
  474. {
  475. o0Extension.DrawLine(drawLineMap.DrawLine(l, (x, y) => 1, new Geometry2D.Float.Vector(0, 10));
  476. lineCount++;
  477. }
  478. }
  479. if (lineCount == 4)
  480. {
  481. var a = quadLines[0].Intersect(quadLines[3], false).Value;
  482. var b = quadLines[0].Intersect(quadLines[1], false).Value;
  483. var c = quadLines[2].Intersect(quadLines[3], false).Value;
  484. var d = quadLines[1].Intersect(quadLines[2], false).Value;
  485. Quad = new Quadrilateral(a, b, c, d);
  486. if (!Quad.IsInScreen(ScreenLocate.Main.WebCamera.Size))
  487. Quad = null;
  488. }
  489. ScreenLocate.Main.ShowScreen(Quad);
  490. //var lines = edge.IdentifyLineLSD(edgeDir, 100);
  491. ////var lines = ScreenLocateTexLightedMat.IdentifyLineLSD();
  492. //var drawLineMap = new MatrixF2D(edge..Size.x, edge.Size.y);
  493. //var returnMaxLines = lines.Sub(0, 10);
  494. //foreach (var (line, sum, gradient) in returnMaxLines)
  495. // o0Extension.DrawLine(drawLineMap.DrawLine(line, (x, y) => 1, new Geometry2D.Float.Vector(0, 10));
  496. ScreenLocate.DebugTexture(3, drawLineMap.ToTex());
  497. //{
  498. // var bytes = drawLineMap.ToTex().EncodeToPNG();
  499. // File.WriteAllBytes($"屏幕定位数据DrawLineMap.png", bytes);
  500. //}
  501. times.Add(watch.ElapsedMilliseconds);
  502. Debug.Log("time: " + (times[times.Count - 1] - times[times.Count - 2]));
  503. //ScreenLocate.DebugTexture(5, edge.IdentifyLine(edgeDir).ToTex());
  504. //ScreenLocate.DebugTexture(4, ScreenLocateTexLighted.Too0Mat().IdentifyEdgeGradientX().ToTex());
  505. //ScreenLocate.DebugTexture(5, ScreenLocateTexLighted.Too0Mat().IdentifyEdgeGradientY().ToTex());
  506. //var convolutionLighted2 = ScreenLocateTexLighted.Too0Mat().IdentifyEdgeVariance().ToTex();
  507. // opecncv处理
  508. // zim
  509. {
  510. //var cvLines = edge.cvHoughLinesP();
  511. //ScreenLocate.DebugTexture(5, cvLines);
  512. //var myLines = Hough.Transform(edgeMat);
  513. //var cvLines = edge.cvLine(myLines);
  514. //ScreenLocate.DebugTexture(5, cvLines);
  515. }
  516. UnityEngine.Object.Destroy(ScreenLocateTex);
  517. //ScreenLocate.DebugTexture(4, convolutionLighted2);
  518. }
  519. return true;
  520. }
  521. /*
  522. var avg = new Geometry4D.Vector<float>();
  523. var pixel = texture.GetPixels();
  524. foreach(var i in pixel.Index())
  525. {
  526. var iP = pixel[i];
  527. avg += new Geometry4D.Vector<float>(iP.r, iP.g, iP.b, iP.a);
  528. }
  529. avg /= pixel.Count();
  530. /*
  531. var (texLightedR, texLightedG, texLightedB) = ToRGB(newTex);
  532. ScreenLocate.DebugTexture(3, texLightedR);
  533. ScreenLocate.DebugTexture(4, texLightedG);
  534. ScreenLocate.DebugTexture(5, texLightedB);
  535. //Debug.Log(avg);
  536. return false;
  537. /**/
  538. #endregion
  539. }
  540. float GetBrightness()
  541. {
  542. UnityEngine.Color[] differPixel = new UnityEngine.Color[Size.x * Size.y];
  543. Parallel.For(0, Size.x * Size.y, i =>
  544. {
  545. var pi = ScreenWhiteTexture[i] - ScreenBlackTexture[i];
  546. differPixel[i] = new UnityEngine.Color(pi.x, pi.y, pi.z);
  547. });
  548. var sum = 0f;
  549. foreach (var i in differPixel)
  550. {
  551. sum += i.Brightness();
  552. }
  553. sum /= differPixel.Length;
  554. //Debug.Log(sum);
  555. return sum;
  556. }
  557. // 转换成屏幕定位所需的纹理图像
  558. Texture2D ToLocateTex(UnityEngine.Color[] pixels)
  559. {
  560. var ScreenLocateTex = new Texture2D(Size.x, Size.y);
  561. ScreenLocateTex.SetPixels(pixels);
  562. ScreenLocateTex.Apply();
  563. //ScreenLocate.DebugTexture(2, ScreenLocateTex);
  564. return ScreenLocateTex.AutoLight(10);
  565. //ScreenLocate.DebugTexture(2, ScreenLocateTexLighted);
  566. //var ScreenLocateTexR = ToLocateTex.ToRGB(ColorChannel.Red);
  567. //var ScreenLocateTexG = ToLocateTex.ToRGB(ColorChannel.Green);
  568. //var ScreenLocateTexB = ToLocateTex.ToRGB(ColorChannel.Blue);
  569. //LocateLightedRedTex = ScreenLocateTexR;
  570. //ScreenLocate.DebugTexture(2, ScreenLocateTexR);
  571. //ScreenLocate.DebugTexture(4, ScreenLocateTexG);
  572. //ScreenLocate.DebugTexture(5, ScreenLocateTexB);
  573. //var ScreenLocateTexLightedMat = texture.Too0Mat();
  574. }
  575. /// <param name="lineWidth">识别的最小线段长度</param>
  576. /// <param name="debugImages">这个参数如果不为null且数量大于0,则执行debug操作</param>
  577. void QuadrilateralFit(List<Texture2D> debugImages = null)
  578. {
  579. // 屏幕黑白差值,存放多批次的图像用于识别, 该List数量不能等于 0
  580. List<UnityEngine.Color[]> PixelsMultipleBatches = new List<UnityEngine.Color[]>();
  581. //var sw = new System.Diagnostics.Stopwatch();
  582. //sw.Start();
  583. //读取数据
  584. if (debugImages != null && debugImages.Count != 0)
  585. {
  586. foreach (var i in debugImages)
  587. {
  588. Debug.Log($"<color=aqua>Debug {i.name}</color>");
  589. PixelsMultipleBatches.Add(i.GetPixels());
  590. }
  591. }
  592. else // 获得屏幕差值
  593. {
  594. var maxWhite = 0f;
  595. foreach (var i in ScreenWhiteTexture)
  596. {
  597. var m = i.x > i.y ? (i.x > i.z ? i.x : i.z) : (i.y > i.z ? i.y : i.z);
  598. if (maxWhite < m)
  599. maxWhite = m;
  600. }
  601. //var scale = 1.0f / maxWhite; // 放大对比度
  602. //Debug.Log("[ScreenIdentification] 提高亮度倍数: " + scale);
  603. var differPixel = new UnityEngine.Color[Size.x * Size.y];
  604. var whitePixel = new UnityEngine.Color[Size.x * Size.y];
  605. Parallel.For(0, Size.x, x =>
  606. {
  607. for (int y = 0; y < Size.y; y++)
  608. {
  609. var i = y * Size.x + x;
  610. var d = ScreenWhiteTexture[i] - ScreenBlackTexture[i];
  611. differPixel[i] = new UnityEngine.Color(d.x, d.y, d.z);
  612. whitePixel[i] = new UnityEngine.Color(ScreenWhiteTexture[i].x, ScreenWhiteTexture[i].y, ScreenWhiteTexture[i].z);
  613. }
  614. });
  615. PixelsMultipleBatches.Add(differPixel); // 色差图
  616. PixelsMultipleBatches.Add(whitePixel); // 原图
  617. }
  618. int conSize = (int)Math.Ceiling(0.007f * Size.y) * 2 + 1;
  619. conSize = Math.Max(conSize, 7); // 设置最小为7
  620. float minLength = conSize * 4f;
  621. minLength = locateIndex == -1 ? minLength : minLength * areaPercent; // minLength需要按areaPercent比例缩小
  622. string log = $"[Log][ScreenLocate Auto] Size: ({Size.x},{Size.y}), 卷积核Size: {conSize}, 最小线段长度: {minLength}";
  623. var allLines = new List<LineIdentified>();
  624. List<Texture2D> LocateTexTemp = new List<Texture2D>();
  625. List<Matrix> ScreenLocateMatList = new List<Matrix>();
  626. var LineCaptureSizes = new Vector[2] { Vector.One * minLength, new Vector(10, 6) };
  627. foreach (var batch in PixelsMultipleBatches.Index())
  628. {
  629. var locateTex = ToLocateTex(PixelsMultipleBatches[batch]);
  630. LocateTexTemp.Add(locateTex);
  631. var ScreenLocateMat = locateTex.Too0Mat(); // 用于获取Lines的Matrix
  632. var lineCount = ZIMIdentifyQuadLSD(
  633. ref allLines,
  634. batch,
  635. ScreenLocateMat.zimIdentifyEdgeGradientAny(conSize),
  636. minLength,
  637. LineCaptureSizes[batch]);
  638. log += $"\r\n识别图片{batch}, 识别到的线段数量为: {lineCount}";
  639. ScreenLocateMatList.Add(ScreenLocateMat);
  640. }
  641. Texture2D ScreenLocateTexture = LocateTexTemp[0]; // for output
  642. // 如果有手动数据,刷新一下Size
  643. QuadManual?.ReSize(new Vector(Size.x, Size.y), ScreenMap.ViewAspectRatioSetting);
  644. // 估算屏幕中点,如果已有手动定位数据,根据现有数据取平均即可,否则从色差计算,ScreenLocateMatList[0]默认是屏幕的黑白色差
  645. Vector AvgPoint = QuadManual != null ? QuadManual.Quad.Centroid : GetAvgPoint(ScreenLocateMatList[0]);
  646. // 过滤得到四边形的四条边,
  647. var (quadLinesSemiAuto, quadLinesAuto) = FilterLines(ScreenLocateMatList, allLines, AvgPoint,
  648. out LineIdentified[] manualLines, out List<LineIdentified> possibleLines,
  649. conSize, conSize, minLength);
  650. #region 全自动识别的结果
  651. List<LineIdentified> LineIdentifiedAuto = new List<LineIdentified>(); // 线段顺序: 下、右、上、左
  652. for (int i = 0; i < 4; i++)
  653. {
  654. if (quadLinesAuto[i] != null)
  655. LineIdentifiedAuto.Add(quadLinesAuto[i]);
  656. }
  657. if (LineIdentifiedAuto.Count == 4) // 判断识别的线段能否拼成屏幕,能拼成则记录
  658. {
  659. var a = LineIdentifiedAuto[0].Line.Intersect(LineIdentifiedAuto[3].Line, false).Value;
  660. var b = LineIdentifiedAuto[0].Line.Intersect(LineIdentifiedAuto[1].Line, false).Value;
  661. var c = LineIdentifiedAuto[2].Line.Intersect(LineIdentifiedAuto[3].Line, false).Value;
  662. var d = LineIdentifiedAuto[1].Line.Intersect(LineIdentifiedAuto[2].Line, false).Value;
  663. QuadAuto = new QuadrilateralInCamera(a, b, c, d, new Vector(Size.x, Size.y));
  664. if (!QuadAuto.IsQuadComplete())
  665. QuadAuto = null;
  666. }
  667. #endregion
  668. #region 半自动识别
  669. List<LineIdentified> LineIdentifiedSemiAuto = new List<LineIdentified>(); // 线段顺序: 下、右、上、左
  670. LastQuadSemiAutoState = new bool[4] { true, true, true, true };
  671. for (int i = 0; i < 4; i++)
  672. {
  673. if (quadLinesSemiAuto[i] != null)
  674. LineIdentifiedSemiAuto.Add(quadLinesSemiAuto[i]);
  675. else if (manualLines != null)
  676. {
  677. LineIdentifiedSemiAuto.Add(manualLines[i]);
  678. LastQuadSemiAutoState[i] = false;
  679. }
  680. }
  681. if (LineIdentifiedSemiAuto.Count == 4) // 判断识别的线段能否拼成屏幕,能拼成则记录
  682. {
  683. var a = LineIdentifiedSemiAuto[0].Line.Intersect(LineIdentifiedSemiAuto[3].Line, false).Value;
  684. var b = LineIdentifiedSemiAuto[0].Line.Intersect(LineIdentifiedSemiAuto[1].Line, false).Value;
  685. var c = LineIdentifiedSemiAuto[2].Line.Intersect(LineIdentifiedSemiAuto[3].Line, false).Value;
  686. var d = LineIdentifiedSemiAuto[1].Line.Intersect(LineIdentifiedSemiAuto[2].Line, false).Value;
  687. QuadSemiAuto = new QuadrilateralInCamera(a, b, c, d, new Vector(Size.x, Size.y));
  688. if (!QuadSemiAuto.IsQuadComplete())
  689. QuadSemiAuto = null;
  690. }
  691. #endregion
  692. // 优先应用自动的结果(也可以在外部手动设置)
  693. if (QuadSemiAuto == null && QuadAuto == null && Screen.QuadInCamera != null) // 如果可能,回退到上一个screen
  694. {
  695. Debug.Log($"<color=yellow>[ScreenIdentification] 本次识别失败,回退到上次的识别结果: {Screen.QuadInCamera}</color>");
  696. quadTemp.Add(Screen.QuadInCamera);
  697. }
  698. else if (QuadAuto != null)
  699. {
  700. Debug.Log($"<color=aqua>[ScreenIdentification] 识别到四边形(全自动): {QuadAuto}</color>");
  701. quadTemp.Add(QuadAuto);
  702. }
  703. else if (QuadSemiAuto != null)
  704. {
  705. Debug.Log($"<color=aqua>[ScreenIdentification] 识别到四边形(半自动): {QuadSemiAuto}</color>");
  706. quadTemp.Add(QuadSemiAuto);
  707. }
  708. #region 绘制 output texture
  709. // 绘制半自动
  710. var ScreenQuadMap = new Matrix(Size, Tiling: true); // 识别的到的屏幕四边形(半自动和自动在一张图上)
  711. foreach (var i in LineIdentifiedSemiAuto.Index())
  712. {
  713. if (LastQuadSemiAutoState[i])
  714. o0Extension.DrawLine(ScreenQuadMap, LineIdentifiedSemiAuto[i].DrawLine, (x, y) => 5, new Geometry2D.Float.Vector(0, 10));
  715. else
  716. o0Extension.DrawLine(ScreenQuadMap, LineIdentifiedSemiAuto[i].DrawLine, (x, y) => 3, new Geometry2D.Float.Vector(0, 6), true);
  717. }
  718. // 绘制全自动
  719. foreach (var i in LineIdentifiedAuto.Index())
  720. o0Extension.DrawLine(ScreenQuadMap, LineIdentifiedAuto[i].DrawLine, (x, y) => 4, new Geometry2D.Float.Vector(0, 4), true);
  721. Texture2D ScreenQuad = ScreenQuadMap.ToTexRGBA(FloatValueToColor);
  722. Texture2D ScreenQuadWithScreen = ScreenQuad.Overlay(ScreenLocateTexture); // 叠加屏幕色差图
  723. // 绘制allLines
  724. var allLinesMap = new Matrix(Size, Tiling: true);
  725. foreach (var l in allLines)
  726. {
  727. if (l.DrawLine != null)
  728. o0Extension.DrawLine(allLinesMap, l.DrawLine, (x, y) => 1, new Geometry2D.Float.Vector(0, 2), true);
  729. }
  730. var allLinesTex = allLinesMap.ToTexRGBA(FloatValueToColor);
  731. ScreenLocate.DebugTexture(1, allLinesTex);
  732. // 还需要输出一张识别结果图,包含干扰线段
  733. var ChoosableLineMap = new Matrix(Size, Tiling: true);
  734. foreach (var l in possibleLines)
  735. {
  736. if (l != null && !quadLinesSemiAuto.Contains(l) && !manualLines.Contains(l))
  737. o0Extension.DrawLine(ChoosableLineMap, l.DrawLine, (x, y) => 1, new Geometry2D.Float.Vector(0, 2), true); // 其他的备选线段
  738. }
  739. foreach (var l in LineIdentifiedSemiAuto)
  740. {
  741. if (l != null)
  742. o0Extension.DrawLine(ChoosableLineMap, l.DrawLine, (x, y) => 5, new Geometry2D.Float.Vector(0, 5)); // 识别的结果
  743. }
  744. if (manualLines != null)
  745. {
  746. foreach (var l in manualLines)
  747. o0Extension.DrawLine(ChoosableLineMap, l.DrawLine, (x, y) => 3, new Geometry2D.Float.Vector(0, 2), true); // 旧的屏幕线段(例如上次手动识别的)
  748. }
  749. Texture2D ChoosableLineTex = ChoosableLineMap.ToTexRGBA(FloatValueToColor);
  750. #endregion
  751. log += $"\r\n屏幕四边形_手动识别{QuadManual != null}\r\n屏幕四边形_半自动识别{QuadSemiAuto != null}\r\n屏幕四边形_全自动识别{QuadAuto != null}";
  752. Debug.Log(log);
  753. // 是否将图片保存到本地
  754. if (ScreenLocate.Main.SaveToggle?.isOn ?? false && ScreenLocate.Main.DebugOnZIMDemo)
  755. {
  756. var FileDirectory = $"Debug_屏幕定位/";
  757. SaveImages(FileDirectory, log, ScreenLocateTexture, allLinesTex, ChoosableLineTex, ScreenQuad);
  758. }
  759. //times.Add(watch.ElapsedMilliseconds);
  760. //Debug.Log("time: " + (times[times.Count - 1] - times[times.Count - 2]));
  761. // opecncv处理, zim
  762. {
  763. //var cvLines = edge.cvHoughLinesP();
  764. //ScreenLocate.DebugTexture(5, cvLines);
  765. //var myLines = Hough.Transform(edgeMat);
  766. //var cvLines = edge.cvLine(myLines);
  767. //ScreenLocate.DebugTexture(5, cvLines);
  768. }
  769. {
  770. ScreenLocate.DebugTexture(2, ScreenLocateTexture);
  771. ScreenLocate.DebugTexture(3, ScreenQuad);
  772. ScreenLocate.DebugTexture(4, ScreenQuadWithScreen);
  773. ScreenLocate.DebugTexture(5, ChoosableLineTex);
  774. }
  775. foreach (var i in LocateTexTemp)
  776. {
  777. if (i != ScreenLocateTexture) // ScreenLocateTexture 由 ScreenLocate.DebugTexture 释放
  778. GameObject.Destroy(i);
  779. }
  780. }
  781. Vector GetAvgPoint(Matrix screenLocateMat)
  782. {
  783. // 加权平均
  784. Vector[] avgPointsColumn = new Vector[screenLocateMat.Size.x];
  785. float[] valueSumsColumn = new float[screenLocateMat.Size.x];
  786. Parallel.For(0, screenLocateMat.Size.x, i =>
  787. {
  788. for (int j = 0; j < screenLocateMat.Size.y; j++)
  789. {
  790. var value = screenLocateMat[i, j];
  791. valueSumsColumn[i] += value;
  792. avgPointsColumn[i] += new Vector(i, j) * value;
  793. }
  794. });
  795. Vector avgPoint = Vector.Zero;
  796. var valueSum = 0f;
  797. for (int i = 0; i < screenLocateMat.Size.x; i++)
  798. {
  799. avgPoint += avgPointsColumn[i];
  800. valueSum += valueSumsColumn[i];
  801. }
  802. avgPoint /= valueSum;
  803. return avgPoint;
  804. }
  805. // 返回查找到的线段数量,0是查找失败
  806. int ZIMIdentifyQuadLSD(ref List<LineIdentified> allLines, int batch, (Matrix edgeMat, Matrix edgeDirMat) edgeGradient,
  807. float minLength, Vector LineCaptureSize)
  808. {
  809. var l = edgeGradient.edgeMat.IdentifyLineLSD(edgeGradient.edgeDirMat, minLength, 50, LineCaptureSize);
  810. if (l == null || l.Count == 0)
  811. return 0;
  812. allLines.AddRange(l.Select((i) => new LineIdentified(batch, i)));
  813. return l.Count;
  814. }
  815. // 返回四边形的四条边(半自动、全自动),List长度一定是4 (如果没有识别到就是null),且线段顺序是: 下、右、上、左
  816. (List<LineIdentified>, List<LineIdentified>) FilterLines(List<Matrix> screenLocateMatList, List<LineIdentified> allLines, Vector avgPoint,
  817. out LineIdentified[] manualLines, out List<LineIdentified> possibleLines, float conSize, float gradientLength, float minLength = 100)
  818. {
  819. //Debug.Log("[IdentifyLineLSD] lines.Count: " + lines.Count);
  820. var offset = new Vector((conSize - 1) / 2, (conSize - 1) / 2);
  821. // LSD计算得到的矩阵尺寸较小(因为卷积),这里必须进行位移
  822. for (int i = 0; i < allLines.Count; i++)
  823. allLines[i].Offset(offset);
  824. // 筛掉椭圆框外的线段(超出一半会筛掉)
  825. var innerLines = new List<LineIdentified>();
  826. for (int i = 0; i < allLines.Count; i++)
  827. {
  828. List<Vector> InArea = new List<Vector>();
  829. var dir = (allLines[i].Line.B - allLines[i].Line.A) / 4;
  830. var points = new Vector[5] { allLines[i].Line.A, allLines[i].Line.A + dir, allLines[i].Line.A + dir * 2f, allLines[i].Line.A + dir * 3f, allLines[i].Line.B }; // A点、中间的点、B点
  831. for (int pI = 0; pI < points.Length; pI++)
  832. {
  833. if (!ScreenLocate.Main.ScreenPixelCheaker.OutArea2D(points[pI], Size))
  834. InArea.Add(points[pI]);
  835. }
  836. if (InArea.Count < 2) // 少于2个点在内部
  837. continue;
  838. else if (InArea.Count < points.Length) // 不完全在内部
  839. allLines[i].DrawLine = new Line(InArea.First(), InArea.Last()); // 将部分线条设置为drawline,用于下一步的绘制
  840. else // 线段全部在椭圆内
  841. allLines[i].DrawLine = allLines[i].Line;
  842. innerLines.Add(allLines[i]);
  843. }
  844. // 角度阈值,用来判断线段的梯度方向是否指向屏幕中心(avgPoint)
  845. var avaAngleHalf = 75f;
  846. // 沿直线计算综合梯度(梯度乘以长度系数,再乘以距离系数), distanceRatio是实际距离除以最大距离
  847. float estimateGradient(LineIdentified line, float distanceRatio)
  848. {
  849. var dir = (line.Line.B - line.Line.A).Normalized;
  850. var vertical = new Vector(-dir.y, dir.x) * (gradientLength / 2);
  851. var step = 2;
  852. var ll = line.Line.Length;
  853. var lg = new List<float>();
  854. for (int i = 0; i <= ll; i += step)
  855. {
  856. var point = line.Line.A + dir * i;
  857. var ga = point + vertical;
  858. var gb = point - vertical;
  859. lg.Add(screenLocateMatList[line.Batch][(int)ga.x, (int)ga.y] - screenLocateMatList[line.Batch][(int)gb.x, (int)gb.y]);
  860. }
  861. //float e = (float)Math.Sqrt(Math.Ceiling(line.Line.Length / minLength)); // 长度系数,筛选时梯度更大、长度更长的线段更优
  862. float d = (1.5f - distanceRatio) * 2; // 距离系数,距离越近,系数越大
  863. return d * Math.Abs(lg.Mean());
  864. }
  865. // 根据线段梯度的角度,判断是不是屏幕的边,out index代表是哪条边(顺序是: 下、右、上、左)
  866. bool isScreenLine(LineIdentified line, out int index)
  867. {
  868. var a = (avgPoint - (line.Line.A + line.Line.B) / 2).DegreeToXAxis();
  869. //Debug.Log(a + ", " + gradient + ", " + sum);
  870. index = -1;
  871. if (Math.Abs(a - line.GradientDegree) < avaAngleHalf || Math.Abs(a - 360 - line.GradientDegree) < avaAngleHalf || Math.Abs(a + 360 - line.GradientDegree) < avaAngleHalf)
  872. {
  873. if (line.GradientDegree > 45 && line.GradientDegree < 135) // 下
  874. index = 0;
  875. else if (line.GradientDegree > 135 && line.GradientDegree < 225) // 右
  876. index = 1;
  877. else if (line.GradientDegree > 225 && line.GradientDegree < 315) // 上
  878. index = 2;
  879. else
  880. index = 3;
  881. return true;
  882. }
  883. return false;
  884. }
  885. // 下、右、上、左, 半自动和自动
  886. var quadLinesSemiAuto = new List<(float, LineIdentified)>[4] { new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>() };
  887. var quadLinesAuto = new List<(float, LineIdentified)>[4] { new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>() };
  888. possibleLines = new List<LineIdentified>();
  889. // 如果已有手动定位数据,根据现有数据筛选线条(半自动)
  890. manualLines = null;
  891. if (QuadManual != null)
  892. {
  893. Debug.Log($"[IdentifyLineLSD] 根据已有定位数据做筛选, QuadManual: {QuadManual}");
  894. manualLines = QuadManual.GetLines().Select((i) => new LineIdentified(0, i, 0, 0, true)).ToArray();
  895. var calibration = ScreenLocate.Main.ReDoLocateCalibrationRatio * Size.y;
  896. var avgPointCross = manualLines.Select((i) => i.Line.LineCrossWithPoint(avgPoint)).ToArray(); // 对于平均点的corss值
  897. var avgPointPedal = manualLines.Select((i) => o0Extension.PointPedal(i.Line, avgPoint, out _)).ToArray(); // 当前定位的垂足,下、右、上、左
  898. foreach (var line in innerLines)
  899. {
  900. // 筛选条件:1-梯度方向匹配,2-垂足的距离足够近, 3-线段的AB点均在旧线段外部, 4-新的线段的中点,到旧线段的垂足,要在旧线段内
  901. if (isScreenLine(line, out int index))
  902. {
  903. var distanceToOld = (o0Extension.PointPedal(line.Line, avgPoint, out _) - avgPointPedal[index]).Length;
  904. if (distanceToOld < calibration &&
  905. manualLines[index].Line.LineCrossWithPoint(line.Line.A) * avgPointCross[index] <= 0 &&
  906. manualLines[index].Line.LineCrossWithPoint(line.Line.B) * avgPointCross[index] <= 0)
  907. {
  908. var middleToOldLine = o0Extension.PointPedal(manualLines[index].Line, (line.Line.A + line.Line.B) / 2, out bool inLineSegment);
  909. if (inLineSegment)
  910. {
  911. quadLinesSemiAuto[index].Add((estimateGradient(line, distanceToOld / calibration), line));
  912. possibleLines.Add(line);
  913. }
  914. }
  915. }
  916. }
  917. }
  918. // 全自动
  919. foreach (var line in allLines)
  920. {
  921. if (isScreenLine(line, out int index))
  922. {
  923. if (line.Batch < 1) // 全自动只处理第一张图,默认是色差图
  924. {
  925. quadLinesAuto[index].Add((estimateGradient(line, 1), line));
  926. }
  927. }
  928. }
  929. var resultSemiAuto = new LineIdentified[4];
  930. var resultAuto = new LineIdentified[4];
  931. for (int i = 0; i < 4; i++)
  932. {
  933. if (quadLinesSemiAuto[i].Count > 0)
  934. resultSemiAuto[i] = quadLinesSemiAuto[i].Max((a, b) => a.Item1.CompareTo(b.Item1)).Item2;
  935. if (quadLinesAuto[i].Count > 0)
  936. resultAuto[i] = quadLinesAuto[i].Max((a, b) => a.Item1.CompareTo(b.Item1)).Item2;
  937. }
  938. return (resultSemiAuto.ToList(), resultAuto.ToList());
  939. }
  940. void SaveImages(string FileDirectory, string log,
  941. Texture2D ScreenLocateTex, Texture2D allLinesTex, Texture2D ChoosableLineTex, Texture2D ScreenQuadTex)
  942. {
  943. if (!Directory.Exists(FileDirectory))
  944. Directory.CreateDirectory(FileDirectory);
  945. var time = DateTime.Now.ToString("yyyyMMdd_HHmmss");
  946. var pngData = ScreenLocate.Main.OutputTextures[7]?.EncodeToPNG();
  947. if (pngData != null)
  948. File.WriteAllBytes($"{FileDirectory}{time}A屏幕原图.png", pngData);
  949. var pngData1 = ScreenLocateTex?.EncodeToPNG();
  950. if (pngData1 != null)
  951. File.WriteAllBytes($"{FileDirectory}{time}B黑白色差.png", pngData1);
  952. var pngData2 = allLinesTex?.EncodeToPNG();
  953. if (pngData2 != null)
  954. File.WriteAllBytes($"{FileDirectory}{time}C全部识别线段_半自动.png", pngData2);
  955. var pngData3 = ChoosableLineTex?.EncodeToPNG();
  956. if (pngData3 != null)
  957. File.WriteAllBytes($"{FileDirectory}{time}D备选线段_半自动.png", pngData3);
  958. var pngData4 = ScreenQuadTex?.EncodeToPNG();
  959. if (pngData4 != null)
  960. File.WriteAllBytes($"{FileDirectory}{time}E识别结果.png", pngData4);
  961. Debug.Log($"<color=aqua>({time}) 屏幕识别图片保存至:程序根目录/{FileDirectory}</color>");
  962. log +=
  963. $"\r\n屏幕原图保存{pngData != null}, " +
  964. $"\r\n黑白色差保存{pngData1 != null}, " +
  965. $"\r\n全部识别线段(半自动)保存{pngData2 != null}, " +
  966. $"\r\n备选线段(半自动)保存{pngData3 != null}, " +
  967. $"\r\n识别结果保存{pngData4 != null}";
  968. File.WriteAllText($"{FileDirectory}{time}屏幕自动定位_日志.log", log);
  969. }
  970. }
  971. }