ScreenIdentification.cs 49 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100
  1. #define ENABLE_LOG
  2. using o0.Geometry2D.Float;
  3. using o0.Num;
  4. using System;
  5. using System.Collections.Generic;
  6. using System.IO;
  7. using System.Linq;
  8. using System.Threading.Tasks;
  9. using UnityEngine;
  10. using UnityEngine.UIElements;
  11. using ZIM;
  12. using ZIM.Unity;
  13. namespace o0.Project
  14. {
  15. public partial class ScreenIdentification
  16. {
  17. private const string TAG = "ScreenIdentification#";
  18. // LocateAreaData表示每次屏幕的色差变化的区域,可能有多次。通过设置LocateSingleStep可调整为仅识别一次色差
  19. static Rect[][] LocateAreaData = new Rect[][] {
  20. new Rect[] { new Rect(0f, 0f, 0.3f, 0.3f), new Rect(0f, 0f, 0.4f, 0.4f), new Rect(0f, 0f, 0.5f, 0.5f) },
  21. new Rect[] { new Rect(0.7f, 0f, 0.3f, 0.3f), new Rect(0.6f, 0f, 0.4f, 0.4f), new Rect(0.5f, 0f, 0.5f, 0.5f) },
  22. new Rect[] { new Rect(0f, 0.7f, 0.3f, 0.3f), new Rect(0f, 0.6f, 0.4f, 0.4f), new Rect(0f, 0.5f, 0.5f, 0.5f) },
  23. new Rect[] { new Rect(0.7f, 0.7f, 0.3f, 0.3f), new Rect(0.6f, 0.6f, 0.4f, 0.4f), new Rect(0.5f, 0.5f, 0.5f, 0.5f) }
  24. };
  25. //static Rect[][] LocateAreaData = new Rect[][] {
  26. // new Rect[] { new Rect(0f, 0f, 0.3f, 0.3f), new Rect(0f, 0f, 0.4f, 0.4f), new Rect(0f, 0f, 0.5f, 0.5f), new Rect(0f, 0f, 0.6f, 0.6f) },
  27. // new Rect[] { new Rect(0.7f, 0f, 0.3f, 0.3f), new Rect(0.6f, 0f, 0.4f, 0.4f), new Rect(0.5f, 0f, 0.5f, 0.5f), new Rect(0.4f, 0f, 0.6f, 0.6f) },
  28. // new Rect[] { new Rect(0f, 0.7f, 0.3f, 0.3f), new Rect(0f, 0.6f, 0.4f, 0.4f), new Rect(0f, 0.5f, 0.5f, 0.5f), new Rect(0f, 0.4f, 0.6f, 0.6f) },
  29. // new Rect[] { new Rect(0.7f, 0.7f, 0.3f, 0.3f), new Rect(0.6f, 0.6f, 0.4f, 0.4f), new Rect(0.5f, 0.5f, 0.5f, 0.5f), new Rect(0.4f, 0.4f, 0.6f, 0.6f) }
  30. //};
  31. //static bool LocateSingleStep = false;
  32. static bool LocateSingleStep = true;
  33. public Geometry2D.Vector<int> Size => ScreenLocate.Main.CameraSize;
  34. public QuadrilateralInCamera QuadManual;
  35. public QuadrilateralInCamera QuadAuto; // 全自动,可以给用户选择(赋值给Screen.QuadInCamera即生效)
  36. public QuadrilateralInCamera QuadSemiAuto; // 半自动,可以给用户选择(赋值给Screen.QuadInCamera即生效)
  37. public ScreenMap Screen; // 识别到的屏幕,用于执行透视变换
  38. int capture = 0;
  39. int delay = 0;
  40. int maxCapture;
  41. int maxDelay;
  42. Geometry.Vector<float>[] ScreenBlackTexture;
  43. Geometry.Vector<float>[] ScreenWhiteTexture;
  44. int locateIndex = -1;
  45. readonly List<Rect> locateArea = new List<Rect> {
  46. new Rect(0f, 0f, 0.5f, 0.5f), new Rect(0.5f, 0f, 0.5f, 0.5f), new Rect(0f, 0.5f, 0.5f, 0.5f), new Rect(0.5f, 0.5f, 0.5f, 0.5f)
  47. }; // 屏幕显示白色的区域大小
  48. float areaPercent => locateArea[locateIndex].size.x; // 当前白色区域的占比
  49. int areaSelected = -1; // 选择哪个区域,顺序与Quadrilateral对应
  50. readonly List<float> sumTemp = new List<float>();
  51. readonly List<QuadrilateralInCamera> quadTemp = new List<QuadrilateralInCamera>();
  52. //public ScreenIdentification(WebCamTexture texture)
  53. //{
  54. // Size = new Geometry2D.Vector<int>(texture.width, texture.height);
  55. // Screen = new ScreenMap();
  56. //}
  57. public static UnityEngine.Color FloatValueToColor(float i)
  58. {
  59. return i switch
  60. {
  61. 1 => UnityEngine.Color.yellow,
  62. 2 => new UnityEngine.Color(0,1,1,1),
  63. 3 => UnityEngine.Color.green,
  64. 4 => UnityEngine.Color.white,
  65. 5 => UnityEngine.Color.red,
  66. _ => UnityEngine.Color.black,
  67. };
  68. }
  69. public ScreenIdentification()
  70. {
  71. Screen = new ScreenMap();
  72. OnLocateScreenEnter += () => Application.targetFrameRate = 30; // 固定识别的帧率,确保摄像机拍到正确的画面
  73. OnLocateScreenEnd += () => Application.targetFrameRate = 60;
  74. }
  75. public void SetScreenQuad(QuadrilateralInCamera quad) => Screen.QuadInCamera = quad;
  76. // 上一次半自动识别的情况,false代表这条边识别失败,线段顺序: 下、右、上、左
  77. public bool[] LastQuadSemiAutoState;
  78. public event Action OnLocateScreenEnter;
  79. public event Action OnLocateScreenEnd;
  80. public bool bStartLocateScreen { get; set; } = false;//是否进行捕获
  81. public bool SelectScreenAfterLocate(ScreenLocate.ScreenIdentificationTag tag)
  82. {
  83. QuadrilateralInCamera target = tag switch
  84. {
  85. ScreenLocate.ScreenIdentificationTag.Manual => QuadManual,
  86. ScreenLocate.ScreenIdentificationTag.SemiAuto => QuadSemiAuto,
  87. ScreenLocate.ScreenIdentificationTag.Auto => QuadAuto,
  88. _ => null
  89. };
  90. if (target == null)
  91. return false;
  92. Debug.Log($"<color=aqua>[ScreenIdentification] 选择已识别到的屏幕({Enum.GetName(typeof(ScreenLocate.ScreenIdentificationTag), tag)}), {target}</color>");
  93. SetScreenQuad(target);
  94. return true;
  95. }
  96. // 自动识别开始的入口
  97. public void LocateScreen(int Capture = 30, int Delay = 30) //数值单位是frame
  98. {
  99. if (ScreenLocate.Main.DebugScreenImages.Count != 0 && ScreenLocate.Main.DebugOnZIMDemo) // 这段仅用于测试图片
  100. {
  101. ScreenLocate.Main.CameraSize = new Geometry2D.Vector<int>(ScreenLocate.Main.DebugScreenImages[0].width, ScreenLocate.Main.DebugScreenImages[0].height);
  102. DebugImage(ScreenLocate.Main.DebugScreenImages);
  103. Screen.QuadInCamera = quadTemp[0];
  104. ScreenLocate.SetScreen(null);
  105. ScreenLocate.Main.ShowScreen(ScreenLocate.Main.ScreenQuad, Screen.QuadInCamera);
  106. delay = 0;
  107. capture = 0;
  108. ScreenWhiteTexture = null;
  109. ScreenBlackTexture = null;
  110. locateIndex = -1;
  111. areaSelected = -1;
  112. quadTemp.Clear();
  113. sumTemp.Clear();
  114. ScreenLocate.Main.DebugScreenImages.Clear();
  115. return;
  116. }
  117. delay = Math.Max(Delay, 5);
  118. capture = Math.Max(Capture, 5);
  119. maxDelay = Delay;
  120. maxCapture = Capture;
  121. ScreenLocate.SetScreen(new Rect(0f, 0f, 1f, 1f), UnityEngine.Color.black);
  122. //ScreenLocate.SetScreen(new Rect(0f, 0f, 0.6f, 0.6f), UnityEngine.Color.white);
  123. //bStartLocateScreen = false;
  124. ScreenWhiteTexture = null;
  125. ScreenBlackTexture = null;
  126. OnLocateScreenEnter?.Invoke();
  127. }
  128. /// <summary>
  129. /// 开始进行捕获
  130. /// 初始化了两个数据 capture 和 delay
  131. /// </summary>
  132. /// <returns></returns>
  133. public bool isInitLocateScreen()
  134. {
  135. return capture != 0 && delay != 0;
  136. }
  137. void DebugImage(List<Texture2D> images)
  138. {
  139. QuadrilateralFit(images);
  140. //var watch = new System.Diagnostics.Stopwatch();
  141. //watch.Start();
  142. //var times = new List<double>() { 0.0 };
  143. #if (!NDEBUG && DEBUG && ENABLE_LOG)
  144. Console.WriteLine($"{TAG} quadTemp.Count:{quadTemp.Count}");
  145. #endif
  146. if (quadTemp.Count > 0)
  147. {
  148. ScreenLocate.Main.ShowScreen(ScreenLocate.Main.outputRawImages[4].transform.GetChild(0) as RectTransform, quadTemp[0]);
  149. // 透视变换
  150. // var srcWidth = LocateLightedRedTex.width;
  151. // var transformWidth = (int)((quad.B.x - quad.A.x + quad.D.x - quad.C.x) / 2);
  152. // var transformHeight = (int)((quad.C.y - quad.A.y + quad.D.y - quad.B.y) / 2);
  153. // var transformTex = new Texture2D(transformWidth, transformHeight);
  154. // var pt = new ZIMPerspectiveTransform(new OrdinalQuadrilateral(new Vector(0, 0), new Vector(transformWidth, 0), new Vector(0, transformHeight), new Vector(transformWidth, transformHeight)), quad);
  155. // var dstPixel = new UnityEngine.Color[transformWidth * transformHeight];
  156. // var srcPixel = LocateLightedRedTex.GetPixels();
  157. // Parallel.For(0, transformWidth, (x) =>
  158. // {
  159. // for (int y = 0; y < transformHeight; y++)
  160. // {
  161. // var index = y * transformWidth + x;
  162. // var sampleCoord = pt.TransformRound(x, y);
  163. // dstPixel[index] = srcPixel[sampleCoord.y * srcWidth + sampleCoord.x];
  164. // }
  165. // });
  166. // transformTex.SetPixels(dstPixel);
  167. // transformTex.Apply();
  168. // //ScreenLocate.DebugTexture(1, transformTex);
  169. //#if (!NDEBUG && DEBUG && ENABLE_LOG)
  170. // Console.WriteLine($"{TAG} ScreenLocate.DebugTexture 1:{transformTex.GetNativeTexturePtr()}");
  171. //#endif
  172. }
  173. //times.Add(watch.ElapsedMilliseconds);
  174. //Debug.Log("time: " + (times[times.Count - 1] - times[times.Count - 2]));
  175. }
  176. public void NextScreen()
  177. {
  178. // 只识别一次色差变化
  179. if (LocateSingleStep && areaSelected == -1)
  180. {
  181. LocateAreaData = new Rect[][] { new Rect[] { new Rect(0, 0, 1f, 1f) } };
  182. locateIndex = 3;
  183. areaSelected = 0;
  184. locateArea.AddRange(LocateAreaData[0]);
  185. }
  186. // index从-1开始
  187. locateIndex++;
  188. if (locateIndex < locateArea.Count) // 依次点亮屏幕区域
  189. {
  190. ScreenLocate.SetScreen(locateArea[locateIndex], UnityEngine.Color.white);
  191. delay = maxDelay;
  192. capture = maxCapture;
  193. }
  194. else // 退出屏幕黑白控制
  195. {
  196. ScreenLocate.SetScreen(null);
  197. ScreenLocate.Main.ShowScreen(ScreenLocate.Main.ScreenQuad, Screen.QuadInCamera);
  198. Reset();
  199. }
  200. }
  201. // 清除记录的屏幕识别数据(手动、自动等)
  202. public void ClearQuadCache()
  203. {
  204. SetScreenQuad(null);
  205. QuadManual = null;
  206. QuadSemiAuto = null;
  207. QuadAuto = null;
  208. }
  209. public void Reset()
  210. {
  211. // bStartLocateScreen = false;
  212. delay = 0;
  213. capture = 0;
  214. ScreenWhiteTexture = null;
  215. ScreenBlackTexture = null;
  216. locateIndex = -1;
  217. areaSelected = -1;
  218. if (locateArea.Count > 4)
  219. locateArea.RemoveRange(4, LocateAreaData[0].Length);
  220. quadTemp.Clear();
  221. sumTemp.Clear();
  222. }
  223. public void CaptureBlack(Texture2D cam)
  224. {
  225. if (ScreenBlackTexture == null)
  226. ScreenBlackTexture = new Geometry.Vector<float>[Size.x * Size.y];
  227. var pixel = cam.GetPixels();
  228. Parallel.For(0, Size.x * Size.y, i =>
  229. {
  230. var ip = pixel[i];
  231. ScreenBlackTexture[i] += new Geometry.Vector<float>(ip.r / maxCapture, ip.g / maxCapture, ip.b / maxCapture);
  232. });
  233. }
  234. public void CaptureWhite(Texture2D cam)
  235. {
  236. if (ScreenWhiteTexture == null)
  237. ScreenWhiteTexture = new Geometry.Vector<float>[Size.x * Size.y];
  238. var pixel = cam.GetPixels();
  239. Parallel.For(0, Size.x * Size.y, i =>
  240. {
  241. var ip = pixel[i];
  242. ScreenWhiteTexture[i] += new Geometry.Vector<float>(ip.r / maxCapture, ip.g / maxCapture, ip.b / maxCapture);
  243. });
  244. }
  245. public void CaptureStay(Texture2D cam)
  246. {
  247. if (locateIndex == -1) // 屏幕黑色
  248. {
  249. CaptureBlack(cam);
  250. }
  251. else // 屏幕部分为白色
  252. {
  253. CaptureWhite(cam);
  254. }
  255. }
  256. public void CaptureEnd()
  257. {
  258. //Debug.Log("locateIndex: " + locateIndex + ", quad: " + quadTemp.Count);
  259. if (locateIndex == -1)
  260. return;
  261. if (locateIndex < 4)
  262. {
  263. sumTemp.Add(GetBrightness());
  264. ScreenWhiteTexture = null;
  265. // 选择亮度差最大的区域
  266. if (locateIndex == 3)
  267. {
  268. areaSelected = sumTemp.MaxIndex();
  269. locateArea.AddRange(LocateAreaData[areaSelected]);
  270. }
  271. }
  272. else if (locateIndex >= 4 && locateIndex < locateArea.Count - 1)
  273. {
  274. QuadrilateralFit();
  275. ScreenWhiteTexture = null;
  276. }
  277. else
  278. {
  279. QuadrilateralFit();
  280. if (quadTemp.Count != LocateAreaData[0].Length)
  281. {
  282. Debug.Log($"<color=yellow>[ScreenIdentification] 拟合四边形失败, quadTemp.Count: {quadTemp.Count}</color>");
  283. }
  284. else if (quadTemp.Count == 1)
  285. {
  286. SetScreenQuad(quadTemp[0]);
  287. Debug.Log($"[ScreenIdentification] 拟合成功,识别数据: {Screen.QuadInCamera}");
  288. }
  289. else
  290. {
  291. // Debug.Log($"拟合四边形 2 , quadTemp.Count: {quadTemp.Count}");
  292. // 线性拟合
  293. var xValue = new List<float>() { 0 };
  294. var predicts = new List<Vector>();
  295. foreach (var i in LocateAreaData[0])
  296. xValue.Add(i.size.x);
  297. Vector baseVertex = Vector.Zero; // x==0 时的点
  298. {
  299. foreach (var q in quadTemp)
  300. {
  301. baseVertex += q.Quad[areaSelected];
  302. }
  303. baseVertex /= quadTemp.Count;
  304. }
  305. double rs = 0.0;
  306. for (int i = 0; i < 4; i++)
  307. {
  308. if (i == areaSelected)
  309. {
  310. predicts.Add(baseVertex);
  311. }
  312. else
  313. {
  314. var yValue = new List<Vector>() { baseVertex };
  315. foreach (var q in quadTemp)
  316. {
  317. yValue.Add(q.Quad[i]);
  318. }
  319. var lr = LinerRegression1D.Fit(2, xValue.ToArray(), yValue.ToArray());
  320. rs += lr.RSquared / 3;
  321. predicts.Add(lr.Predict<Vector>(1));
  322. }
  323. }
  324. SetScreenQuad(new QuadrilateralInCamera(predicts, new Vector(Size.x, Size.y)));
  325. Debug.Log($"[ScreenIdentification] 拟合成功,RSquared: {rs}, Quad: {Screen.QuadInCamera}");
  326. //if (rs < 0.8) Screen.Quad = null;
  327. }
  328. OnLocateScreenEnd?.Invoke();
  329. }
  330. }
  331. public bool Update(Texture2D cam)
  332. {
  333. //if (!bStartLocateScreen) return false;
  334. if (delay != 0)
  335. {
  336. //ScreenLocate.Main.CreateUVCTexture2DFocusSizeIfNeeded(1280, 720);
  337. delay--;
  338. if (delay == 0)
  339. {
  340. ScreenLocate.Main.CameraSize = new Geometry2D.Vector<int>(cam.width, cam.height); // 记录当前的分辨率
  341. Debug.Log("[ScreenIdentification] 采样纹理,记录采样分辨率: [" + Size.x + ", " + Size.y + "]");
  342. }
  343. return true;
  344. }
  345. if (capture != 0)
  346. {
  347. //ScreenLocate.Main.CreateUVCTexture2DFocusSizeIfNeeded(1280, 720);
  348. CaptureStay(cam);
  349. capture--;
  350. if (capture == 0)
  351. {
  352. CaptureEnd();
  353. NextScreen();
  354. }
  355. return true;
  356. }
  357. return false;
  358. #region Old
  359. /*
  360. if (delay != 0)
  361. {
  362. delay--;
  363. return true;
  364. }
  365. if (capture != 0)
  366. {
  367. capture--;
  368. if (ScreenBlackTexture == null)
  369. ScreenBlackTexture = new Geometry.Vector<float>[Size.x * Size.y];
  370. var pixel = cam.GetPixels();
  371. Parallel.For(0, Size.x * Size.y, i =>
  372. {
  373. var ip = pixel[i];
  374. ScreenBlackTexture[i] += new Geometry.Vector<float>(ip.r, ip.g, ip.b);
  375. });
  376. if (capture == 0)
  377. ScreenLocate.SetScreen(UnityEngine.Color.black);
  378. return true;
  379. }
  380. if (delay != 0)
  381. {
  382. delay--;
  383. return true;
  384. }
  385. if (capture != 0)
  386. {
  387. capture--;
  388. if (ScreenWhiteTexture == null)
  389. ScreenWhiteTexture = new Geometry.Vector<float>[Size.x * Size.y];
  390. var pixel = cam.GetPixels();
  391. Parallel.For(0, Size.x * Size.y, i =>
  392. {
  393. var ip = pixel[i];
  394. ScreenWhiteTexture[i] += new Geometry.Vector<float>(ip.r, ip.g, ip.b);
  395. });
  396. if (capture == 0)
  397. ScreenLocate.SetScreen(UnityEngine.Color.black);
  398. return true;
  399. }
  400. if (delay != 0)
  401. {
  402. delay--;
  403. return true;
  404. }
  405. if (capture != 0)
  406. {
  407. capture--;
  408. var pixel = cam.GetPixels();
  409. Parallel.For(0, Size.x * Size.y, i =>
  410. {
  411. var ip = pixel[i];
  412. ScreenWhiteTexture[i] -= new Geometry.Vector<float>(ip.r, ip.g, ip.b);
  413. });
  414. if (capture == 0)
  415. {
  416. ScreenLocate.SetScreen(null);
  417. UnityEngine.Color[] newPixel = new UnityEngine.Color[Size.x * Size.y];
  418. Parallel.For(0, Size.x * Size.y, i => {
  419. var pi = ScreenWhiteTexture[i] /= capture;
  420. newPixel[i] = new UnityEngine.Color(pi.x, pi.y, pi.z);
  421. });
  422. //读取数据
  423. //{
  424. // var fileName = "3.bin";
  425. // ScreenLocateTexture = $"2023 04 16 厦门测试数据/{fileName}".FileReadByte<Vector<float>[]>();
  426. // Debug.Log($"Read {fileName}");
  427. // Parallel.For(0, Size.x * Size.y, i =>
  428. // {
  429. // var pi = ScreenLocateTexture[i];
  430. // newPixel[i] = new UnityEngine.Color(pi.x, pi.y, pi.z);
  431. // });
  432. //}
  433. var ScreenLocateTex = new Texture2D(Size.x, Size.y);
  434. ScreenLocateTex.SetPixels(newPixel);
  435. ScreenLocateTex.Apply();
  436. //ScreenLocate.DebugTexture(2, ScreenLocateTex);
  437. var ScreenLocateTexLighted = ScreenLocateTex.AutoLight(10);
  438. //ScreenLocate.DebugTexture(2, ScreenLocateTexLighted);
  439. //var FileSavePath = Application.persistentDataPath + "/ScreenLocateTexture.bin";
  440. bool Save = ScreenLocate.Main.SaveToggle.isOn;
  441. string time;
  442. if (Save)
  443. {
  444. time = DateTime.Now.ToString("yyyyMMdd_HHmmss");
  445. var FileSavePath = $"屏幕定位数据{time}.bin";
  446. FileSavePath.FileWriteByte(ScreenWhiteTexture);
  447. var bytes = ScreenLocateTexLighted.EncodeToPNG();
  448. File.WriteAllBytes($"屏幕定位数据{time}.png", bytes);
  449. Debug.Log("ScreenLocateTexture Saved To: " + FileSavePath);
  450. }
  451. var ScreenLocateTexR = ScreenLocateTexLighted.ToRGB(ColorChannel.Red);
  452. var ScreenLocateTexG = ScreenLocateTexLighted.ToRGB(ColorChannel.Green);
  453. var ScreenLocateTexB = ScreenLocateTexLighted.ToRGB(ColorChannel.Blue);
  454. ScreenLocate.DebugTexture(2, ScreenLocateTexR);
  455. //ScreenLocate.DebugTexture(4, ScreenLocateTexG);
  456. //ScreenLocate.DebugTexture(5, ScreenLocateTexB);
  457. var watch = new System.Diagnostics.Stopwatch();
  458. watch.Start();
  459. var times = new List<double>() { 0.0 };
  460. var ScreenLocateTexLightedMat = ScreenLocateTexLighted.Too0Mat();
  461. //var ScreenLocateTexLightedMat = texture.Too0Mat();
  462. //var (edge, edgeDir) = ScreenLocateTexLightedMat.IdentifyEdge();
  463. var (edge, edgeDir) = ScreenLocateTexLightedMat.zimIdentifyEdgeGradientAny(15);
  464. //ScreenLocate.DebugTexture(4, ScreenLocateTexLighted.Too0Mat().IdentifyEdgeGradient().ToTex());
  465. //ScreenLocate.DebugTexture(4, edge.ToTex());
  466. var quadLines = ScreenLocateTexLightedMat.IdentifyQuadLSD(edge, edgeDir, out List<Line> lightLines, 30);
  467. var drawLineMap = new MatrixF2D(edge..Size.x, edge.Size.y);
  468. int lineCount = 0;
  469. foreach (var l in quadLines)
  470. {
  471. if (l != null)
  472. {
  473. o0Extension.DrawLine(drawLineMap.DrawLine(l, (x, y) => 1, new Geometry2D.Float.Vector(0, 10));
  474. lineCount++;
  475. }
  476. }
  477. if (lineCount == 4)
  478. {
  479. var a = quadLines[0].Intersect(quadLines[3], false).Value;
  480. var b = quadLines[0].Intersect(quadLines[1], false).Value;
  481. var c = quadLines[2].Intersect(quadLines[3], false).Value;
  482. var d = quadLines[1].Intersect(quadLines[2], false).Value;
  483. Quad = new Quadrilateral(a, b, c, d);
  484. if (!Quad.IsInScreen(ScreenLocate.Main.WebCamera.Size))
  485. Quad = null;
  486. }
  487. ScreenLocate.Main.ShowScreen(Quad);
  488. //var lines = edge.IdentifyLineLSD(edgeDir, 100);
  489. ////var lines = ScreenLocateTexLightedMat.IdentifyLineLSD();
  490. //var drawLineMap = new MatrixF2D(edge..Size.x, edge.Size.y);
  491. //var returnMaxLines = lines.Sub(0, 10);
  492. //foreach (var (line, sum, gradient) in returnMaxLines)
  493. // o0Extension.DrawLine(drawLineMap.DrawLine(line, (x, y) => 1, new Geometry2D.Float.Vector(0, 10));
  494. ScreenLocate.DebugTexture(3, drawLineMap.ToTex());
  495. //{
  496. // var bytes = drawLineMap.ToTex().EncodeToPNG();
  497. // File.WriteAllBytes($"屏幕定位数据DrawLineMap.png", bytes);
  498. //}
  499. times.Add(watch.ElapsedMilliseconds);
  500. Debug.Log("time: " + (times[times.Count - 1] - times[times.Count - 2]));
  501. //ScreenLocate.DebugTexture(5, edge.IdentifyLine(edgeDir).ToTex());
  502. //ScreenLocate.DebugTexture(4, ScreenLocateTexLighted.Too0Mat().IdentifyEdgeGradientX().ToTex());
  503. //ScreenLocate.DebugTexture(5, ScreenLocateTexLighted.Too0Mat().IdentifyEdgeGradientY().ToTex());
  504. //var convolutionLighted2 = ScreenLocateTexLighted.Too0Mat().IdentifyEdgeVariance().ToTex();
  505. // opecncv处理
  506. // zim
  507. {
  508. //var cvLines = edge.cvHoughLinesP();
  509. //ScreenLocate.DebugTexture(5, cvLines);
  510. //var myLines = Hough.Transform(edgeMat);
  511. //var cvLines = edge.cvLine(myLines);
  512. //ScreenLocate.DebugTexture(5, cvLines);
  513. }
  514. UnityEngine.Object.Destroy(ScreenLocateTex);
  515. //ScreenLocate.DebugTexture(4, convolutionLighted2);
  516. }
  517. return true;
  518. }
  519. /*
  520. var avg = new Geometry4D.Vector<float>();
  521. var pixel = texture.GetPixels();
  522. foreach(var i in pixel.Index())
  523. {
  524. var iP = pixel[i];
  525. avg += new Geometry4D.Vector<float>(iP.r, iP.g, iP.b, iP.a);
  526. }
  527. avg /= pixel.Count();
  528. /*
  529. var (texLightedR, texLightedG, texLightedB) = ToRGB(newTex);
  530. ScreenLocate.DebugTexture(3, texLightedR);
  531. ScreenLocate.DebugTexture(4, texLightedG);
  532. ScreenLocate.DebugTexture(5, texLightedB);
  533. //Debug.Log(avg);
  534. return false;
  535. /**/
  536. #endregion
  537. }
  538. float GetBrightness()
  539. {
  540. UnityEngine.Color[] differPixel = new UnityEngine.Color[Size.x * Size.y];
  541. Parallel.For(0, Size.x * Size.y, i =>
  542. {
  543. var pi = ScreenWhiteTexture[i] - ScreenBlackTexture[i];
  544. differPixel[i] = new UnityEngine.Color(pi.x, pi.y, pi.z);
  545. });
  546. var sum = 0f;
  547. foreach (var i in differPixel)
  548. {
  549. sum += i.Brightness();
  550. }
  551. sum /= differPixel.Length;
  552. //Debug.Log(sum);
  553. return sum;
  554. }
  555. // 转换成屏幕定位所需的纹理图像
  556. Texture2D ToLocateTex(UnityEngine.Color[] pixels)
  557. {
  558. var ScreenLocateTex = new Texture2D(Size.x, Size.y);
  559. ScreenLocateTex.SetPixels(pixels);
  560. ScreenLocateTex.Apply();
  561. //ScreenLocate.DebugTexture(2, ScreenLocateTex);
  562. return ScreenLocateTex.AutoLight(10);
  563. //ScreenLocate.DebugTexture(2, ScreenLocateTexLighted);
  564. //var ScreenLocateTexR = ToLocateTex.ToRGB(ColorChannel.Red);
  565. //var ScreenLocateTexG = ToLocateTex.ToRGB(ColorChannel.Green);
  566. //var ScreenLocateTexB = ToLocateTex.ToRGB(ColorChannel.Blue);
  567. //LocateLightedRedTex = ScreenLocateTexR;
  568. //ScreenLocate.DebugTexture(2, ScreenLocateTexR);
  569. //ScreenLocate.DebugTexture(4, ScreenLocateTexG);
  570. //ScreenLocate.DebugTexture(5, ScreenLocateTexB);
  571. //var ScreenLocateTexLightedMat = texture.Too0Mat();
  572. }
  573. /// <param name="lineWidth">识别的最小线段长度</param>
  574. /// <param name="debugImages">这个参数如果不为null且数量大于0,则执行debug操作</param>
  575. void QuadrilateralFit(List<Texture2D> debugImages = null)
  576. {
  577. // 屏幕黑白差值,存放多批次的图像用于识别, 该List数量不能等于 0
  578. List<UnityEngine.Color[]> PixelsMultipleBatches = new List<UnityEngine.Color[]>();
  579. var sw = new System.Diagnostics.Stopwatch();
  580. sw.Start();
  581. //读取数据
  582. if (debugImages != null && debugImages.Count != 0)
  583. {
  584. foreach (var i in debugImages)
  585. {
  586. Debug.Log($"<color=aqua>Debug {i.name}</color>");
  587. PixelsMultipleBatches.Add(i.GetPixels());
  588. }
  589. }
  590. else // 获得屏幕差值
  591. {
  592. var maxWhite = 0f;
  593. foreach (var i in ScreenWhiteTexture)
  594. {
  595. var m = i.x > i.y ? (i.x > i.z ? i.x : i.z) : (i.y > i.z ? i.y : i.z);
  596. if (maxWhite < m)
  597. maxWhite = m;
  598. }
  599. var scale = 1.0f / maxWhite; // 放大对比度
  600. var differPixel = new UnityEngine.Color[Size.x * Size.y];
  601. var whitePixel = new UnityEngine.Color[Size.x * Size.y];
  602. Parallel.For(0, Size.x, x =>
  603. {
  604. for (int y = 0; y < Size.y; y++)
  605. {
  606. var i = y * Size.x + x;
  607. var d = ScreenWhiteTexture[i] - ScreenBlackTexture[i];
  608. differPixel[i] = new UnityEngine.Color(d.x, d.y, d.z) * scale;
  609. whitePixel[i] = new UnityEngine.Color(ScreenWhiteTexture[i].x, ScreenWhiteTexture[i].y, ScreenWhiteTexture[i].z) * scale;
  610. }
  611. });
  612. PixelsMultipleBatches.Add(differPixel); // 色差图
  613. PixelsMultipleBatches.Add(whitePixel); // 原图
  614. }
  615. int conSize = (int)Math.Ceiling(0.007f * Size.y) * 2 + 1;
  616. conSize = Math.Max(conSize, 7); // 设置最小为7
  617. float minLength = conSize * 7.7f;
  618. minLength = locateIndex == -1 ? minLength : minLength * areaPercent; // minLength需要按比例缩小
  619. string log = $"[Log][ScreenLocate Auto] Size: ({Size.x},{Size.y}), 卷积核Size: {conSize}, 最小线段长度: {minLength}";
  620. var allLines = new List<LineIdentified>();
  621. List<Texture2D> LocateTexTemp = new List<Texture2D>();
  622. List<Matrix> ScreenLocateMatList = new List<Matrix>();
  623. foreach (var batch in PixelsMultipleBatches.Index())
  624. {
  625. var locateTex = ToLocateTex(PixelsMultipleBatches[batch]);
  626. LocateTexTemp.Add(locateTex);
  627. var ScreenLocateMat = locateTex.Too0Mat(); // 用于获取Lines的Matrix
  628. var lineCount = ZIMIdentifyQuadLSD(ref allLines, batch, ScreenLocateMat.zimIdentifyEdgeGradientAny(conSize), minLength);
  629. log += $"\r\n识别图片{batch}, 识别到的线段数量为: {lineCount}";
  630. ScreenLocateMatList.Add(ScreenLocateMat);
  631. }
  632. Texture2D ScreenLocateTexture = LocateTexTemp[0]; // for output
  633. // 如果有手动数据,刷新一下Size
  634. QuadManual?.ReSize(new Vector(Size.x, Size.y), ScreenMap.ViewAspectRatioSetting);
  635. // 估算屏幕中点,如果已有手动定位数据,根据现有数据取平均即可,否则从色差计算,ScreenLocateMatList[0]默认是屏幕的黑白色差
  636. Vector AvgPoint = QuadManual != null ? QuadManual.Quad.Centroid : GetAvgPoint(ScreenLocateMatList[0]);
  637. // 过滤得到四边形的四条边,
  638. var (quadLinesSemiAuto, quadLinesAuto) = FilterLines(ScreenLocateMatList, allLines, AvgPoint,
  639. out LineIdentified[] manualLines, out List<LineIdentified> possibleLines,
  640. conSize, conSize, minLength);
  641. #region 全自动识别的结果
  642. List<LineIdentified> LineIdentifiedAuto = new List<LineIdentified>(); // 线段顺序: 下、右、上、左
  643. for (int i = 0; i < 4; i++)
  644. {
  645. if (quadLinesAuto[i] != null)
  646. LineIdentifiedAuto.Add(quadLinesAuto[i]);
  647. }
  648. if (LineIdentifiedAuto.Count == 4) // 判断识别的线段能否拼成屏幕,能拼成则记录
  649. {
  650. var a = LineIdentifiedAuto[0].Line.Intersect(LineIdentifiedAuto[3].Line, false).Value;
  651. var b = LineIdentifiedAuto[0].Line.Intersect(LineIdentifiedAuto[1].Line, false).Value;
  652. var c = LineIdentifiedAuto[2].Line.Intersect(LineIdentifiedAuto[3].Line, false).Value;
  653. var d = LineIdentifiedAuto[1].Line.Intersect(LineIdentifiedAuto[2].Line, false).Value;
  654. QuadAuto = new QuadrilateralInCamera(a, b, c, d, new Vector(Size.x, Size.y));
  655. if (!QuadAuto.IsQuadComplete())
  656. QuadAuto = null;
  657. }
  658. #endregion
  659. #region 半自动识别
  660. List<LineIdentified> LineIdentifiedSemiAuto = new List<LineIdentified>(); // 线段顺序: 下、右、上、左
  661. LastQuadSemiAutoState = new bool[4] { true, true, true, true };
  662. for (int i = 0; i < 4; i++)
  663. {
  664. if (quadLinesSemiAuto[i] != null)
  665. LineIdentifiedSemiAuto.Add(quadLinesSemiAuto[i]);
  666. else if (manualLines != null)
  667. {
  668. LineIdentifiedSemiAuto.Add(manualLines[i]);
  669. LastQuadSemiAutoState[i] = false;
  670. }
  671. }
  672. if (LineIdentifiedSemiAuto.Count == 4) // 判断识别的线段能否拼成屏幕,能拼成则记录
  673. {
  674. var a = LineIdentifiedSemiAuto[0].Line.Intersect(LineIdentifiedSemiAuto[3].Line, false).Value;
  675. var b = LineIdentifiedSemiAuto[0].Line.Intersect(LineIdentifiedSemiAuto[1].Line, false).Value;
  676. var c = LineIdentifiedSemiAuto[2].Line.Intersect(LineIdentifiedSemiAuto[3].Line, false).Value;
  677. var d = LineIdentifiedSemiAuto[1].Line.Intersect(LineIdentifiedSemiAuto[2].Line, false).Value;
  678. QuadSemiAuto = new QuadrilateralInCamera(a, b, c, d, new Vector(Size.x, Size.y));
  679. if (!QuadSemiAuto.IsQuadComplete())
  680. QuadSemiAuto = null;
  681. }
  682. #endregion
  683. // 优先应用自动的结果(也可以在外部手动设置)
  684. if (QuadSemiAuto == null && QuadAuto == null && Screen.QuadInCamera != null) // 如果可能,回退到上一个screen
  685. {
  686. Debug.Log($"<color=yellow>[ScreenIdentification] 本次识别失败,回退到上次的识别结果: {Screen.QuadInCamera}</color>");
  687. quadTemp.Add(Screen.QuadInCamera);
  688. }
  689. else if (QuadAuto != null)
  690. {
  691. Debug.Log($"<color=aqua>[ScreenIdentification] 识别到四边形(全自动): {QuadAuto}</color>");
  692. quadTemp.Add(QuadAuto);
  693. }
  694. else if (QuadSemiAuto != null)
  695. {
  696. Debug.Log($"<color=aqua>[ScreenIdentification] 识别到四边形(半自动): {QuadSemiAuto}</color>");
  697. quadTemp.Add(QuadSemiAuto);
  698. }
  699. #region 绘制 output texture
  700. // 绘制半自动
  701. var ScreenQuadMap = new Matrix(Size, Tiling: true); // 识别的到的屏幕四边形(半自动和自动在一张图上)
  702. foreach (var i in LineIdentifiedSemiAuto.Index())
  703. {
  704. if (LastQuadSemiAutoState[i])
  705. o0Extension.DrawLine(ScreenQuadMap, LineIdentifiedSemiAuto[i].DrawLine, (x, y) => 5, new Geometry2D.Float.Vector(0, 10));
  706. else
  707. o0Extension.DrawLine(ScreenQuadMap, LineIdentifiedSemiAuto[i].DrawLine, (x, y) => 3, new Geometry2D.Float.Vector(0, 6), true);
  708. }
  709. // 绘制全自动
  710. foreach (var i in LineIdentifiedAuto.Index())
  711. o0Extension.DrawLine(ScreenQuadMap, LineIdentifiedAuto[i].DrawLine, (x, y) => 4, new Geometry2D.Float.Vector(0, 4), true);
  712. Texture2D ScreenQuad = ScreenQuadMap.ToTexRGBA(FloatValueToColor);
  713. Texture2D ScreenQuadWithScreen = ScreenQuad.Overlay(ScreenLocateTexture); // 叠加屏幕色差图
  714. // 绘制allLines
  715. var allLinesMap = new Matrix(Size, Tiling: true);
  716. foreach (var l in allLines)
  717. {
  718. if (l.DrawLine != null)
  719. o0Extension.DrawLine(allLinesMap, l.DrawLine, (x, y) => 1, new Geometry2D.Float.Vector(0, 2), true);
  720. }
  721. var allLinesTex = allLinesMap.ToTexRGBA(FloatValueToColor);
  722. ScreenLocate.DebugTexture(1, allLinesTex);
  723. // 还需要输出一张识别结果图,包含干扰线段
  724. var ChoosableLineMap = new Matrix(Size, Tiling: true);
  725. foreach (var l in possibleLines)
  726. {
  727. if (l != null && !quadLinesSemiAuto.Contains(l) && !manualLines.Contains(l))
  728. o0Extension.DrawLine(ChoosableLineMap, l.DrawLine, (x, y) => 1, new Geometry2D.Float.Vector(0, 2), true); // 其他的备选线段
  729. }
  730. foreach (var l in LineIdentifiedSemiAuto)
  731. {
  732. if (l != null)
  733. o0Extension.DrawLine(ChoosableLineMap, l.DrawLine, (x, y) => 5, new Geometry2D.Float.Vector(0, 5)); // 识别的结果
  734. }
  735. if (manualLines != null)
  736. {
  737. foreach (var l in manualLines)
  738. o0Extension.DrawLine(ChoosableLineMap, l.DrawLine, (x, y) => 3, new Geometry2D.Float.Vector(0, 2), true); // 旧的屏幕线段(例如上次手动识别的)
  739. }
  740. Texture2D ChoosableLineTex = ChoosableLineMap.ToTexRGBA(FloatValueToColor);
  741. #endregion
  742. log += $"\r\n屏幕四边形_手动识别{QuadManual != null}\r\n屏幕四边形_半自动识别{QuadSemiAuto != null}\r\n屏幕四边形_全自动识别{QuadAuto != null}";
  743. Debug.Log(log);
  744. // 是否将图片保存到本地
  745. if (ScreenLocate.Main.SaveToggle?.isOn ?? false && ScreenLocate.Main.DebugOnZIMDemo)
  746. {
  747. var FileDirectory = $"Debug_屏幕定位/";
  748. SaveImages(FileDirectory, log, ScreenLocateTexture, allLinesTex, ChoosableLineTex, ScreenQuad);
  749. }
  750. //times.Add(watch.ElapsedMilliseconds);
  751. //Debug.Log("time: " + (times[times.Count - 1] - times[times.Count - 2]));
  752. // opecncv处理, zim
  753. {
  754. //var cvLines = edge.cvHoughLinesP();
  755. //ScreenLocate.DebugTexture(5, cvLines);
  756. //var myLines = Hough.Transform(edgeMat);
  757. //var cvLines = edge.cvLine(myLines);
  758. //ScreenLocate.DebugTexture(5, cvLines);
  759. }
  760. {
  761. ScreenLocate.DebugTexture(2, ScreenLocateTexture);
  762. ScreenLocate.DebugTexture(3, ScreenQuad);
  763. ScreenLocate.DebugTexture(4, ScreenQuadWithScreen);
  764. ScreenLocate.DebugTexture(5, ChoosableLineTex);
  765. }
  766. foreach (var i in LocateTexTemp)
  767. {
  768. if (i != ScreenLocateTexture) // ScreenLocateTexture 由 ScreenLocate.DebugTexture 释放
  769. GameObject.Destroy(i);
  770. }
  771. }
  772. Vector GetAvgPoint(Matrix screenLocateMat)
  773. {
  774. // 加权平均
  775. Vector[] avgPointsColumn = new Vector[screenLocateMat.Size.x];
  776. float[] valueSumsColumn = new float[screenLocateMat.Size.x];
  777. Parallel.For(0, screenLocateMat.Size.x, i =>
  778. {
  779. for (int j = 0; j < screenLocateMat.Size.y; j++)
  780. {
  781. var value = screenLocateMat[i, j];
  782. valueSumsColumn[i] += value;
  783. avgPointsColumn[i] += new Vector(i, j) * value;
  784. }
  785. });
  786. Vector avgPoint = Vector.Zero;
  787. var valueSum = 0f;
  788. for (int i = 0; i < screenLocateMat.Size.x; i++)
  789. {
  790. avgPoint += avgPointsColumn[i];
  791. valueSum += valueSumsColumn[i];
  792. }
  793. avgPoint /= valueSum;
  794. return avgPoint;
  795. }
  796. // 返回查找到的线段数量,0是查找失败
  797. int ZIMIdentifyQuadLSD(ref List<LineIdentified> allLines, int batch, (Matrix edgeMat, Matrix edgeDirMat) edgeGradient,
  798. float minLength = 100)
  799. {
  800. var l = edgeGradient.edgeMat.IdentifyLineLSD(edgeGradient.edgeDirMat, minLength, 50, LineCaptureSize: new Vector(10, 6));
  801. if (l == null || l.Count == 0)
  802. return 0;
  803. allLines.AddRange(l.Select((i) => new LineIdentified(batch, i)));
  804. return l.Count;
  805. }
  806. // TODO ,ZIM, 给梯度加个阈值,达不到则筛去
  807. // 返回四边形的四条边(半自动、全自动),List长度一定是4 (如果没有识别到就是null),且线段顺序是: 下、右、上、左
  808. (List<LineIdentified>, List<LineIdentified>) FilterLines(List<Matrix> screenLocateMatList, List<LineIdentified> allLines, Vector avgPoint,
  809. out LineIdentified[] manualLines, out List<LineIdentified> possibleLines, float conSize, float gradientLength, float minLength = 100)
  810. {
  811. //Debug.Log("[IdentifyLineLSD] lines.Count: " + lines.Count);
  812. var offset = new Vector((conSize - 1) / 2, (conSize - 1) / 2);
  813. // LSD计算得到的矩阵尺寸较小(因为卷积),这里必须进行位移
  814. for (int i = 0; i < allLines.Count; i++)
  815. allLines[i].Offset(offset);
  816. // 筛掉椭圆框外的线段(超出一半会筛掉)
  817. var innerLines = new List<LineIdentified>();
  818. for (int i = 0; i < allLines.Count; i++)
  819. {
  820. List<Vector> InArea = new List<Vector>();
  821. var dir = (allLines[i].Line.B - allLines[i].Line.A) / 4;
  822. var points = new Vector[5] { allLines[i].Line.A, allLines[i].Line.A + dir, allLines[i].Line.A + dir * 2f, allLines[i].Line.A + dir * 3f, allLines[i].Line.B }; // A点、中间的点、B点
  823. for (int pI = 0; pI < points.Length; pI++)
  824. {
  825. if (!ScreenLocate.Main.ScreenPixelCheaker.OutArea2D(points[pI], Size))
  826. InArea.Add(points[pI]);
  827. }
  828. if (InArea.Count < 2) // 少于2个点在内部
  829. continue;
  830. else if (InArea.Count < points.Length) // 不完全在内部
  831. allLines[i].DrawLine = new Line(InArea.First(), InArea.Last()); // 将部分线条设置为drawline,用于下一步的绘制
  832. else // 线段全部在椭圆内
  833. allLines[i].DrawLine = allLines[i].Line;
  834. innerLines.Add(allLines[i]);
  835. }
  836. // 角度阈值,用来判断线段的梯度方向是否指向屏幕中心(avgPoint)
  837. var avaAngleHalf = 75f;
  838. // 沿直线计算综合梯度(梯度乘以长度系数,再乘以距离系数), distanceRatio是实际距离除以最大距离
  839. float estimateGradient(LineIdentified line, float distanceRatio)
  840. {
  841. var dir = (line.Line.B - line.Line.A).Normalized;
  842. var vertical = new Vector(-dir.y, dir.x) * (gradientLength / 2);
  843. var step = 2;
  844. var ll = line.Line.Length;
  845. var lg = new List<float>();
  846. for (int i = 0; i <= ll; i += step)
  847. {
  848. var point = line.Line.A + dir * i;
  849. var ga = point + vertical;
  850. var gb = point - vertical;
  851. lg.Add(screenLocateMatList[line.Batch][(int)ga.x, (int)ga.y] - screenLocateMatList[line.Batch][(int)gb.x, (int)gb.y]);
  852. }
  853. float e = (float)Math.Sqrt(Math.Ceiling(line.Line.Length / minLength)); // 长度系数,筛选时梯度更大、长度更长的线段更优
  854. float d = (3 - distanceRatio) / 2; // 距离系数,距离越近,系数越大
  855. return e * d * Math.Abs(lg.Mean());
  856. }
  857. // 根据线段梯度的角度,判断是不是屏幕的边,out index代表是哪条边(顺序是: 下、右、上、左)
  858. bool isScreenLine(LineIdentified line, out int index)
  859. {
  860. var a = (avgPoint - (line.Line.A + line.Line.B) / 2).DegreeToXAxis();
  861. //Debug.Log(a + ", " + gradient + ", " + sum);
  862. index = -1;
  863. if (Math.Abs(a - line.GradientDegree) < avaAngleHalf || Math.Abs(a - 360 - line.GradientDegree) < avaAngleHalf || Math.Abs(a + 360 - line.GradientDegree) < avaAngleHalf)
  864. {
  865. if (line.GradientDegree > 45 && line.GradientDegree < 135) // 下
  866. index = 0;
  867. else if (line.GradientDegree > 135 && line.GradientDegree < 225) // 右
  868. index = 1;
  869. else if (line.GradientDegree > 225 && line.GradientDegree < 315) // 上
  870. index = 2;
  871. else
  872. index = 3;
  873. return true;
  874. }
  875. return false;
  876. }
  877. // 下、右、上、左, 半自动和自动
  878. var quadLinesSemiAuto = new List<(float, LineIdentified)>[4] { new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>() };
  879. var quadLinesAuto = new List<(float, LineIdentified)>[4] { new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>(), new List<(float, LineIdentified)>() };
  880. possibleLines = new List<LineIdentified>();
  881. // 如果已有手动定位数据,根据现有数据筛选线条(半自动)
  882. manualLines = null;
  883. if (QuadManual != null)
  884. {
  885. Debug.Log($"[IdentifyLineLSD] 根据已有定位数据做筛选, QuadManual: {QuadManual}");
  886. manualLines = QuadManual.GetLines().Select((i) => new LineIdentified(0, i, 0, 0, true)).ToArray();
  887. var calibration = ScreenLocate.Main.ReDoLocateCalibrationRatio * Size.y;
  888. var avgPointCross = manualLines.Select((i) => i.Line.LineCrossWithPoint(avgPoint)).ToArray(); // 对于平均点的corss值
  889. var avgPointPedal = manualLines.Select((i) => o0Extension.PointPedal(i.Line, avgPoint, out _)).ToArray(); // 当前定位的垂足,下、右、上、左
  890. foreach (var line in innerLines)
  891. {
  892. // 筛选条件:1-梯度方向匹配,2-垂足的距离足够近, 3-线段的AB点均在旧线段外部, 4-新的线段的中点,到旧线段的垂足,要在旧线段内
  893. if (isScreenLine(line, out int index))
  894. {
  895. var distanceToOld = (o0Extension.PointPedal(line.Line, avgPoint, out _) - avgPointPedal[index]).Length;
  896. if (distanceToOld < calibration &&
  897. manualLines[index].Line.LineCrossWithPoint(line.Line.A) * avgPointCross[index] <= 0 &&
  898. manualLines[index].Line.LineCrossWithPoint(line.Line.B) * avgPointCross[index] <= 0)
  899. {
  900. var middleToOldLine = o0Extension.PointPedal(manualLines[index].Line, (line.Line.A + line.Line.B) / 2, out bool inLineSegment);
  901. if (inLineSegment)
  902. {
  903. quadLinesSemiAuto[index].Add((estimateGradient(line, distanceToOld / calibration), line));
  904. possibleLines.Add(line);
  905. }
  906. }
  907. }
  908. }
  909. }
  910. // 全自动
  911. foreach (var line in allLines)
  912. {
  913. if (isScreenLine(line, out int index))
  914. {
  915. if (line.Batch < 1) // 全自动只处理第一张图,默认是色差图
  916. {
  917. quadLinesAuto[index].Add((estimateGradient(line, 1), line));
  918. }
  919. }
  920. }
  921. var resultSemiAuto = new LineIdentified[4];
  922. var resultAuto = new LineIdentified[4];
  923. for (int i = 0; i < 4; i++)
  924. {
  925. if (quadLinesSemiAuto[i].Count > 0)
  926. resultSemiAuto[i] = quadLinesSemiAuto[i].Max((a, b) => a.Item1.CompareTo(b.Item1)).Item2;
  927. if (quadLinesAuto[i].Count > 0)
  928. resultAuto[i] = quadLinesAuto[i].Max((a, b) => a.Item1.CompareTo(b.Item1)).Item2;
  929. }
  930. return (resultSemiAuto.ToList(), resultAuto.ToList());
  931. }
  932. void SaveImages(string FileDirectory, string log,
  933. Texture2D ScreenLocateTex, Texture2D allLinesTex, Texture2D ChoosableLineTex, Texture2D ScreenQuadTex)
  934. {
  935. if (!Directory.Exists(FileDirectory))
  936. Directory.CreateDirectory(FileDirectory);
  937. var time = DateTime.Now.ToString("yyyyMMdd_HHmmss");
  938. var pngData = ScreenLocate.Main.OutputTextures[7]?.EncodeToPNG();
  939. if (pngData != null)
  940. File.WriteAllBytes($"{FileDirectory}{time}A屏幕原图.png", pngData);
  941. var pngData1 = ScreenLocateTex?.EncodeToPNG();
  942. if (pngData1 != null)
  943. File.WriteAllBytes($"{FileDirectory}{time}B黑白色差.png", pngData1);
  944. var pngData2 = allLinesTex?.EncodeToPNG();
  945. if (pngData2 != null)
  946. File.WriteAllBytes($"{FileDirectory}{time}C全部识别线段_半自动.png", pngData2);
  947. var pngData3 = ChoosableLineTex?.EncodeToPNG();
  948. if (pngData3 != null)
  949. File.WriteAllBytes($"{FileDirectory}{time}D备选线段_半自动.png", pngData3);
  950. var pngData4 = ScreenQuadTex?.EncodeToPNG();
  951. if (pngData4 != null)
  952. File.WriteAllBytes($"{FileDirectory}{time}E识别结果.png", pngData4);
  953. Debug.Log($"<color=aqua>({time}) 屏幕识别图片保存至:程序根目录/{FileDirectory}</color>");
  954. log +=
  955. $"\r\n屏幕原图保存{pngData != null}, " +
  956. $"\r\n黑白色差保存{pngData1 != null}, " +
  957. $"\r\n全部识别线段(半自动)保存{pngData2 != null}, " +
  958. $"\r\n备选线段(半自动)保存{pngData3 != null}, " +
  959. $"\r\n识别结果保存{pngData4 != null}";
  960. File.WriteAllText($"{FileDirectory}{time}屏幕自动定位_日志.log", log);
  961. }
  962. }
  963. }