clsNode.cs 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480
  1. using System;
  2. using System.Diagnostics;
  3. using System.Linq;
  4. using System.Net.NetworkInformation;
  5. using System.Threading.Tasks;
  6. using RestSharp;
  7. namespace TFABot
  8. {
  9. public class clsNode : ISpreadsheet<clsNode>
  10. {
  11. clsRollingAverage LatencyList = new clsRollingAverage(10);
  12. clsRollingAverage PacketLoss = new clsRollingAverage(100);
  13. public clsNode()
  14. {
  15. }
  16. public clsNodeGroup NodeGroup;
  17. public uint LatencyLowest { get; private set;}
  18. [ASheetColumnHeader(true,"name")]
  19. public string Name {get;set;}
  20. [ASheetColumnHeader("group")]
  21. public string Group {get;set;}
  22. [ASheetColumnHeader("host")]
  23. public string Host {get;set;}
  24. [ASheetColumnHeader("monitor")]
  25. public bool Monitor {get;set;}
  26. public String ErrorMsg {get; private set;}
  27. public String NodeVersion {get; private set;}
  28. public int Latency
  29. {
  30. get{ return LatencyList.CurrentAverage; }
  31. }
  32. public DateTime? LastLeaderHeight { get; private set;}
  33. uint _leaderHeight = 0;
  34. public uint LeaderHeight
  35. {
  36. get
  37. {
  38. return _leaderHeight;
  39. }
  40. private set
  41. {
  42. if (_leaderHeight != value)
  43. {
  44. _leaderHeight = value;
  45. if (LastLeaderHeight.HasValue)
  46. {
  47. if (_leaderHeight < NodeGroup.Network.TopHeight && //We are behind the network.
  48. (DateTime.UtcNow - LastLeaderHeight.Value).TotalMinutes>1 ) //Been offline for > 1 minute?
  49. {
  50. SyncMode = true;
  51. GetVersionAsync(); //Get version no if leader now known.
  52. }
  53. }
  54. else
  55. {
  56. GetVersionAsync(); //Get version no if leader now known.
  57. }
  58. LastLeaderHeight = DateTime.UtcNow;
  59. }
  60. }
  61. }
  62. clsAlarm AlarmSyncing = null;
  63. clsAlarm AlarmHeightLow = null;
  64. uint _heightLowCount;
  65. public uint HeightLowCount
  66. {
  67. get
  68. {
  69. return _heightLowCount;
  70. }
  71. set
  72. {
  73. _heightLowCount = value;
  74. if (SyncMode && _heightLowCount==0) SyncMode=false; //Reset sync mode if active
  75. if (!SyncMode)
  76. {
  77. if (_heightLowCount==0 && AlarmHeightLow!=null)
  78. {
  79. Program.AlarmManager.Clear(AlarmHeightLow,$"CLEARED: {Name} height low.");
  80. AlarmHeightLow = null;
  81. }
  82. else if (_heightLowCount > 3 && _requestFailCount ==0 && AlarmHeightLow==null )
  83. {
  84. AlarmHeightLow = new clsAlarm(clsAlarm.enumAlarmType.Height,$"WARNING: {Name} height low.",this);
  85. Program.AlarmManager.New(AlarmHeightLow);
  86. }
  87. }
  88. }
  89. }
  90. clsAlarm AlarmLatencyLow = null;
  91. uint _latencyLowCount;
  92. public uint LatencyLowCount
  93. {
  94. get
  95. {
  96. return _latencyLowCount;
  97. }
  98. set
  99. {
  100. _latencyLowCount = value;
  101. if (_latencyLowCount==0 && AlarmLatencyLow!=null)
  102. {
  103. Program.AlarmManager.Clear(AlarmLatencyLow,$"CLEARED: {Name} poor latency cleared.");
  104. AlarmLatencyLow = null;
  105. }
  106. else if (_requestFailCount ==0 && AlarmLatencyLow==null && _latencyLowCount > 3)
  107. {
  108. AlarmLatencyLow = new clsAlarm(clsAlarm.enumAlarmType.Latency,$"WARNING: {Name} latency poor.",this);
  109. Program.AlarmManager.New(AlarmLatencyLow);
  110. }
  111. }
  112. }
  113. clsAlarm AlarmRequestFail = null;
  114. uint _requestFailCount;
  115. public uint RequestFailCount
  116. {
  117. get
  118. {
  119. return _requestFailCount;
  120. }
  121. set
  122. {
  123. if (_requestFailCount == value) return; //no change
  124. _requestFailCount = value;
  125. if (_requestFailCount==0)
  126. {
  127. if (AlarmRequestFail!=null)
  128. {
  129. Program.AlarmManager.Clear(AlarmRequestFail,$"CLEARED: {Name} now responding.");
  130. AlarmRequestFail = null;
  131. GetVersionAsync();
  132. }
  133. if (!String.IsNullOrEmpty(ErrorMsg)) ErrorMsg="";
  134. if (NodeGroup.Network.TopHeight > LeaderHeight) SyncMode=true;
  135. }
  136. else if (AlarmRequestFail ==null && _requestFailCount == 2)
  137. {
  138. AlarmRequestFail = new clsAlarm(clsAlarm.enumAlarmType.NoResponse,$"WARNING: {Name} not responding.",this);
  139. Program.AlarmManager.New(AlarmRequestFail);
  140. RunMTRAsync();
  141. }
  142. }
  143. }
  144. public async Task GetHeightAsync(int timeout = 2000)
  145. {
  146. await Task.Run(() => {GetHeight(timeout);});
  147. }
  148. public void GetHeight(int timeout = 2000)
  149. {
  150. try {
  151. RestClient client;
  152. if (Host.Contains(":"))
  153. {
  154. client = new RestClient($"http://{Host}");
  155. }
  156. else
  157. {
  158. client = new RestClient($"http://{Host}:8088");
  159. }
  160. client.Timeout = timeout;
  161. var request = new RestRequest("v2", Method.POST);
  162. request.AddHeader("Content-type", "application/json");
  163. request.AddHeader("header", "value");
  164. request.AddJsonBody(
  165. new { jsonrpc = "2.0", id = 0, method = "heights" }
  166. );
  167. // execute the request
  168. var sw = Stopwatch.StartNew();
  169. IRestResponse response = client.Execute(request);
  170. sw.Stop();
  171. if(response.ResponseStatus == ResponseStatus.Completed)
  172. {
  173. var content = response.Content; // raw content as string
  174. var pos1 = 0;
  175. var pos2= 0;
  176. if (!string.IsNullOrEmpty(content))
  177. {
  178. pos1 = content.IndexOf("leaderheight\":");
  179. pos1+=14;
  180. pos2 = content.IndexOf(",",pos1);
  181. uint msgheight=0;
  182. if (UInt32.TryParse(content.Substring(pos1,pos2-pos1),out msgheight))
  183. {
  184. LeaderHeight = msgheight;
  185. }
  186. else
  187. {
  188. ErrorMsg="Invalid data";
  189. }
  190. }
  191. else
  192. {
  193. ErrorMsg="Empty data";
  194. }
  195. LatencyList.Add((int)sw.ElapsedMilliseconds);
  196. PacketLoss.Add(0);
  197. RequestFailCount = 0;
  198. } else if(response.ResponseStatus == ResponseStatus.Error || response.ResponseStatus == ResponseStatus.TimedOut)
  199. {
  200. PacketLoss.Add(100);
  201. ErrorMsg=response.ErrorMessage;
  202. RequestFailCount++;
  203. }
  204. else if(response.ResponseStatus == ResponseStatus.None)
  205. {
  206. ErrorMsg="Empty data";
  207. }
  208. // Console.WriteLine(ToString());
  209. }
  210. catch (Exception ex)
  211. {
  212. Console.WriteLine(ex.Message);
  213. RequestFailCount++;
  214. }
  215. }
  216. public bool SyncMode
  217. {
  218. get
  219. {
  220. return AlarmSyncing!=null;
  221. }
  222. set
  223. {
  224. if (value)
  225. {
  226. if (AlarmSyncing==null)
  227. {
  228. AlarmSyncing = new clsAlarm(clsAlarm.enumAlarmType.Syncing,$"{Name} in SYNC mode.",this);
  229. Program.AlarmManager.New(AlarmSyncing);
  230. }
  231. }
  232. else if (AlarmSyncing != null)
  233. {
  234. Program.AlarmManager.Clear(AlarmSyncing,$"{Name} SYNC cleared.");
  235. AlarmSyncing=null;
  236. }
  237. }
  238. }
  239. public async Task GetVersionAsync(int timeout = 2000)
  240. {
  241. await Task.Run(() => {GetVersion(timeout);});
  242. }
  243. public void GetVersion(int timeout = 2000)
  244. {
  245. try {
  246. RestClient client;
  247. if (Host.Contains(":"))
  248. {
  249. client = new RestClient($"http://{Host}");
  250. }
  251. else
  252. {
  253. client = new RestClient($"http://{Host}:8088");
  254. }
  255. client.Timeout = timeout;
  256. var request = new RestRequest("v2", Method.POST);
  257. request.AddHeader("Content-type", "application/json");
  258. request.AddHeader("header", "value");
  259. request.AddJsonBody(
  260. new { jsonrpc = "2.0", id = 0, method = "properties" }
  261. );
  262. // execute the request
  263. IRestResponse response = client.Execute(request);
  264. if(response.ResponseStatus == ResponseStatus.Completed)
  265. {
  266. var content = response.Content; // raw content as string
  267. var pos1 = 0;
  268. var pos2= 0;
  269. if (!string.IsNullOrEmpty(content))
  270. {
  271. pos1 = content.IndexOf("factomdversion\":");
  272. pos1+=17;
  273. pos2 = content.IndexOf("\"",pos1);
  274. if (pos2>pos1) NodeVersion = content.Substring(pos1,pos2-pos1);
  275. }
  276. else
  277. {
  278. ErrorMsg="Empty version data";
  279. }
  280. } else if(response.ResponseStatus == ResponseStatus.Error || response.ResponseStatus == ResponseStatus.TimedOut)
  281. {
  282. ErrorMsg=response.ErrorMessage;
  283. }
  284. else if(response.ResponseStatus == ResponseStatus.None)
  285. {
  286. ErrorMsg="Empty data";
  287. }
  288. }
  289. catch (Exception ex)
  290. {
  291. Console.WriteLine(ex.Message);
  292. }
  293. }
  294. async public void PingHostAsync()
  295. {
  296. try
  297. {
  298. var pingTask = Task.Run(() =>
  299. {
  300. Uri myUri = new Uri(Host);
  301. clsRollingAverage pingLatency = new clsRollingAverage(10);
  302. clsRollingAverage pingPacketLoss = new clsRollingAverage(10);
  303. try
  304. {
  305. using (var pinger = new Ping())
  306. {
  307. for (var f=0;f<10;f++)
  308. {
  309. PingReply reply = pinger.Send(myUri.Host,2000);
  310. if (reply.Status == IPStatus.Success)
  311. {
  312. pingPacketLoss.Add((int)reply.RoundtripTime);
  313. pingPacketLoss.Add(0);
  314. Program.SendAlert($"Ping {myUri.Host} {reply.RoundtripTime:0} ms {pingPacketLoss.CurrentAverage:0.0}%");
  315. }
  316. else
  317. {
  318. pingPacketLoss.Add(100);
  319. Program.SendAlert($"Ping {myUri.Host} {reply.Status} ms {pingPacketLoss.CurrentAverage:0.0}%");
  320. }
  321. }
  322. }
  323. }
  324. catch (PingException ex)
  325. {
  326. Program.SendAlert($"Ping error {myUri.Host} {ex.Message}");
  327. }
  328. });
  329. }
  330. catch (Exception ex)
  331. {
  332. Program.SendAlert($"Ping error {Name} {ex.Message}");
  333. }
  334. }
  335. public void RunMTRAsync()
  336. {
  337. try{
  338. Console.WriteLine($"Starting MTR on {Name} {Host}");
  339. var process = new Process
  340. {
  341. StartInfo = { FileName = "/usr/bin/mtr",
  342. Arguments = $"-rw {Host}",
  343. UseShellExecute = false,
  344. RedirectStandardOutput = true,
  345. // RedirectStandardError = true
  346. },
  347. EnableRaisingEvents = true
  348. };
  349. process.Exited += (sender, eargs) =>
  350. {
  351. var mtrOutput = process.StandardOutput.ReadToEnd();
  352. Console.WriteLine(mtrOutput);
  353. if (AlarmRequestFail!=null) AlarmRequestFail.AddNote($"{Name}```{mtrOutput}```");
  354. process.Dispose();
  355. };
  356. process.Start();
  357. }catch (Exception ex)
  358. {
  359. Console.WriteLine($"RunMTRAsync {ex.Message}");
  360. }
  361. }
  362. public void Update(clsNode node)
  363. {
  364. if (Name != node.Name) throw new Exception("index name does not match");
  365. Group = node.Group;
  366. Host = node.Host;
  367. if (Monitor != node.Monitor)
  368. {
  369. Monitor = node.Monitor;
  370. HeightLowCount=0;
  371. LatencyLowCount=0;
  372. RequestFailCount=0;
  373. Program.AlarmManager.Remove(AlarmSyncing);
  374. Program.AlarmManager.Remove(AlarmHeightLow);
  375. Program.AlarmManager.Remove(AlarmLatencyLow);
  376. Program.AlarmManager.Remove(AlarmRequestFail);
  377. AlarmSyncing=null;
  378. AlarmHeightLow=null;
  379. AlarmLatencyLow=null;
  380. AlarmRequestFail=null;
  381. }
  382. PostPopulate();
  383. }
  384. public string PostPopulate()
  385. {
  386. ErrorMsg = "";
  387. return (!Program.NodeGroupList.TryGetValue(Group,out NodeGroup)) ? "Error: Node Group Not Found!" : null;
  388. }
  389. public new String ToString()
  390. {
  391. return $"{Name}\t{Host}\t{LeaderHeight}\t{LatencyList.CurrentAverage.ToString().PadLeft(3)} ms ({(100-PacketLoss.CurrentAverage):0.#}%) {ErrorMsg??""}";
  392. }
  393. public void AppendDisplayColumns(ref clsColumnDisplay columnDisplay)
  394. {
  395. columnDisplay.AppendCol(Name ?? "?");
  396. columnDisplay.AppendCol(Host ?? "?");
  397. columnDisplay.AppendCol(NodeVersion??"");
  398. columnDisplay.AppendCol($"{LeaderHeight}");
  399. columnDisplay.AppendCol($"{LatencyList.CurrentAverage.ToString().PadLeft(3)} ms ({(100-PacketLoss.CurrentAverage):0.#}%) ");
  400. if (!Monitor)
  401. {
  402. columnDisplay.Append ("MONITOR OFF");
  403. }
  404. else if (AlarmRequestFail !=null)
  405. {
  406. columnDisplay.Append ("NOT RESPONDING");
  407. }
  408. else
  409. {
  410. if (AlarmHeightLow!=null) columnDisplay.Append ("HEIGHT LOW ");
  411. if (AlarmSyncing!=null) columnDisplay.Append ("SYNCING ");
  412. if (AlarmLatencyLow !=null) columnDisplay.Append ("LATENCY LOW ");
  413. }
  414. if (String.IsNullOrEmpty(ErrorMsg)) columnDisplay.Append(ErrorMsg);
  415. }
  416. }
  417. }