clsNode.cs 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393
  1. using System;
  2. using System.Diagnostics;
  3. using System.Linq;
  4. using System.Net.NetworkInformation;
  5. using System.Threading.Tasks;
  6. using RestSharp;
  7. namespace TFABot
  8. {
  9. public class clsNode : ISpreadsheet<clsNode>
  10. {
  11. clsRollingAverage LatencyList = new clsRollingAverage(10);
  12. clsRollingAverage PacketLoss = new clsRollingAverage(100);
  13. public clsNode()
  14. {
  15. }
  16. public clsNodeGroup NodeGroup;
  17. public uint LatencyLowest { get; private set;}
  18. [ASheetColumnHeader(true,"name")]
  19. public string Name {get;set;}
  20. [ASheetColumnHeader("group")]
  21. public string Group {get;set;}
  22. [ASheetColumnHeader("host")]
  23. public string Host {get;set;}
  24. [ASheetColumnHeader("monitor")]
  25. public bool Monitor {get;set;}
  26. public String ErrorMsg {get; private set;}
  27. public String NodeVersion {get; private set;}
  28. public int Latency
  29. {
  30. get{ return LatencyList.CurrentAverage; }
  31. }
  32. public DateTime? LastLeaderHeight { get; private set;}
  33. uint _leaderHeight = 0;
  34. public uint LeaderHeight
  35. {
  36. get
  37. {
  38. return _leaderHeight;
  39. }
  40. private set
  41. {
  42. if (_leaderHeight < value)
  43. {
  44. _leaderHeight = value;
  45. if (!LastLeaderHeight.HasValue) GetVersionAsync(); //Get version no if leader now known.
  46. LastLeaderHeight = DateTime.UtcNow;
  47. }
  48. }
  49. }
  50. clsAlarm AlarmHeightLow = null;
  51. uint _heightLowCount;
  52. public uint HeightLowCount
  53. {
  54. get
  55. {
  56. return _heightLowCount;
  57. }
  58. set
  59. {
  60. _heightLowCount = value;
  61. if (_heightLowCount==0 && AlarmHeightLow!=null)
  62. {
  63. Program.AlarmManager.Clear(AlarmHeightLow,$"CLEARED: {Name} height low.");
  64. AlarmHeightLow = null;
  65. ErrorMsg="";
  66. }
  67. else if (_heightLowCount > 3 && _requestFailCount ==0 && AlarmHeightLow==null )
  68. {
  69. AlarmHeightLow = new clsAlarm(clsAlarm.enumAlarmType.Height,$"WARNING: {Name} height low.",this);
  70. ErrorMsg="HEIGHT LOW";
  71. Program.AlarmManager.New(AlarmHeightLow);
  72. }
  73. }
  74. }
  75. clsAlarm AlarmLatencyLow = null;
  76. uint _latencyLowCount;
  77. public uint LatencyLowCount
  78. {
  79. get
  80. {
  81. return _latencyLowCount;
  82. }
  83. set
  84. {
  85. _latencyLowCount = value;
  86. if (_latencyLowCount==0 && AlarmLatencyLow!=null)
  87. {
  88. Program.AlarmManager.Clear(AlarmLatencyLow,$"CLEARED: {Name} poor latency cleared.");
  89. AlarmLatencyLow = null;
  90. ErrorMsg="";
  91. }
  92. else if (_requestFailCount ==0 && AlarmLatencyLow==null && _latencyLowCount > 3)
  93. {
  94. AlarmLatencyLow = new clsAlarm(clsAlarm.enumAlarmType.Latency,$"WARNING: {Name} latency poor.",this);
  95. ErrorMsg="LATENCY POOR";
  96. Program.AlarmManager.New(AlarmLatencyLow);
  97. }
  98. }
  99. }
  100. clsAlarm AlarmRequestFail = null;
  101. uint _requestFailCount;
  102. public uint RequestFailCount
  103. {
  104. get
  105. {
  106. return _requestFailCount;
  107. }
  108. set
  109. {
  110. _requestFailCount = value;
  111. if (_requestFailCount==0)
  112. {
  113. if (AlarmRequestFail!=null)
  114. {
  115. Program.AlarmManager.Clear(AlarmRequestFail,$"CLEARED: {Name} now responding.");
  116. AlarmRequestFail = null;
  117. GetVersionAsync();
  118. }
  119. if (!String.IsNullOrEmpty(ErrorMsg)) ErrorMsg="";
  120. }
  121. else if (AlarmRequestFail ==null && _requestFailCount == 2)
  122. {
  123. AlarmRequestFail = new clsAlarm(clsAlarm.enumAlarmType.NoResponse,$"WARNING: {Name} not responding.",this);
  124. ErrorMsg="NOT RESPONDING";
  125. Program.AlarmManager.New(AlarmRequestFail);
  126. RunMTRAsync();
  127. }
  128. }
  129. }
  130. public async Task GetHeightAsync(int timeout = 2000)
  131. {
  132. await Task.Run(() => {GetHeight(timeout);});
  133. }
  134. public void GetHeight(int timeout = 2000)
  135. {
  136. try {
  137. var client = new RestClient($"http://{Host}:8088");
  138. client.Timeout = timeout;
  139. var request = new RestRequest("v2", Method.POST);
  140. request.AddHeader("Content-type", "application/json");
  141. request.AddHeader("header", "value");
  142. request.AddJsonBody(
  143. new { jsonrpc = "2.0", id = 0, method = "heights" }
  144. );
  145. // execute the request
  146. var sw = Stopwatch.StartNew();
  147. IRestResponse response = client.Execute(request);
  148. sw.Stop();
  149. if(response.ResponseStatus == ResponseStatus.Completed)
  150. {
  151. var content = response.Content; // raw content as string
  152. var pos1 = 0;
  153. var pos2= 0;
  154. if (!string.IsNullOrEmpty(content))
  155. {
  156. pos1 = content.IndexOf("leaderheight\":");
  157. pos1+=14;
  158. pos2 = content.IndexOf(",",pos1);
  159. uint msgheight=0;
  160. if (UInt32.TryParse(content.Substring(pos1,pos2-pos1),out msgheight))
  161. {
  162. LeaderHeight = msgheight;
  163. }
  164. else
  165. {
  166. ErrorMsg="Invalid data";
  167. }
  168. }
  169. else
  170. {
  171. ErrorMsg="Empty data";
  172. }
  173. LatencyList.Add((int)sw.ElapsedMilliseconds);
  174. PacketLoss.Add(0);
  175. RequestFailCount = 0;
  176. } else if(response.ResponseStatus == ResponseStatus.Error || response.ResponseStatus == ResponseStatus.TimedOut)
  177. {
  178. PacketLoss.Add(100);
  179. ErrorMsg=response.ErrorMessage;
  180. RequestFailCount++;
  181. }
  182. else if(response.ResponseStatus == ResponseStatus.None)
  183. {
  184. ErrorMsg="Empty data";
  185. }
  186. Console.WriteLine(ToString());
  187. }
  188. catch (Exception ex)
  189. {
  190. Console.WriteLine(ex.Message);
  191. RequestFailCount++;
  192. }
  193. }
  194. public async Task GetVersionAsync(int timeout = 2000)
  195. {
  196. await Task.Run(() => {GetVersion(timeout);});
  197. }
  198. public void GetVersion(int timeout = 2000)
  199. {
  200. try {
  201. var client = new RestClient($"http://{Host}:8088");
  202. client.Timeout = timeout;
  203. var request = new RestRequest("v2", Method.POST);
  204. request.AddHeader("Content-type", "application/json");
  205. request.AddHeader("header", "value");
  206. request.AddJsonBody(
  207. new { jsonrpc = "2.0", id = 0, method = "properties" }
  208. );
  209. // execute the request
  210. IRestResponse response = client.Execute(request);
  211. if(response.ResponseStatus == ResponseStatus.Completed)
  212. {
  213. var content = response.Content; // raw content as string
  214. var pos1 = 0;
  215. var pos2= 0;
  216. if (!string.IsNullOrEmpty(content))
  217. {
  218. pos1 = content.IndexOf("factomdversion\":");
  219. pos1+=17;
  220. pos2 = content.IndexOf("\"",pos1);
  221. if (pos2>pos1) NodeVersion = content.Substring(pos1,pos2-pos1);
  222. }
  223. else
  224. {
  225. ErrorMsg="Empty version data";
  226. }
  227. } else if(response.ResponseStatus == ResponseStatus.Error || response.ResponseStatus == ResponseStatus.TimedOut)
  228. {
  229. ErrorMsg=response.ErrorMessage;
  230. }
  231. else if(response.ResponseStatus == ResponseStatus.None)
  232. {
  233. ErrorMsg="Empty data";
  234. }
  235. }
  236. catch (Exception ex)
  237. {
  238. Console.WriteLine(ex.Message);
  239. }
  240. }
  241. async public void PingHostAsync()
  242. {
  243. try
  244. {
  245. var pingTask = Task.Run(() =>
  246. {
  247. Uri myUri = new Uri(Host);
  248. clsRollingAverage pingLatency = new clsRollingAverage(10);
  249. clsRollingAverage pingPacketLoss = new clsRollingAverage(10);
  250. try
  251. {
  252. using (var pinger = new Ping())
  253. {
  254. for (var f=0;f<10;f++)
  255. {
  256. PingReply reply = pinger.Send(myUri.Host,2000);
  257. if (reply.Status == IPStatus.Success)
  258. {
  259. pingPacketLoss.Add((int)reply.RoundtripTime);
  260. pingPacketLoss.Add(0);
  261. Program.SendAlert($"Ping {myUri.Host} {reply.RoundtripTime:0} ms {pingPacketLoss.CurrentAverage:0.0}%");
  262. }
  263. else
  264. {
  265. pingPacketLoss.Add(100);
  266. Program.SendAlert($"Ping {myUri.Host} {reply.Status} ms {pingPacketLoss.CurrentAverage:0.0}%");
  267. }
  268. }
  269. }
  270. }
  271. catch (PingException ex)
  272. {
  273. Program.SendAlert($"Ping error {myUri.Host} {ex.Message}");
  274. }
  275. });
  276. }
  277. catch (Exception ex)
  278. {
  279. Program.SendAlert($"Ping error {Name} {ex.Message}");
  280. }
  281. }
  282. public void RunMTRAsync()
  283. {
  284. try{
  285. Console.WriteLine($"Starting MTR on {Name} {Host}");
  286. var process = new Process
  287. {
  288. StartInfo = { FileName = "/usr/bin/mtr",
  289. Arguments = $"-rw {Host}",
  290. UseShellExecute = false,
  291. RedirectStandardOutput = true,
  292. // RedirectStandardError = true
  293. },
  294. EnableRaisingEvents = true
  295. };
  296. process.Exited += (sender, eargs) =>
  297. {
  298. var mtrOutput = process.StandardOutput.ReadToEnd();
  299. Console.WriteLine(mtrOutput);
  300. if (AlarmRequestFail!=null) AlarmRequestFail.AddNote($"{Name}```{mtrOutput}```");
  301. process.Dispose();
  302. };
  303. process.Start();
  304. }catch (Exception ex)
  305. {
  306. Console.WriteLine($"RunMTRAsync {ex.Message}");
  307. }
  308. }
  309. public void Update(clsNode node)
  310. {
  311. if (Name != node.Name) throw new Exception("index name does not match");
  312. Group = node.Group;
  313. Host = node.Host;
  314. if (Monitor != node.Monitor)
  315. {
  316. Monitor = node.Monitor;
  317. HeightLowCount=0;
  318. LatencyLowCount=0;
  319. RequestFailCount=0;
  320. }
  321. PostPopulate();
  322. }
  323. public string PostPopulate()
  324. {
  325. ErrorMsg = Monitor ? "" : "MONITOR OFF";
  326. return (!Program.NodeGroupList.TryGetValue(Group,out NodeGroup)) ? "Error: Node Group Not Found!" : null;
  327. }
  328. public new String ToString()
  329. {
  330. return $"{Name}\t{Host}\t{LeaderHeight}\t{LatencyList.CurrentAverage.ToString().PadLeft(3)} ms ({(100-PacketLoss.CurrentAverage):0.#}%) {ErrorMsg}";
  331. }
  332. public void AppendDisplayColumns(ref clsColumnDisplay columnDisplay)
  333. {
  334. columnDisplay.AppendCol(Name);
  335. columnDisplay.AppendCol(Host);
  336. columnDisplay.AppendCol(NodeVersion);
  337. columnDisplay.AppendCol($"{LeaderHeight}");
  338. columnDisplay.AppendCol($"{LatencyList.CurrentAverage.ToString().PadLeft(3)} ms ({(100-PacketLoss.CurrentAverage):0.#}%) {ErrorMsg}");
  339. }
  340. }
  341. }