source: gs2-extensions/parallel-building/trunk/src/opt/Perseus/perseus.pl@ 29649

Last change on this file since 29649 was 29649, checked in by jmt12, 9 years ago

Perseus was an attempt to add functionality to automatically and remote fix Medusa is processes went run-away. The idea was the Persus server would either run as a daemon or be periodically called by cron, it would look for a sentinel file or be contacted by a client, and then it would slay processes belonging to a certain user. Never actually used, because the problems were found to be NFS/system level and impossible to solve without a hard reset anyway

  • Property svn:executable set to *
File size: 10.6 KB
Line 
1#!/usr/bin/perl
2
3# Pragma
4use strict;
5use warnings;
6$|++; # Unbuffer STDOUT
7
8# Perl Built-in Modules
9use Getopt::Long;
10use IO::Socket::INET;
11use LWP::Simple;
12use Pod::Usage;
13
14my $txt_buffer = '';
15
16my $start_time = time();
17&printBuffer("[" . localtime($start_time) . "]\n");
18&printBuffer("===== Persus =====\n");
19&printBuffer("To slay a Medusa you need an appropriately shiney shield - this is\n";
20&printBuffer("that shield.\n");
21&printBuffer("\n");
22
23# 1. Configuration
24my $debug = 0;
25my $help = 0;
26my $kill_delay = 5;
27my $listen_time = 60;
28my $man = 0;
29my $search_path = '';
30my $search_port = 39875;
31my $search_url = 'http://www.cms.waikato.ac.nz/~jmt12/perseus.txt';
32my $log_path = '/tmp/perseus.log';
33GetOptions ('debug' => \$debug,
34 'kill_delay=i' => \$kill_delay,
35 'listen_port=i' => \$search_port,
36 'listen_time=i' => \$listen_time,
37 'path=s' => \$search_path,
38 'url=s' => \$search_url,
39 'log=s' => \$log_path,
40 'help|?' => \$help,
41 man => \$man
42 )
43or pod2usage(2);
44if ($help)
45{
46 pod2usage(1);
47}
48if ($man)
49{
50 pod2usage(-exitval => 0, -verbose => 2);
51}
52&printBuffer(" * Configuration\n");
53&printBuffer(" - Debug? " . ($debug ? "Yes" : "No") . "\n");
54&printBuffer(" - Search path: " . $search_path . "\n");
55&printBuffer(" - Search URL: " . $search_url . "\n");
56&printBuffer(" - Search port: " . $search_port . "\n");
57&printBuffer(" - Listen time: " . $listen_time . " seconds\n");
58&printBuffer(" - Delay before KILL: " . $kill_delay . " seconds\n");
59&printBuffer(" - Log path: " . $log_path . "\n");
60
61# 2. Search
62&printBuffer(" * Looking for Instructions\n");
63my $instructions = [];
64&printBuffer(" - Listen on port... ");
65if ($search_port > 0)
66{
67 if (&searchPort())
68 {
69 &printBuffer("Contacted\n");
70 }
71 else
72 {
73 &printBuffer("No contact\n");
74 }
75}
76else
77{
78 &printBuffer("Skipped\n");
79}
80&printBuffer(" - Search for internet page... ");
81if (!scalar(@{$instructions}))
82{
83 if (&searchURL())
84 {
85 &printBuffer("Found\n");
86 }
87 else
88 {
89 &printBuffer("Not found\n");
90 }
91}
92else
93{
94 &printBuffer("Skipped\n");
95}
96&printBuffer(" - Search for sentinel file... ");
97if (defined $search_path && $search_path ne '' && !scalar(@{$instructions}))
98{
99 if (&searchPath())
100 {
101 &printBuffer("Found\n");
102 }
103 else
104 {
105 &printBuffer("Not found\n");
106 }
107}
108else
109{
110 &printBuffer("Skipped\n");
111}
112
113# 3. Action
114if (scalar(@{$instructions}))
115{
116 &printBuffer(" * Processing Instructions\n");
117 my @instructions2;
118 # - terms and kills first
119 my $instruction_ptr = shift(@{$instructions});
120 while (defined $instruction_ptr)
121 {
122 my @instruction = @{$instruction_ptr};
123 &printDebug("[instruction=" . $instruction[0] . ':' . $instruction[1] . "]");
124 if ($instruction[0] eq 'pid')
125 {
126 &commandTerminate($instruction[1]);
127 }
128 else
129 {
130 push(@instructions2, $instruction_ptr);
131 }
132 $instruction_ptr = shift(@{$instructions});
133 }
134 # - then slays
135 $instruction_ptr = shift(@instructions2);
136 while (defined $instruction_ptr)
137 {
138 my @instruction = @{$instruction_ptr};
139 if ($instruction[0] eq 'user')
140 {
141 &commandSlay($instruction[1]);
142 }
143 else
144 {
145 push(@{$instructions}, $instruction_ptr);
146 }
147 $instruction_ptr = shift(@instructions2);
148 }
149 # - then commands (reboot)
150 $instruction_ptr = shift(@{$instructions});
151 while (defined $instruction_ptr)
152 {
153 my @instruction = @{$instruction_ptr};
154 if ($instruction[0] eq 'cmd' && $instruction[1] eq 'reboot')
155 {
156 &commandReboot();
157 }
158 else
159 {
160 push(@instructions2, $instruction_ptr);
161 }
162 $instruction_ptr = shift(@{$instructions});
163 }
164 # - left overs
165 foreach $instruction_ptr (@instructions2)
166 {
167 my @instruction = @{$instruction_ptr};
168 &printBuffer("Error! Unrecognized instruction \"" . $instruction[0] . ":" . $instruction[1] . "\"\n");
169 }
170}
171else
172{
173 &printBuffer(" * No Instructions!\n");
174}
175
176# 4. Complete!
177my $end_time = time();
178my $duration = $end_time - $start_time;
179&printBuffer("===== Complete in " . $duration . " seconds =====\n");
180&printBuffer("[" . localtime($end_time) . "]\n\n");
181
182# 5. Try and write to log path (hopefully before a reboot shuts us down)
183# Needs to be in eval{} in case filesystem is non-responsive
184eval
185{
186 local $SIG{ALRM} = sub { die 'timeout' };
187 alarm 5; # This is a lifetime for file writing
188 if (open(LOGOUT, '>:utf8', $log_path))
189 {
190 print LOGOUT $txt_buffer;
191 close (LOGOUT);
192 }
193 else
194 {
195 print "Error! Failed to open log for writing\n";
196 }
197 alarm 0;
198};
199alarm 0;
200if (defined $@ && $@ ne '')
201{
202 print "Error! Failed to write log - " . $@ . ":" . $! . "\n";
203}
204
205exit;
206
207## @function
208#
209sub printBuffer
210{
211 my ($msg) = @_;
212 print $msg;
213 if (defined $log_path && $log_path ne '')
214 {
215 $txt_buffer .= $msg;
216 }
217}
218## printBuffer() ##
219
220## @function commandIsRunning
221#
222sub commandIsRunning
223{
224 my ($pid) = @_;
225 my $result == 1;
226 # yeah - timeout this command too, as I don't know if PS is hang-safe
227 eval
228 {
229 local $SIG{ALRM} = sub { die 'timeout' };
230 alarm 5; # This is a lifetime for file writing
231 $result = `ps -p $pid > /dev/null; echo $?`;
232 chomp($result);
233 alarm 0;
234 };
235 alarm 0;
236 if (defined $@ && $@ ne '')
237 {
238 &printBuffer("[ps timed out] ");
239 }
240 return ($result == 0);
241}
242## commandIsRunning() ##
243
244## @function commandReboot
245#
246# Wait five minutes then reboot the machine
247#
248sub commandReboot
249{
250 &printBuffer(" - rebooting\n");
251 &printBuffer(&shellCommand('shutdown -r +5', 1));
252}
253## commandReboot ##
254
255## @function commandSlay
256#
257sub commandSlay
258{
259 my ($user) = @_;
260 &printBuffer(" - slay all processes by: " . $user . "\n");
261 $user =~ s/"//g;
262 &printBuffer(&shellCommand('slay -clean "' . $user . '"'));
263}
264## commandSlay() ##
265
266## @function commandTerminate
267#
268sub commandTerminate
269{
270 my ($pid) = @_;
271 &printBuffer(" - terminate process: " . $pid . "... ");
272 # send SIGTERM
273 &printBuffer(&shellCommand('kill -s SIGTERM ' . $pid));
274 # wait delay seconds
275 sleep($kill_delay);
276 # see if process still there
277 if (&commandIsRunning($pid))
278 {
279 &printBuffer("Failed\n");
280 &printBuffer(" - kill process: " . $pid . "... ");
281 &printBuffer(&shellCommand('kill -s SIGKILL ' . $pid));
282 sleep($kill_delay);
283 if (&commandIsRunning($pid))
284 {
285 &printBuffer("Failed\n");
286 }
287 else
288 {
289 &printBuffer("Killed\n");
290 }
291 }
292 else
293 {
294 &printBuffer("Terminated\n");
295 }
296 # if it is still running then it must be uninterruptable... nothing we can
297 # do from here
298}
299## commandTerminate() ##
300
301
302## @function parseInstructions
303#
304sub parseInstructions
305{
306 my ($content) = @_;
307 my $instruction_count = 0;
308 while ($content =~ s/(pid|user|cmd)\:([^\s]+)//i)
309 {
310 my $type = lc($1);
311 my $value = $2;
312 push(@{$instructions}, [$1, $2]);
313 $instruction_count++;
314 }
315 return $instruction_count;
316}
317## parseInstructions() ##
318
319
320## @function printDebug()
321#
322sub printDebug
323{
324 my ($msg) = @_;
325 if ($debug)
326 {
327 &printBuffer("[DEBUG] " . $msg . " [" . time() . "]\n");
328 }
329}
330## printDebug() ##
331
332
333## @function searchPath()
334#
335sub searchPath
336{
337 my $result = 0;
338 # A user can ask us to do stuff by dumping a file here
339 eval
340 {
341 local $SIG{ALRM} = sub { die 'timeout' };
342 alarm $listen_time;
343 &printBuffer("[opening... ");
344 # read instructions (<type>:<value>\n) from the file
345 if (open(FIN, '<:utf8', $search_path))
346 {
347 &printBuffer("reading... ");
348 my $content;
349 sysread(FIN, $content, -s $search_path);
350 &printBuffer("parsing... ");
351 $result = &parseInstructions($content);
352 close(FIN);
353 # remove the file (if we can)
354 &printBuffer("deleting... ");
355 unlink($search_path);
356 &printBuffer("complete] ");
357 }
358 else
359 {
360 &printBuffer("failed] ");
361 }
362 alarm 0; # reset alarm
363 };
364 alarm 0; # reset alarm
365 if (defined $@ && $@ ne '')
366 {
367 if ($@ =~ /timeout/)
368 {
369 &printBuffer("timed out] ");
370 }
371 else
372 {
373 &printBuffer("error] ");
374 }
375 }
376 return $result;
377}
378## searchPath() ##
379
380
381## @function searchPort
382#
383sub searchPort
384{
385 my $result = 0;
386 # timeout isn't implemented in Perl sockets, so we cheat and use an alarm
387 # that interrupts an eval block
388 eval {
389 local $SIG{ALRM} = sub { die 'timeout' };
390 alarm $listen_time;
391 my $socket = IO::Socket::INET->new(Proto => 'tcp',
392 LocalPort => $search_port,
393 Listen => 1,
394 ReuseAddr => 1
395 );
396 if (defined $socket)
397 {
398 &printBuffer("[listening... ");
399 while (my $client = $socket->accept())
400 {
401 &printBuffer("connected... ");
402 $client->autoflush(1);
403 my $line;
404 while ($line = <$client>)
405 {
406 &printBuffer("receiving... ");
407 $result += &parseInstructions($line);
408 }
409 close $client;
410 &printBuffer("complete] [listening... ");
411 }
412 }
413 else
414 {
415 &printBuffer("failed] ");
416 }
417 alarm 0; # reset alarm
418 };
419 alarm 0; # reset alarm
420 if (defined $@ && $@ ne '')
421 {
422 if ($@ =~ /timeout/)
423 {
424 &printBuffer("timed out] ");
425 }
426 else
427 {
428 &printBuffer("error] ");
429 }
430 }
431 return $result;
432}
433## searchPort() ##
434
435
436## @function searchURL
437#
438sub searchURL
439{
440 my $result = 0;
441 &printBuffer("[downloading... ");
442 my $content = get($search_url);
443 if (defined $content)
444 {
445 &printBuffer("parsing... ");
446 $result = &parseInstructions($content);
447 &printBuffer("complete] ");
448 }
449 else
450 {
451 &printBuffer("failed] ");
452 }
453 return $result;
454}
455## searchURL() ##
456
457
458## @function shellCommand()
459#
460sub shellCommand
461{
462 my ($cmd, $background) = @_;
463 my $result = '';
464 # timeout all shell commands
465 eval
466 {
467 local $SIG{ALRM} = sub { die 'timeout' };
468 alarm 5; # This is a lifetime for file writing
469 if ($debug)
470 {
471 &printDebug('shellCommand(\'' . $cmd . '\')');
472 }
473 elsif (defined $background)
474 {
475 $result = `$cmd 2>&1 &`;
476 }
477 else
478 {
479 $result = `$cmd 2>&1`;
480 }
481 alarm 0;
482 };
483 alarm 0;
484 if (defined $@ && $@ ne '')
485 {
486 &printBuffer("[shell command timed out] ");
487 }
488 return $result;
489}
490## shellCommand() ##
491
492__END__
493=head1 NAME
494sample - Using GetOpt::Long and Pod::Usage
495=head1 SYNOPSIS
496sample [options] [file ...]
497 Options:
498 -help brief help message
499 -man full documentation
500=head1 OPTIONS
501=over 8
502=item B<-help>
503Print a brief help message and exits.
504=item B<-man>
505Prints the manual page and exits.
506=back
507=head1 DESCRIPTION
508B<This program> will read the given input file(s) and do something useful with the contents thereof.
509=cut
Note: See TracBrowser for help on using the repository browser.