From 65e37d012749326210258cfbdabc0f053fe536ac Mon Sep 17 00:00:00 2001 From: Matt Zagrabelny Date: Fri, 3 Jun 2016 08:17:13 -0500 Subject: [PATCH 1/4] allow exporting process to run arbitrary hooks This commit allows local Netdot instances to run their own code at various hook points in the exporting process. This commit only adds hook points for exporting BIND configs. However, adding hook points for other classes should be straightforward and simple due to leveraging the 'hook' subroutine. Pertinent data is passed to the hook programs via a JSON encoded data structure on the command line. --- etc/Default.conf | 3 + etc/Makefile | 3 +- htdocs/export/config_tasks.html | 10 ++- lib/Netdot/Exporter.pm | 136 ++++++++++++++++++++++++++++++++ lib/Netdot/Exporter/BIND.pm | 38 ++++++++- 5 files changed, 186 insertions(+), 4 deletions(-) diff --git a/etc/Default.conf b/etc/Default.conf index 0f987d247..9c361cafb 100644 --- a/etc/Default.conf +++ b/etc/Default.conf @@ -948,6 +948,9 @@ RANCID_TYPE_MAP => { 'netscreen' => 'netscreen', }, +# Directory that contains export modules and their corresponding hooks +EXPORTER_HOOKS_DIR => '<>/etc/exporter/hooks', + ##################################################################### # - BIND - www.isc.org # diff --git a/etc/Makefile b/etc/Makefile index 4c88c7ad7..d15a3dadb 100644 --- a/etc/Makefile +++ b/etc/Makefile @@ -3,10 +3,11 @@ include $(SRCROOT)/etc/utility-Makefile # # makefile for etc/ - +NDIR = exporter exporter/hooks FILES = Default.conf netdot_apache2_radius.conf netdot_apache2_ldap.conf netdot_apache2_local.conf netdot_apache24_local.conf netdot.meta all: + $(mkdirs) $(substitute) if ! test -r $(PREFIX)/$(DIR)/Site.conf; then \ $(SED) -r $(REPLACEMENT_EXPRESSIONS) Site.conf \ diff --git a/htdocs/export/config_tasks.html b/htdocs/export/config_tasks.html index 4e6335daa..5b93b48ca 100644 --- a/htdocs/export/config_tasks.html +++ b/htdocs/export/config_tasks.html @@ -112,6 +112,7 @@ <%perl> if ( $submit ){ + my $person = $ui->get_user_person($user); unless ( $manager && $manager->can($user, 'access_admin_section', 'Export:Submit_Configuration') ){ $m->comp('/generic/error.mhtml', error=>"You don't have permission to perform this operation") } @@ -130,7 +131,14 @@ $dhcp_logger->add_appender($logstr); foreach my $type ( @config_types ){ - my %args; + my %args = ( + user => { + person_username => $person->username, + person_firstname => $person->firstname, + person_lastname => $person->lastname, + person_email => $person->email, + }, + ); if ( $type eq 'BIND' ){ $args{zone_ids} = \@zones if ( scalar @zones && $zones[0] ne "" ); $args{force} = 1 if ($bind_force); diff --git a/lib/Netdot/Exporter.pm b/lib/Netdot/Exporter.pm index de71231c2..ca1d973e9 100644 --- a/lib/Netdot/Exporter.pm +++ b/lib/Netdot/Exporter.pm @@ -6,6 +6,12 @@ use warnings; use strict; use Data::Dumper; use Fcntl qw(:DEFAULT :flock); +use JSON; +use IPC::Open3; +use File::Spec; +use Symbol 'gensym'; + +use File::Spec::Functions qw(catpath); my $logger = Netdot->log->get_logger('Netdot::Exporter'); @@ -308,6 +314,136 @@ sub print_eof { print $fh "\n#### EOF ####\n"; } +######################################################################## + +=head2 get_short_type - Return the "short" type that this module is. + + Arguments: + None + Returns: + Short type. i.e. "Nagios" or "DHCPD" or etc. + +=cut + +sub get_short_type { + my ($self) = @_; + + my $long_type = ref($self); + + $logger->trace("Getting short type for $long_type."); + + my %long_to_short_lookup = reverse %types; + + if (exists $long_to_short_lookup{$long_type}) { + my $short_type = $long_to_short_lookup{$long_type}; + $logger->debug("Short type for $long_type is $short_type."); + return $short_type; + } + else { + $self->throw_fatal("Netdot::Exporter::get_short_type: No mapping found for long type ($long_type)."); + } +} + +######################################################################## + +=head2 hook - Run the hooks for this point in the export process + + Arguments: + Hash of arguments containing: + name + data + keys. Name is a string that will define which hook programs will run at + various points in the exporting process. Data is arbitrary data passed + to the hook program as a JSON encoded string. Usually "data" is a hash + reference. + Returns: + Nothing. + +=cut + +sub hook { + my $self = shift; + my %args = ( + name => undef, + data => {}, + @_, + ); + + my $name = $args{name}; + my $data = $args{data}; + + if (! defined $name) { + $self->throw_fatal("Netdot::Exporter::hook Name of hook is not defined."); + } + + my $hooks_dir = Netdot->config->get('EXPORTER_HOOKS_DIR'); + $logger->trace("Configuration set hooks dir to $hooks_dir."); + + my $short_type = $self->get_short_type; + + my $module_hook_directory = catpath(undef, $hooks_dir, $short_type); + $logger->trace("Netdot::Exporter::hook for $short_type with name $name has a module_hook_directory of: $module_hook_directory"); + + my $named_hook_directory = catpath(undef, $module_hook_directory, $name); + $logger->trace("Netdot::Exporter::hook for $short_type with name $name has a named_hook_directory of: $named_hook_directory"); + + open(my $dev_null, '<', File::Spec->devnull) or die $!; + + if (-e $named_hook_directory) { + if (-d $named_hook_directory) { + opendir(my $dh, $named_hook_directory) or die; + while(readdir $dh) { + my $entry = catpath(undef, $named_hook_directory, $_); + if (-f $entry && -x $entry) { + $logger->debug("Found executable file $entry for hook $short_type with name $name."); + + # There were issues with the open3 call not being able to + # read from the $output filehandle. The following two + # lines seemed to fix it. + # See: + # http://stackoverflow.com/questions/23770338/perl-embperl-ipcopen3 + local *STDOUT; + open(STDOUT, '>&=', 1) or die $!; + + my $pid = open3( + $dev_null, + my $output, # autovivified filehandle to read from + my $error = gensym, # we can't autovivify stderr filehandle, so use gensym + $entry, # the actual program to run + encode_json($data), # and the arguments to pass on the command line + ) or die $!; + + while (<$output>) { + chomp; + $logger->info("$_"); + } + + while (<$error>) { + chomp; + $logger->error("$_ [from: hook $short_type:$name $entry]"); + } + + waitpid($pid, 0); + + my $child_exit_status = $? >> 8; + if ($child_exit_status != 0) { + $logger->warn("$entry had an exit status of: $child_exit_status"); + } + + close $output or die $!; + close $error or die $!; + } + } + closedir $dh or die; + } + else { + $logger->warn("$named_hook_directory exists, but is not a directory."); + } + } + + close $dev_null or die $!; +} + =head1 AUTHORS Carlos Vicente, C<< >> diff --git a/lib/Netdot/Exporter/BIND.pm b/lib/Netdot/Exporter/BIND.pm index 15b2525c6..f3a98cc04 100644 --- a/lib/Netdot/Exporter/BIND.pm +++ b/lib/Netdot/Exporter/BIND.pm @@ -57,7 +57,7 @@ sub generate_configs { my ($self, %argv) = @_; my @zones; - + if ( $argv{zones} ){ unless ( ref($argv{zones}) eq 'ARRAY' ){ $self->throw_fatal("zones argument must be arrayref!"); @@ -84,7 +84,15 @@ sub generate_configs { }else{ @zones = Zone->retrieve_all(); } - + + $self->hook( + name => 'before-all-zones-written', + data => { + user => $argv{user}, + }, + ); + + my @written_zones = (); foreach my $zone ( @zones ){ next unless $zone->active; eval { @@ -100,6 +108,23 @@ sub generate_configs { $record->update({pending=>0}); } $logger->info("Zone ".$zone->name." written to file: $path"); + my %data = ( + zone_name => $zone->name, + path => $path, + ); + + my %copy_of_data = %data; + + # save a copy so we can send the aggregate to a later "hook". + push @written_zones, \%copy_of_data; + + # add user data in case the hook'ed programs want to use it. + $data{user} = $argv{user}; + + $self->hook( + name => 'after-zone-written', + data => \%data, + ); }else{ $logger->debug("Exporter::BIND::generate_configs: ".$zone->name. ": No pending changes. Use -f to force."); @@ -108,6 +133,15 @@ sub generate_configs { }; $logger->error($@) if $@; } + + my $data = { + zones_written => \@written_zones, + user => $argv{user}, + }; + $self->hook( + name => 'after-all-zones-written', + data => $data, + ); } ############################################################################ From 2114fd228b2768cbd9d91350dbbf7ca66bc7e316 Mon Sep 17 00:00:00 2001 From: Matt Zagrabelny Date: Wed, 27 Jul 2016 10:45:54 -0500 Subject: [PATCH 2/4] include the Netdot name for external hook programs to consume External programs can make good use of various metadata - such as the Netdot name - thus we pass it along. --- lib/Netdot/Exporter.pm | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/Netdot/Exporter.pm b/lib/Netdot/Exporter.pm index ca1d973e9..73fb53268 100644 --- a/lib/Netdot/Exporter.pm +++ b/lib/Netdot/Exporter.pm @@ -376,6 +376,9 @@ sub hook { $self->throw_fatal("Netdot::Exporter::hook Name of hook is not defined."); } + # Always include the Netdot name for the external program to consume. + $data->{netdot_name} = Netdot->config->get('NETDOTNAME'); + my $hooks_dir = Netdot->config->get('EXPORTER_HOOKS_DIR'); $logger->trace("Configuration set hooks dir to $hooks_dir."); From f8b5dba5f60f414a468621735b9f32493436ed3d Mon Sep 17 00:00:00 2001 From: Matt Zagrabelny Date: Fri, 29 Jul 2016 10:45:14 -0500 Subject: [PATCH 3/4] ensure external hook programs are run in predictable order It will beneficial to be able to order the hook programs. Such as: hooks/BIND/after-zone-written/0001-copy-zone-to-production hooks/BIND/after-zone-written/0002-reload-zone hooks/BIND/after-zone-written/0003-flush-cache-on-recursive-servers --- lib/Netdot/Exporter.pm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/Netdot/Exporter.pm b/lib/Netdot/Exporter.pm index 73fb53268..ce071461b 100644 --- a/lib/Netdot/Exporter.pm +++ b/lib/Netdot/Exporter.pm @@ -395,7 +395,8 @@ sub hook { if (-e $named_hook_directory) { if (-d $named_hook_directory) { opendir(my $dh, $named_hook_directory) or die; - while(readdir $dh) { + my @files = sort readdir $dh; + for (@files) { my $entry = catpath(undef, $named_hook_directory, $_); if (-f $entry && -x $entry) { $logger->debug("Found executable file $entry for hook $short_type with name $name."); From 80b807c98baa5ab522857bc2e0a17cb2dce3c74b Mon Sep 17 00:00:00 2001 From: Matt Zagrabelny Date: Fri, 3 Mar 2017 10:54:41 -0600 Subject: [PATCH 4/4] add hooks for DHCPD export --- lib/Netdot/Exporter/DHCPD.pm | 41 ++++++++++++++++++++++++++++++++++- lib/Netdot/Model/DhcpScope.pm | 6 +++-- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/lib/Netdot/Exporter/DHCPD.pm b/lib/Netdot/Exporter/DHCPD.pm index 82bd08e1e..705aad3ab 100644 --- a/lib/Netdot/Exporter/DHCPD.pm +++ b/lib/Netdot/Exporter/DHCPD.pm @@ -68,6 +68,15 @@ sub generate_configs { } } } + + $self->hook( + name => 'before-all-scopes-written', + data => { + user => $argv{user}, + }, + ); + + my @written_scopes = (); foreach my $s ( @gscopes ){ Netdot::Model->do_transaction(sub{ if ( (my @pending = HostAudit->search(scope=>$s->name, pending=>1)) || $argv{force} ){ @@ -76,13 +85,43 @@ sub generate_configs { # Un-mark audit records as pending $record->update({pending=>0}); } - $s->print_to_file(); + my $path = $s->print_to_file(); + + # Only perform "hook" things if we actually wrote out a file... + if (defined $path) { + my %data = ( + scope_name => $s->name, + path => $path, + ); + + my %copy_of_data = %data; + + # save a copy so we can send the aggregate to a later "hook". + push @written_scopes, \%copy_of_data; + + # add user data in case the hook'ed programs want to use it. + $data{user} = $argv{user}; + + $self->hook( + name => 'after-scope-written', + data => \%data, + ); + } }else{ $logger->debug("Exporter::DHCPD::generate_configs: ".$s->name. ": No pending changes. Use -f to force."); } }); } + + my $data = { + scopes_written => \@written_scopes, + user => $argv{user}, + }; + $self->hook( + name => 'after-all-scopes-written', + data => $data, + ); } =head1 AUTHOR diff --git a/lib/Netdot/Model/DhcpScope.pm b/lib/Netdot/Model/DhcpScope.pm index 066ec2935..39e93f761 100644 --- a/lib/Netdot/Model/DhcpScope.pm +++ b/lib/Netdot/Model/DhcpScope.pm @@ -278,7 +278,8 @@ sub delete{ Hash with following keys: filename - (Optional) Returns: - True + Path of file written to if successfully written + undef if scope is not active Examples: $scope->print_to_file(); @@ -293,7 +294,7 @@ sub print_to_file{ unless ( $self->active ){ $logger->info(sprintf("DhcpScope::print_to_file: Scope %s is marked ". "as not active. Aborting", $self->get_label)); - return; + return undef; } my $start = time; @@ -329,6 +330,7 @@ sub print_to_file{ $logger->info(sprintf("DHCPD Scope %s exported to %s, in %s", $self->name, $path, $class->sec2dhms($end-$start) )); + return $path; }