Multiple scripts, go packages, shared configs...

Documentation, links, renaming files for clarity, and all that.

local/bin/README.md

@@ -0,0 +1,33 @@
+# Scripts
+
+All scripts have been normalized to same code standard using `shfmt`.
+Some problematic code has been fixed per `shellcheck` suggestions.
+
+## Homegrown
+
+- dfm
+- x-backup-mysql-with-prefix
+- x-check-git-attributes
+- x-open-ports
+
+## Sourced
+
+| Script                  | Source            |
+|-------------------------|-------------------|
+| `x-dupes`               | skx/sysadmin-util |
+| `x-foreach`             | mvdan/dotfiles    |
+| `x-multi-ping`          | skx/sysadmin-util |
+| `x-ssl-expiry-date`     | skx/sysadmin-util |
+| `x-until-error`         | skx/sysadmin-util |
+| `x-until-success`       | skx/sysadmin-util |
+| `x-validate-sha256-sum` | onnimonni         |
+| `x-when-down`           | skx/sysadmin-util |
+| `x-when-up`             | skx/sysadmin-util |
+
+- Sources:
+    - [skx/sysadmin-utils](https://github.com/skx/sysadmin-util/)
+        - Tools for Linux/Unix sysadmins.
+        - [Licence](https://github.com/skx/sysadmin-util/blob/master/LICENSE)
+    - onnimonni
+        - https://gist.github.com/onnimonni/b49779ebc96216771a6be3de46449fa1
+    - [mvdan/dotfiles](https://github.com/mvdan/dotfiles)

local/bin/dfm

@@ -14,35 +14,42 @@
 
 SCRIPT=$(basename "$0")
 
+source "$HOME/.dotfiles/scripts/shared.sh"
+
 function section_install
 {
-  USAGE_PREFIX="-> $SCRIPT install"
+  USAGE_PREFIX="$SCRIPT install"
 
   case "$1" in
     all)
-      bash "$DOTFILES/scripts/settler.sh" && echo "🎉 Done!"
+      bash "$DOTFILES/scripts/settler.sh" && msg_done "🎉 Done!"
       ;;
     antigen)
-      curl -L git.io/antigen > "$DOTFILES/local/bin/antigen.zsh" && echo "🎉 Done!"
+      curl -L git.io/antigen > "$DOTFILES/local/bin/antigen.zsh" && msg_done "🎉 Done!"
       ;;
     defaults)
-      bash "$DOTFILES/scripts/set-defaults.sh" && echo "🎉 Done!"
+      bash "$DOTFILES/scripts/set-defaults.sh" && msg_done "🎉 Done!"
       ;;
-    gh-extensions)
-      bash "$DOTFILES/scripts/gh-extensions.sh" && echo "🎉 Done!"
+    ext_go)
+      bash "$DOTFILES/scripts/install-go-packages" && msg_done "🎉 Done!"
+      ;;
+    ext_gh)
+      bash "$DOTFILES/scripts/install-gh-extensions" && msg_done "🎉 Done!"
       ;;
     *)
-      echo "$USAGE_PREFIX [antigen | defaults | gh-extensions]"
-      echo " * antigen:       Updates the antigen.zsh file"
-      echo " * defaults:      Setup nice macOS defaults"
-      echo " * gh-extensions: Install GitHub CLI Extensions"
+      menu_section "$USAGE_PREFIX" "all | antigen | defaults | ext_go | ext_gh"
+      menu_item "all"      "Runs the WIP settler.sh"
+      menu_item "antigen"  "Updates the antigen.zsh file"
+      menu_item "defaults" "Setup nice macOS defaults"
+      menu_item "ext_go"   "Install Go Packages"
+      menu_item "ext_gh"   "Install GitHub CLI Extensions"
       ;;
   esac
 }
 
 function section_brew
 {
-  USAGE_PREFIX="-> $SCRIPT brew"
+  USAGE_PREFIX="$SCRIPT brew"
 
   if ! command -v brew &> /dev/null; then
     echo "brew could not be found, please install it first"
@@ -60,24 +67,27 @@ function section_brew
       ;;
     updatebundle)
       # Updates .dotfiles/Brewfile with descriptions
-      brew bundle dump --force --file="$BREWFILE" --describe && echo "🎉 Done!"
+      brew bundle dump \
+        --force \
+        --file="$BREWFILE" \
+        --describe && msg_ok "🎉 Done!"
       ;;
     autoupdate)
       brew autoupdate start 43200 --upgrade --cleanup --immediate
       ;;
     *)
-      echo "$USAGE_PREFIX [install | update | updatebundle]"
-      echo " * install:      Installs items defined in Brewfile"
-      echo " * update:       Updates and upgrades brew packages"
-      echo " * updatebundle: Updates Brewfile with descriptions"
-      echo " * autoupdate:   Setups brew auto-update and runs it immediately"
+      menu_section "$USAGE_PREFIX" "install | update | updatebundle | autoupdate"
+      menu_item "install" "Installs items defined in Brewfile"
+      menu_item "update" "Updates and upgrades brew packages"
+      menu_item "updatebundle" "Updates Brewfile with descriptions"
+      menu_item "autoupdate" "Setups brew auto-update and runs it immediately"
       ;;
   esac
 }
 
 function section_dotfiles
 {
-  USAGE_PREFIX="-> $SCRIPT dotfiles"
+  USAGE_PREFIX="$SCRIPT dotfiles"
 
   case "$1" in
     link)
@@ -108,18 +118,44 @@ function section_dotfiles
         "$DOTFILES"
       ;;
     *)
-      echo "$USAGE_PREFIX [link | update | shfmt]"
-      echo " * link:         Use rcrc to update dotfile links"
-      echo " * update:       Updates dotfile links, installs host specific overrides automatically"
-      echo " * shfmt:        Run shfmt to all dotfiles"
+      menu_section "$USAGE_PREFIX" "link | update | shfmt"
+      menu_item "link" "Use rcrc to update dotfile links"
+      menu_item "update" "Updates dotfile links, installs host specific overrides automatically"
+      menu_item "shfmt" "Run shfmt to all dotfiles"
+      ;;
+  esac
+}
+
+# Secret menu for visual tests
+function section_tests
+{
+  USAGE_PREFIX="$SCRIPT tests"
+
+  case "$1" in
+    msg)
+      msg "msg"
+      msg_done "msg_done"
+      msg_prompt "msg_prompt"
+      msg_prompt_done "msg_prompt_done"
+      msg_nested "msg_nested"
+      msg_nested_done "msg_nested_done"
+      msg_run "msg_run" "second_param"
+      msg_ok "msg_ok"
+      msg_warn "msg_warn"
+      msg_err "msg_err"
+      ;;
+    *)
+      menu_section "$USAGE_PREFIX" "msg"
+      menu_item "msg" "List all log functions from shared.sh"
       ;;
   esac
 }
 
 function usage
 {
-  echo $"Usage: $SCRIPT [install | brew | dotfiles]"
-  echo $" All commands have their own subcommands."
+  echo ""
+  menu_section "Usage: $SCRIPT" "install | brew | dotfiles"
+  echo $"   All commands have their own subcommands."
   echo ""
   section_install
   echo ""
@@ -133,5 +169,6 @@ case "$1" in
   install) section_install "$2" ;;
   brew) section_brew "$2" ;;
   dotfiles) section_dotfiles "$2" ;;
+  tests) section_tests "$2" ;;
   *) usage && exit 1 ;;
 esac

local/bin/x-dupes

@@ -0,0 +1,208 @@
+#!/usr/bin/env perl
+
+=head1 NAME
+
+dupes - Report on files with duplicate contents, via SHA1 hash.
+
+=cut
+
+=head1 SYNOPSIS
+
+  dupes [options] directory
+
+  General Options:
+
+   --help      Show the help information for this script.
+   --verbose   Show useful debugging information.
+
+=cut
+
+
+=head1 ABOUT
+
+dupes is a simple script to report upon files that are identical,
+recursively.
+
+The process involves calculating the SHA1 hash of the file contents
+and reporting on anything collisions we see.
+
+Note that a collision might be caused by a symbolic link, or hardlink,
+so blindly deleting duplicates without investigation is almost certainly
+a mistake.
+
+=cut
+
+=head1 AUTHOR
+
+ Steve
+ --
+ http://www.steve.org.uk/
+
+=cut
+
+
+=head1 LICENSE
+
+Copyright (c) 2013 by Steve Kemp.  All rights reserved.
+
+This script is free software;you can redistribute it and/or modify it under
+the same terms as Perl itself.
+
+The LICENSE file contains the full text of the license.
+
+=cut
+
+
+
+use strict;
+use warnings;
+
+use File::Find;
+use Getopt::Long;
+use Pod::Usage;
+
+
+#
+#  Parse the arguments
+#
+my %config = parsedOptions();
+
+
+#
+#  The path to examine.
+#
+my $path = $ARGV[0] || '.';
+
+
+#
+#  Get the hashing object, dynamically.
+#
+my $ctx = getHashObject();
+my %digest;
+
+
+#
+#  Find files and store the hash of their contents.
+#
+find( {
+       'wanted' => sub {
+           if ( -f $_ )
+           {
+               lstat;
+               if ( ( -r _ ) && ( !-l _ ) )
+               {
+                   $ctx->reset;
+                   $ctx->addfile($_);
+                   my $md5 = $ctx->hexdigest;
+                   if ( exists $digest{ $md5 } )
+                   {
+                       push @{ $digest{ $md5 }->{ 'dupes' } }, $_;
+                   }
+                   else
+                   {
+                       $digest{ $md5 } = { 'file'  => $_,
+                                           'dupes' => [] };
+                   }
+               }
+           }
+           else
+           {
+               $config{ 'verbose' } && print "Entering $_\n";
+           }
+       },
+       'no_chdir' => 1
+    },
+    $path
+    );
+
+
+#
+#  Report upon collisions.
+#
+foreach my $hash ( keys %digest )
+{
+    my $dupes = $digest{ $hash }->{ 'dupes' };
+    my $src   = $digest{ $hash }->{ 'file' };
+
+    if (@$dupes)
+    {
+        print $src . "\n";
+        foreach my $dupe (@$dupes)
+        {
+            print "\t$dupe\n";
+        }
+    }
+}
+
+
+#
+#  All done.
+#
+exit(0);
+
+
+=begin doc
+
+Load one of M<Digest::SHA> and M<Digest::SHA1>, depending on what is available.
+
+=end doc
+
+=cut
+
+sub getHashObject
+{
+    my $hash = undef;
+
+    foreach my $module (qw! Digest::SHA Digest::SHA1 !)
+    {
+
+        # If we succeeded in calculating the hash we're done.
+        next if ( defined($hash) );
+
+        # Attempt to load the module
+        my $eval = "use $module;";
+
+        ## no critic (Eval)
+        eval($eval);
+        ## use critic
+
+        if ( !$@ )
+        {
+            $hash = $module->new;
+        }
+
+    }
+
+    if ($hash)
+    {
+        return ($hash);
+    }
+    else
+    {
+        print "Failed to load either DIgest::SHA or Digest::SHA1\n";
+        exit(1);
+    }
+}
+
+=begin doc
+
+Parse the options and return suitable values.
+
+=end doc
+
+=cut
+
+sub parsedOptions
+{
+    my %vars;
+
+    exit
+      if (
+           !GetOptions( "help"    => \$vars{ 'help' },
+                        "verbose" => \$vars{ 'verbose' } ) );
+
+    pod2usage(1) if ( $vars{ 'help' } );
+
+    return (%vars);
+
+}

local/bin/x-multi-ping

@@ -0,0 +1,228 @@
+#!/usr/bin/env perl
+
+=head1 NAME
+
+multi-ping - Multi-protocol ping wrapper
+
+=head1 SYNOPSIS
+
+  multi-ping [--loop|--forever] [--sleep=N]  hostname1 hostname2 .. hostnameN
+
+=cut
+
+=head1 DESCRIPTION
+
+This wrapper script will invoke one of 'ping' or 'ping6', as appropriate,
+to test the connectivity of a remote host and your route to it.
+
+=cut
+
+=head1 AUTHOR
+
+ Steve
+ --
+ http://www.steve.org.uk/
+
+=cut
+
+=head1 LICENSE
+
+Copyright (c) 2013-2014 by Steve Kemp.  All rights reserved.
+
+This script is free software; you can redistribute it and/or modify it under
+the same terms as Perl itself.
+
+The LICENSE file contains the full text of the license.
+
+=cut
+
+
+use strict;
+use warnings;
+
+use Getopt::Long;
+use Pod::Usage;
+
+
+#
+#  Check the dependencies.
+#
+checkSystem();
+
+#
+#  Parse the command-line.
+#
+my %config = parsedOptions();
+
+
+#
+#  We could parse arguments here, for the moment we will hard-wire
+# a timeout of five seconds.
+#
+my $timeout = 5;
+
+
+#
+# If we didn't get any arguments then we should complain.
+#
+if ( scalar @ARGV < 1 )
+{
+    print "Usage: multi-ping hostname1 hostname2 .. hostnameN\n";
+    exit(1);
+}
+
+
+#
+# Process each hostname specified upon the command-line.
+#
+# Looping if applicable.
+#
+do
+{
+    foreach my $host (@ARGV)
+    {
+        pingHost($host);
+    }
+
+    sleep( $config{ 'sleep' } );
+
+} until ( !$config{ 'loop' } );
+
+
+
+
+=begin doc
+
+Given a hostname lookup both the AAAA & A records. For each result
+perform the appropriate ping request.
+
+=end doc
+
+=cut
+
+sub pingHost
+{
+    my ($hostname) = (@_);
+
+    #
+    #  If we've been given an URI then we'll remove the leading-scheme
+    # and use the hostname only.
+    #
+    if ( $hostname =~ m!^([a-z]+:\/\/)([^/]+)/?!i )
+    {
+        $hostname = $2;
+
+        #
+        #  Port might be specified too, drop that if present.
+        #
+        if ( $hostname =~ /^(.*):([0-9]+)$/ )
+        {
+            $hostname = $1;
+        }
+    }
+
+    #
+    #  Lookup the IP for the name specified
+    #
+    my $res = Net::DNS::Resolver->new;
+
+    #
+    #  The two types we'll lookup.
+    #
+    #  NOTE: This shouldn't be required but my laptop resolver seems to
+    # struggle with lookups of the type "ANY".  Sigh.
+    #
+    #
+    foreach my $type (qw! A AAAA !)
+    {
+        my $query = $res->query( $hostname, $type );
+
+        if ($query)
+        {
+            foreach my $rr ( $query->answer )
+            {
+                my $ping_binary =
+                  $rr->type eq "A"    ? "ping" :
+                  $rr->type eq "AAAA" ? "ping6" :
+                                        "";
+                if ($ping_binary)
+                {
+                    my $result = system(
+                        "$ping_binary -c1 -w$timeout -W$timeout $hostname >/dev/null 2>/dev/null"
+                    );
+
+                    print "Host $hostname - " . $rr->address() .
+                      ( ( $result == 0 ) ? " - alive" : " - FAILED"  ) . "\n";
+                }
+            }
+        }
+        else
+        {
+            print "WARNING: Failed to resolve $hostname [$type]\n";
+        }
+    }
+}
+
+
+
+=begin doc
+
+Test that we have the required perl dependencies present.
+
+=end doc
+
+=cut
+
+sub checkSystem
+{
+    my $eval = "use Net::DNS;";
+
+    ## no critic (Eval)
+    eval($eval);
+    ## use critic
+
+    #
+    #  If we don't have Net::DNS we're out of luck.
+    #
+    if ($@)
+    {
+        print "The required Net::DNS module is missing.  Aborting.\n";
+        exit(1);
+    }
+
+}
+
+
+
+=begin doc
+
+Parse the options and return suitable values.
+
+=end doc
+
+=cut
+
+sub parsedOptions
+{
+    my %vars;
+
+    #
+    # Defaults
+    #
+    $vars{ 'loop' }  = 0;
+    $vars{ 'sleep' } = 1;
+
+    exit
+      if (
+           !GetOptions( "help"    => \$vars{ 'help' },
+                        "verbose" => \$vars{ 'verbose' },
+                        "forever" => \$vars{ 'loop' },
+                        "loop"    => \$vars{ 'loop' },
+                        "sleep=i" => \$vars{ 'sleep' },
+                      ) );
+
+    pod2usage(1) if ( $vars{ 'help' } );
+
+    return (%vars);
+
+}

local/bin/x-ssl-expiry-date

@@ -0,0 +1,133 @@
+#!/bin/sh
+#
+# About
+# -----
+# Check the expiry date of the SSL certificate on the given host.
+#
+#
+# License
+# -------
+#
+# Copyright (c) 2013-2015 by Steve Kemp.  All rights reserved.
+#
+# This script is free software; you can redistribute it and/or modify it under
+# the same terms as Perl itself.
+#
+# The LICENSE file contains the full text of the license.
+#
+#
+
+#
+#  Simple function to show usage information, and exit.
+#
+usage() {
+    echo "Usage: $0 [-d] [-p 443] domain1.org domain2.com .. domainN"
+    exit 0
+}
+
+#
+#  Default settings for flags set by the command-line arguments
+#
+days=0
+port=443
+
+#
+#  Parse the argument(s) - i.e. look for "-d" / "-p 443".
+#
+while getopts "h?dp:" opt; do
+    case $opt in
+        h)
+            usage
+            ;;
+        \?)
+            usage
+            ;;
+        d)
+            days=1
+            ;;
+        p)
+            port=$OPTARG
+            ;;
+  esac
+done
+shift $((OPTIND - 1))
+
+#
+#  Ensure we have some arguments
+#
+if [ "$#" = "0" ]; then
+    usage
+fi
+
+#
+# For each domain-name on the command-line.
+#
+for name in "$@"; do
+
+    #
+    #  Make a temporary file
+    #
+    # Test if we have BSD or GNU version of mktemp
+    if ( strings "$(which mktemp)" | grep -q GNU); then
+        # We have the GNU version
+        tmp=$(mktemp)
+  else
+        # We have the BSD version
+        tmp=$(mktemp -t tmp)
+  fi
+
+    #
+    #  Download the certificate
+    #
+    if ( ! echo "" | openssl s_client -connect $name:$port > $tmp 2> /dev/null); then
+        echo "Failed to get cert from https://$name:$port/"
+        exit 3
+  fi
+
+    #
+    #  Get the expiry date
+    #
+    date=$(openssl x509 -in "$tmp" -noout -enddate | awk -F= '{print $2}')
+
+    #
+    #  Remove the temporary file
+    #
+    rm -f "$tmp"
+
+    #
+    #  Convert the expiry date + todays date to seconds-past epoch
+    #
+    # Check if we have the BSD or the GNU version of date
+    if (strings "$(which date)" | grep -q GNU); then
+        # We have GNU this is easy
+        then=$(date --date "$date" +%s)
+  else
+        # We have BSD now it is getting complicated
+        year=$(echo "$date" | awk '{print $4}')
+        month=$(echo "$date" | awk '{print $1}')
+        day=$(echo "$date" | awk '{print $2}')
+        hour=$(echo "$date" | awk '{print $3}' | awk -F: '{print $1}')
+        minute=$(echo "$date" | awk '{print $3}' | awk -F: '{print $2}')
+        second=$(echo "$date" | awk '{print $3}' | awk -F: '{print $3}')
+        then=$(date -v${year}y -v${month} -v${day}d -v${hour}H -v${minute}M -v${second}S -u +%s)
+  fi
+
+    now=$(date +%s)
+
+    #
+    #  Day diff
+    #
+    diff=$(("$then" - "$now"))
+    diff=$($diff / 86400)
+
+    #
+    #  All done
+    #
+    if [ "$days" = "1" ]; then
+        echo "${name}: ${diff}"
+  else
+        echo "$name"
+        echo "    Expires: ${date}"
+        echo "    Days: ${diff}"
+  fi
+done

local/bin/x-until-error

@@ -0,0 +1,15 @@
+#!/bin/sh
+#
+# About
+# -----
+# Repeat the command until it fails - always run at least once.
+#
+
+"$@"
+
+#
+#  If the status code was zero then repeat.
+#
+while [ $? -eq 0 ]; do
+    "$@"
+done

local/bin/x-until-success

@@ -0,0 +1,30 @@
+#!/bin/sh
+#
+# About
+# -----
+# Repeat the command until it succeeds - always run at least once.
+#
+#
+# License
+# -------
+#
+# Copyright (c) 2013 by Steve Kemp.  All rights reserved.
+#
+# This script is free software; you can redistribute it and/or modify it under
+# the same terms as Perl itself.
+#
+# The LICENSE file contains the full text of the license.
+#
+#
+
+#
+#  Run the first time.
+#
+"$@"
+
+#
+#  If the status code was not zero then repeat.
+#
+while [ $? -ne 0 ]; do
+    "$@"
+done

local/bin/x-when-down

@@ -0,0 +1,32 @@
+#!/bin/sh
+#
+# Wait until a given host is down (determined by ping) then execute the
+# given command
+#
+# Usage:
+# ./when-down HOST COMMAND...
+#
+# Example
+# ./when-down 1.2.3.4 ssh 1.2.3.4
+#
+
+#
+#  Ensure we received the correct number of arguments.
+#
+if [ $# -lt 2 ]; then
+    echo "Usage: $0 HOST COMMAND..."
+    exit 1
+fi
+
+HOST=$1
+
+echo "Waiting for $HOST to get down..."
+
+true
+while [ $? -ne 1 ]; do
+    ping -c 1 -W 1 $HOST > /dev/null
+done
+
+shift
+
+"$@"

local/bin/x-when-up

@@ -0,0 +1,45 @@
+#!/bin/sh
+#
+# Wait until a given host is online (determined by ping) then execute the
+# given command
+#
+# Usage:
+# ./when-up HOST COMMAND...
+#
+# Example
+# ./when-up 1.2.3.4 ssh 1.2.3.4
+#
+# Special case:
+# when using when-up to ssh to a host, this host does not need to be given twice
+# ./when-up ssh 1.2.3.4
+#
+
+#
+#  Ensure we received the correct number of arguments.
+#
+if [ $# -lt 2 ]; then
+    echo "Usage: $0 HOST COMMAND..."
+    exit 1
+fi
+
+if [ $1 = "ssh" ]; then
+  HOST=$2
+else
+  HOST=$1
+fi
+
+echo "Waiting for $HOST to come online..."
+
+ping -c 1 -W 1 $HOST > /dev/null
+while [ $? -ne 0 ]; do
+    sleep 1
+    ping -c 1 -W 1 $HOST > /dev/null
+done
+
+# By the time we reach here the ping-command has completed successfully
+# so we can launch the command we were given - along with any arguments.
+if [ $1 != "ssh" ]; then
+  shift
+fi
+
+"$@"