Merge 9a4a6d2b2a into d5b493a53e

Update port location of edk2

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+patreon: # Replace with a single Patreon username
+open_collective: freebsd-vm-bhyve
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2015, churchers
+Copyright (c) 2015-2016, churchers
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

Makefile

@@ -3,34 +3,38 @@
 #
 
 PREFIX?=/usr/local
-MAN=
-BINOWN=root
-BINGRP=wheel
-BINMODE=0500
-BINDIR=$(PREFIX)/sbin
-FILESDIR=$(PREFIX)/lib/vm-bhyve
-EXAMPLESDIR=${PREFIX}/share/examples/vm-bhyve
-RCDIR=$(PREFIX)/etc/rc.d
-MANDIR=$(PREFIX)/man/man8
-MKDIR=/bin/mkdir
+BINDIR=$(DESTDIR)$(PREFIX)/sbin
+EXAMPLESDIR=$(DESTDIR)${PREFIX}/share/examples/vm-bhyve
+LIBDIR=$(DESTDIR)$(PREFIX)/lib/vm-bhyve
+MANDIR=$(DESTDIR)$(PREFIX)/man/man8
+RCDIR=$(DESTDIR)$(PREFIX)/etc/rc.d
+
 CP=/bin/cp
+INSTALL=/usr/bin/install
+LN=/bin/ln
+MKDIR=/bin/mkdir
 
 PROG=vm
 MAN=$(PROG).8
 
 install:
 	$(MKDIR) -p $(BINDIR)
-	$(MKDIR) -p $(FILESDIR)
+	$(INSTALL) -m 544 $(PROG) $(BINDIR)/
+
+	$(MKDIR) -p $(LIBDIR)
+	$(INSTALL) lib/* $(LIBDIR)/
+
 	$(MKDIR) -p $(EXAMPLESDIR)
-	$(MKDIR) -p $(RCDIR)
-	$(MKDIR) -p $(MANDIR)
-	$(INSTALL) -m $(BINMODE) $(PROG) $(BINDIR)/
-	$(INSTALL) lib/* $(FILESDIR)/
 	$(INSTALL) sample-templates/* $(EXAMPLESDIR)/
+
+	$(MKDIR) -p $(RCDIR)
 	$(INSTALL) -m 555 rc.d/* $(RCDIR)/
-	rm -f $(MAN).gz
-	gzip -k $(MAN)
+
+	$(MKDIR) -p $(MANDIR)
+	gzip -fk $(MAN)
 	$(INSTALL) $(MAN).gz $(MANDIR)/
+	rm -f -- $(MAN).gz
+	$(LN) -sf $(MANDIR)/$(MAN).gz $(MANDIR)/vm-bhyve.8.gz
 
 vmdir:
 	@if [ -z "${PATH}" ]; then \

README.md

@@ -4,60 +4,31 @@ Management system for FreeBSD bhyve virtual machines
 
 Some of the main features include:
 
-* Now with beta Windows/UEFI support as of v0.7.2!
+* Windows/UEFI support
 * Simple commands to create/start/stop bhyve instances
 * Simple configuration file format
-* Virtual switches supporting vlans & nat (no manual tap or bridge devices needed)
+* Virtual switches supporting vlans & automatic device creation
 * ZFS support
-* FreeBSD/NetBSD/OpenBSD/Linux guest support
+* FreeBSD/MidnightBSD/NetBSD/OpenBSD/Linux guest support
 * Automatic assignment of console devices to access guest console
 * Integration with rc.d startup/shutdown
 * Guest reboot handling
 * Designed with multiple compute nodes + shared storage in mind (NFS/iSCSI/etc)
+* Multiple datastores
+* VNC graphics & tmux support (1.1+ only. See wiki for instructions)
+* Dependency free**
 
-See the GitHub wiki for more information and examples.
+** Some additional packages may be required in certain circumstances -
 
-## IMPORTANT - Note for Linux/NetBSD & OpenBSD users moving from 0.9 to 0.10+
+* `sysutils/grub2-bhyve` is required to run Linux or any other guests that need a Grub bootloader.
+* `sysutils/bhyve-firmware` is required to run UEFI guests
+* `sysutils/tmux` is needed to use tmux console access instead of cu/nmdm
 
-The method of supporting these guests has been heavily changed in 0.10 to
-allow more flexibility. These guests will no longer boot without making changes
-to the configuration file. (Note the `vm configure guest` command can be used to open
-the guest configuration in your default editor)
 
-First of all, if you are using Linux, the guest configuration option needs to be changed to `linux`.
-For NetBSD & OpenBSD, the following configuration options should be set.
+##### See the GitHub wiki for more information and examples.
 
-    guest="generic"
-    loader="grub"
-
-Additionally, any grub commands needed to boot the guest (or the guest installer) need to also
-be added to the configuration file. Please look at the sample templates in 0.10+ for examples
-on how these variables are set. This is what the configuration for OpenBSD 5.9 looks like:
-
-    grub_install0="kopenbsd -h com0 /5.9/amd64/bsd.rd"
-    grub_install1="boot"
-    grub_run_partition="openbsd1"
-    grub_run0="kopenbsd -h com0 -r sd0a /bsd"
-    grub_run1="boot"
-
-The `grub_run_partition` option is not required. By default vm-bhyve will use `hd0,1`, which is correct
-in most cases. It's also possible to specify the correct device and partition directly in the grub commands:
-
-    grub_run0="kopenbsd -h com0 -r sd0a (hd0,openbsd1)/bsd"
-    grub_run1="boot"
-
-(However some guests such as Ubuntu will boot automatically, without any boot commands specified,
-if the correct partition is provided)
-
-The `boot` command does not need to be specified if you are running vm-bhyve-0.11 or newer.
-
-This of course means that it is now trivial to adjust these commands if needed, whereas in
-previous versions of vm-bhyve, they were hard-coded.
-
-`grub-bhyve` is always run on the guest console now, so is accessible via the `vm console guest`
-command. If boot commands are provided, we create a grub.cfg file in the guest directory and
-point `grub-bhyve` at it. (Please note this file is re-written on each boot and so if changes to
-the commands are required, it should be done in the main guest configuration file)
+For most users, I recommend using the version in ports (1.1+).
+Main development happens in the master branch on GitHub and it may contain broken or incomplete features.
 
 ## Quick-Start
 
@@ -66,15 +37,15 @@ See the sections below for more in-depth details.
 
     1. pkg install vm-bhyve
     2. zfs create pool/vm
-    3. echo 'vm_enable="YES"' >> /etc/rc.conf
-    4. echo 'vm_dir="zfs:pool/vm"' >> /etc/rc.conf
+    3. sysrc vm_enable="YES"
+    4. sysrc vm_dir="zfs:pool/vm"
     5. vm init
     6. cp /usr/local/share/examples/vm-bhyve/* /mountpoint/for/pool/vm/.templates/
     7. vm switch create public
     8. vm switch add public em0
-    9. vm iso ftp://ftp.freebsd.org/pub/FreeBSD/releases/ISO-IMAGES/10.3/FreeBSD-10.3-RELEASE-amd64-bootonly.iso
+    9. vm iso https://download.freebsd.org/ftp/releases/ISO-IMAGES/14.2/FreeBSD-14.2-RELEASE-amd64-bootonly.iso
     10. vm create myguest
-    11. vm [-f] install myguest FreeBSD-10.3-RELEASE-amd64-bootonly.iso
+    11. vm install [-f] myguest FreeBSD-14.2-RELEASE-amd64-bootonly.iso
     12. vm console myguest
 
 - [ ] Line 1
@@ -110,7 +81,7 @@ in mind that you won't get back to your terminal until the guest is fully shutdo
 
 ## Install
 
-Download the latest release from Github, or install `sysutils/vm-bhyve`
+Download the latest release from GitHub, or install `sysutils/vm-bhyve`
 
 To install, just run the following command inside the vm-bhyve source directory
 
@@ -120,11 +91,6 @@ If you want to run guests other than FreeBSD, you will need the grub2-bhyve pack
 
     # pkg install grub2-bhyve
 
-Additionally, while not specifically required, dnsmasq can be used to provid DHCP services
-when vm-bhyve is configured to run NAT.
-
-    # pkg install dnsmasq
-
 ## Initial configuration
 
 First of all, you will need a directory to store all your virtual machines and vm-bhyve configuration.
@@ -199,16 +165,6 @@ Obviously you will need to replace em0 here with the correct interface name on y
 
     # vm switch add public em0
 
-If you want to use NAT, do not add a physical interface to the switch, as the switch will be on the private
-side of the NAT network. Just enable NAT on the switch:
-
-    # vm switch nat public on
-
-This will automatically create a private network on the switch, and forward guest traffic
-via your default gateway. Please note that pf must be enabled in /etc/rc.conf for NAT functionality to work.
-Whilst not strictly required, dnsmasq can be used to provide DHCP services to guests on the NAT network.
-vm-bhyve will generate a sample dnsmasq.conf file which can be installed for this purpose.
-
 If you want guest traffic to be on a specific VLAN when leaving the host, specify a vlan number. To turn
 off vlans, just set the vlan number to 0:
 
@@ -231,17 +187,17 @@ example specifies the templatename.conf template, and tells vm-bhyve to create a
 
 You will need an ISO to install the guest with, so download one using the iso command:
 
-    # vm iso ftp://ftp.freebsd.org/pub/FreeBSD/releases/ISO-IMAGES/10.1/FreeBSD-10.1-RELEASE-amd64-disc1.iso
+    # vm iso https://download.freebsd.org/ftp/releases/ISO-IMAGES/14.2/FreeBSD-14.2-RELEASE-amd64-disc1.iso
 
 To start a guest install, run the following command. vm-bhyve will run the machine in the background,
 so use the console command to connect to it and finish installation.
 
-    # vm install testvm FreeBSD-10.1-RELEASE-amd64-disc1.iso
+    # vm install testvm FreeBSD-14.2-RELEASE-amd64-disc1.iso
     # vm console testvm
 
 You can also specify the foreground option to run the guest directly on your terminal:
 
-    # vm -f install testvm FreeBSD-10.1-RELEASE-amd64-disc1.iso
+    # vm install -f testvm FreeBSD-14.2-RELEASE-amd64-disc1.iso
 
 Once installation has finished, you can reboot the guest from inside the console and it will boot up into
 the new OS (assuming installation was successful). Further reboots will work as expected and
@@ -283,7 +239,7 @@ is the number of seconds to wait between starting each one. 5 seconds is the rec
 although a longer delay is useful if you have disk intensive guests and don't want them all booting
 at the same time.
 
-There's also a command which opens a guest's confiuration file in your default text editor, allowing
+There's also a command which opens a guest's configuration file in your default text editor, allowing
 you to easily make changes to the configuration. Please note that changes only take effect after
 a full shutdown and restart of the guest
 
@@ -292,7 +248,52 @@ a full shutdown and restart of the guest
 See the man page for a full description of all available commands.
 
     # man vm
-    
+
+## Using cloud images
+
+You can use cloud images to create virtual machines. The `vm img` command will download the image to datastore and 
+uncompress it if needed (.xz, .tar.gz, and .gz files are supported). The image should be in RAW or QCOW2 format.
+To use this feature you'll need install qemu-tools package:
+
+    # pkg install qemu-tools
+
+To launch FreeBSD using official cloud image:
+
+    # vm img https://download.freebsd.org/ftp/releases/VM-IMAGES/14.2-RELEASE/amd64/Latest/FreeBSD-14.2-RELEASE-amd64.raw.xz
+    # vm create -t freebsd-zvol -i FreeBSD-14.2-RELEASE-amd64.raw freebsd-cloud
+    # vm start freebsd-cloud
+
+To list downloaded images:
+
+    # vm img
+    DATASTORE           FILENAME
+    default             CentOS-7-x86_64-GenericCloud-20180930_02.raw
+    default             debian-9-openstack-amd64.qcow2
+    default             Fedora-AtomicHost-28-1.1.x86_64.raw
+    default             FreeBSD-14.2-RELEASE-amd64.raw
+    default             xenial-server-cloudimg-amd64-uefi1.img
+
+## Using cloud-init
+
+vm-bhyve has basic support for providing cloud-init configuration to the guest. You can enable it with `-C` option
+to `vm create` command. You can also pass public SSH key to be injected into the guest with option `-k <file>`.
+The public key file can contain multiple public SSH keys, one per line, in the `authorized_keys` format.
+
+Example:
+
+    # vm create -t linux -i xenial-server-cloudimg-amd64-uefi1.img -C -k ~/.ssh/id_rsa.pub cloud-init-ubuntu
+    # vm start cloud-init-ubuntu
+    Starting cloud-init-ubuntu
+    * found guest in /zroot/vm/cloud-init-ubuntu
+    * booting...
+    # ssh ubuntu@192.168.0.91
+    The authenticity of host '192.168.0.91 (192.168.0.91)' can't be established.
+    ECDSA key fingerprint is SHA256:6s9uReyhsIXRv0dVRcBCKMHtY0kDYRV7zbM7ot6u604.
+    No matching host key fingerprint found in DNS.
+    Are you sure you want to continue connecting (yes/no)? yes
+    Warning: Permanently added '192.168.0.91' (ECDSA) to the list of known hosts.
+    Welcome to Ubuntu 16.04.5 LTS (GNU/Linux 4.4.0-141-generic x86_64)
+
 ## Adding custom disks
 
 Scenario: If you have a vm on one zpool and would like to add a new virtual disk to it that resides on a different zpool.
@@ -314,37 +315,13 @@ Restart your vm.
 
 ## Windows Support
 
-Windows has been very quickly tested as of version 0.7.2 (Using Server 2012R2).
-I see no reason why other versions supported by bhyve shouldn't work as the basic bhyve
-commands are all the same. Please note that you need FreeBSD 10.3 or 11-CURRENT for the UEFI support
-to be functional.
-
-As there is no VGA console, you must follow the instructions at 
-https://people.freebsd.org/~grehan/bhyve_uefi/windows_iso_repack.txt 
-to create an unattended installation ISO. This requires a few packages to be installed but
-is fairly straight forward if you follow the instructions carefully.
-
-You also need the UEFI firmware, which can be retrieved from
-http://people.freebsd.org/~grehan/bhyve_uefi/BHYVE_UEFI_20151002.fd
-and needs to be placed in `$vm_dir/.config/BHYVE_UEFI.fd`.
-
-Once you have an ISO capable of installing without user interaction, vm-bhyve works as normal.
-Just copy the ISO to `$vm_dir/.iso/`, then run the following to install:
-
-    # vm create -t windows -s 50G winguest
-    # vm install winguest mywiniso.iso
-
-Installation can take around 25 minutes. If you look in the vm-bhyve.log file in the virtual
-machines directory, you should see it reboot twice. After the second reboot (third run in total)
-the machine should boot into Windows. Access the Windows console using the `vm console winguest` command,
-then press `i` to get its IP address (It will use DHCP). You can then RDP to the guest.
-The default login details are Administrator and Test123.
+Please see the Windows section in the [Wiki](https://github.com/churchers/vm-bhyve/wiki/Running-Windows)
 
 ## Autocomplete
 
 If you are using the default csh/tcsh shell built into FreeBSD, running the following command should allow
 autocomplete to work for all the currently supported functions. This is especially useful for viewing
-and completing guest & ISO file names. Please note that there's three ocurrances of '/path/to/vm' which
+and completing guest & ISO file names. Please note that there's three occurrences of '/path/to/vm' which
 need to be changed to the directory containing your virtual machines.
 
 To make the autocomplete features available permanently, add the following to your `$HOME/.cshrc` file. Then either

lib/vm-base

@@ -1,6 +1,6 @@
 #!/bin/sh
 #-------------------------------------------------------------------------+
-# Copyright (C) 2015 Matt Churchyard (churchers@gmail.com)
+# Copyright (C) 2018 Matt Churchyard (churchers@gmail.com)
 # All rights reserved
 #
 # Redistribution and use in source and binary forms, with or without
@@ -24,46 +24,38 @@
 # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-# add a value to an rc file, appending to an existing string value
-#
-# @param string _file the file to update (relative to $vm_dir/)
-# @param string _var the variable to update
-# @param string _value value to add to the string
-#
-__rc_append_string(){
-    local _file="${vm_dir}/$1"
-    local _var="$2"
-    local _value="$3"
-    local _curr
+VERSION=1.6-devel
+VERSION_INT=106001
+VERSION_BSD=$(uname -K)
+PATH=${PATH}:/bin:/usr/bin:/usr/local/bin:/sbin:/usr/sbin:/usr/local/sbin
 
-    _curr=$(sysrc -inqf "${_file}" "${_var}")
-    _curr="${_curr}${_curr:+ }${_value}"
+. /etc/rc.subr
+load_rc_config "vm"
 
-    sysrc -inqf "${_file}" "${_var}=${_curr}" >/dev/null 2>&1
-    [ $? -ne 0 ] && __warn "unable to update configuration file for ${_var}"
-}
+# check informational commands
+cmd::parse_info "$@"
 
-# remove a value from a string list
-# eg. removing "two" from var="one two three" would result in var="one three"
-#
-# @param string _file the file to update (relative to $vm_dir/)
-# @param string _var the variable to update
-# @param string _value the value to remove
-#
-__rc_splice_string(){
-    local _file="${vm_dir}/$1"
-    local _var="$2"
-    local _value="$3"
-    local _curr _key _new
+# we should be enabled in rc.conf
+# or call it using forcestart
+[ -z "$rc_force" ] && ! checkyesno vm_enable && util::err "\$vm_enable is not enabled in /etc/rc.conf!"
 
-    _curr=$(sysrc -inqf "${_file}" "${_var}")
+# check we can run bhyve
+util::check_bhyve_support
 
-    for _key in ${_curr}; do
-        if [ "${_key}" != "${_value}" ]; then
-            _new="${_new}${_new:+ }${_key}"
-        fi
-    done
+# init for zfs
+zfs::init
 
-    sysrc -inqf "${_file}" "${_var}=${_new}" >/dev/null 2>&1
-    [ $? -ne 0 ] && __warn "unable to update configuration file for ${_var}"
-}
+# create directories as needed
+[ ! -d "${vm_dir}" ] && util::err "\$vm_dir has not been configured or is not a valid directory"
+[ ! -d "${vm_dir}/.config" ] && mkdir "${vm_dir}/.config"
+[ ! -e "${vm_dir}/.config/null.iso" ] && touch "${vm_dir}/.config/null.iso"
+[ ! -d "${vm_dir}/.templates" ] && mkdir "${vm_dir}/.templates"
+[ ! -d "${vm_dir}/.iso" ] && mkdir "${vm_dir}/.iso"
+[ ! -d "${vm_dir}/.img" ] && mkdir "${vm_dir}/.img"
+
+# load core configuration
+config::core::load
+datastore::load
+
+# run the requested command
+cmd::parse "$@"

lib/vm-cmd

@@ -1,6 +1,6 @@
 #!/bin/sh
 #-------------------------------------------------------------------------+
-# Copyright (C) 2015 Matt Churchyard (churchers@gmail.com)
+# Copyright (C) 2016 Matt Churchyard (churchers@gmail.com)
 # All rights reserved
 #
 # Redistribution and use in source and binary forms, with or without
@@ -24,44 +24,74 @@
 # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
+CMD_VALID_LIST="init,switch,datastore,image,get,set,list,create,destroy,rename,install,start,stop,restart"
+CMD_VALID_LIST="${CMD_VALID_LIST},add,reset,poweroff,startall,stopall,console,iso,img,configure,passthru,_run"
+CMD_VALID_LIST="${CMD_VALID_LIST},info,clone,snapshot,rollback,migrate,version,usage"
+
+# cmd: vm ...
+#
+# handle simple information commands that don't need any 
+# priviledged access or bhyve support
+#
+# @param string _cmd the command right after 'vm '
+#
+cmd::parse_info(){
+    local _cmd
+
+    cmd::find "_cmd" "$1" "${CMD_VALID_LIST}"
+
+    case "${_cmd}" in
+        version) util::version && exit ;;
+        usage)   util::usage ;;
+    esac
+}
+
 # cmd: vm ...
 #
 # process the vm command line to see which function is requested
 #
 # @param string _cmd the command right after 'vm '
 #
-__parse_cmd(){
-    local _cmd="$1"
+cmd::parse(){
+    local _cmd
+
+    # try to find a matching command
+    cmd::find "_cmd" "$1" "${CMD_VALID_LIST}" || util::usage
     shift
 
     case "${_cmd}" in
-        init)      __setup
-                   __switch_init ;;
-        switch)    __parse_switch_cmd "$@" ;;
-        image)     __zfs_parse_image_cmd "$@" ;;
-        list)      __vm_list ;;
-        info)      __vm_info "$@" ;;
-        create)    __vm_create "$@" ;;
-        destroy)   __vm_destroy "$@" ;;
-        rename)    __vm_rename "$@" ;;
-        install)   __vm_install "$@" ;;
-        start)     __vm_start "$1" ;;
-        stop)      __vm_stop "$@" ;;
-        add)       __vm_add "$@" ;;
-        reset)     __vm_reset "$@" ;;
-        poweroff)  __vm_poweroff "$@" ;;
-        startall)  __vm_startall ;;
-        stopall)   __vm_stopall ;;
-        console)   __vm_console "$@" ;;
-        _run)      __vm_run "$@" ;;
-        iso)       __vm_iso "$@" ;;
-        configure) __vm_configure "$@" ;;
-        passthru)  __vm_passthru ;;
-        clone)     __zfs_clone "$@" ;;
-        snapshot)  __zfs_snapshot "$@" ;;
-        rollback)  __zfs_rollback "$@" ;;
-        version)   __version ;;
-        *)         __usage ;;
+        init)      util::setup
+                   switch::init ;;
+        switch)    cmd::parse_switch "$@" ;;
+        datastore) cmd::parse_datastore "$@" ;;
+        image)     cmd::parse_image "$@" ;;
+        get)       core::get "$@" ;;
+        set)       core::set "$@" ;;
+        list)      core::list "$@" ;;
+        create)    core::create "$@" ;;
+        destroy)   core::destroy "$@" ;;
+        rename)    core::rename "$@" ;;
+        install)   core::install "$@" ;;
+        start)     core::start "$@" ;;
+        stop)      core::stop "$@" ;;
+        restart)   core::restart "$@" ;;
+        add)       core::add "$@" ;;
+        reset)     core::reset "$@" ;;
+        poweroff)  core::poweroff "$@" ;;
+        startall)  core::startall ;;
+        stopall)   core::stopall ;;
+        console)   core::console "$@" ;;
+        iso)       core::iso "$@" ;;
+        img)       core::img "$@" ;;
+        configure) core::configure "$@" ;;
+        passthru)  core::passthru ;;
+        _run)      vm::run "$@" ;;
+        info)      info::guest "$@" ;;
+        clone)     zfs::clone "$@" ;;
+        snapshot)  zfs::snapshot "$@" ;;
+        rollback)  zfs::rollback "$@" ;;
+        migrate)   migration::run "$@" ;;
+        *)         util::err "unknown command '${_user_cmd}'. please run 'vm usage' or view the manpage for help" ;;
     esac
 }
 
@@ -72,43 +102,136 @@ __parse_cmd(){
 #
 # @param string _cmd the command right after 'vm switch '
 #
-__parse_switch_cmd(){
-    local _cmd="$1"
+cmd::parse_switch(){
+    local _cmd
+
+    # try to find a matching command
+    cmd::find "_cmd" "$1" "create,list,destroy,add,remove,vlan,nat,address,private,info" || util::usage
     shift
 
     case "${_cmd}" in
-        create)  __switch_create "$@" ;;
-        list)    __switch_list ;;
-        info)    __vm_info_switch "$@" ;;
-        destroy) __switch_remove "$@" ;;
-        import)  __switch_import "$@" ;;
-        add)     __switch_add_member "$@" ;;
-        remove)  __switch_remove_member "$@" ;;
-        vlan)    __switch_vlan "$@" ;;
-        nat)     __switch_nat "$@" ;;
-        *)       __usage ;;
+        create)  switch::create "$@" ;;
+        list)    switch::list ;;
+        destroy) switch::remove "$@" ;;
+        add)     switch::add_member "$@" ;;
+        remove)  switch::remove_member "$@" ;;
+        vlan)    switch::vlan "$@" ;;
+        nat)     switch::nat "$@" ;;
+        address) switch::address "$@" ;;
+        private) switch::private "$@" ;;
+        info)    info::switch "$@" ;;
+        *)       util::err "unknown command '${_user_cmd}'. please run 'vm usage' or view the manpage for help" ;;
     esac
 }
 
-# look for any arguments to the vm command
-# -d /dir - change the global vm_dir to specified value
-# -f      - run vm-bhyve/loader/bhyve in the foreground
+# cmd: vm datastore ...
 #
-# the argument string without any options is put into 
-# VM_COMMAND for the main command parser to handle
+# parse a datastore command
 #
-# @modifies VM_COMMAND VM_FOREGROUND vm_dir
+# @param string _cmd the command after 'vm datastore ...'
 #
-__parse_cmd_args(){
-    local _opt
+cmd::parse_datastore(){
+    local _cmd
 
-    while getopts d:f _opt; do
+    # try to find a matching command
+    cmd::find "_cmd" "$1" "list,add,remove,iso,img" || util::usage
+    shift
+
+    case "${_cmd}" in
+        list)   datastore::list ;;
+        add)    datastore::add "$@" ;;
+        remove) datastore::remove "$@" ;;
+        iso)    datastore::iso "$@" ;;
+        img)    datastore::img "$@" ;;
+        *)      util::err "unknown command '${_user_cmd}'. please run 'vm usage' or view the manpage for help" ;;
+    esac
+}
+
+# cmd 'vm image ...'
+# parse the image command set
+#
+# @param string _cmd the command after 'vm image '
+#
+cmd::parse_image(){
+    local _cmd
+
+    [ -z "${VM_ZFS}" ] && util::err "\$vm_dir must be a ZFS datastore to use these functions"
+
+    # try to find a matching command
+    cmd::find "_cmd" "$1" "list,create,provision,destroy" || util::usage
+    shift
+
+    case "${_cmd}" in
+        list)      zfs::image_list ;;
+        create)    zfs::image_create "$@" ;;
+        provision) zfs::image_provision "$@" ;;
+        destroy)   zfs::image_destroy "$@" ;;
+        *)         util::err "unknown command '${_user_cmd}'. please run 'vm usage' or view the manpage for help" ;;
+    esac
+}
+
+# many commands accept the same arguments (force being the obvious one)
+# provide a function to parse these so we don't have to keep
+# repeating the same getopt code. the return value here is the number
+# of arguments the caller needs to shift.
+#
+# note that start/install/_run use -f for foreground mode
+#
+# @param _arglist[multiple] the callers $@
+# @return number of arguments caller should shift over
+#
+cmd::parse_args(){
+    local _opt _count
+
+    while getopts fitv _opt; do
         case ${_opt} in
-            d) vm_dir="${OPTARG}" ;;
-            f) VM_FOREGROUND="1" ;;
+            f) VM_OPT_FORCE="1"
+               VM_OPT_FOREGROUND="1" ;;
+            i) VM_OPT_INTERACTIVE="1" ;;
+            t) VM_OPT_TMUX="1" ;;
+            v) VM_OPT_VERBOSE="1" ;;
         esac
     done
 
-    shift $((OPTIND - 1))
-    VM_COMMAND="$@"
+    [ -n "${VM_OPT_FOREGROUND}" ] && [ -n "${VM_OPT_INTERACTIVE}" ] && \
+      util::err "foreground and interactive mode are mutually exclusive"
+
+    return $((OPTIND - 1))
+}
+
+# try to match part of a command name against a list of valid commands
+# if we find more than one match we return an error
+# if we only get one match, return the full command name
+#
+# @param string _var variable to put full command name into
+# @param string _user_cmd the value provided by the user
+# @param string _valid comma-separated list of valid choices
+# @return success if we find one match
+#
+cmd::find(){
+    local _var="$1"
+    local _user_cmd="$2"
+    local _valid="$3"
+    local _opt _choice _found=""
+    local IFS=","
+
+    [ -n "${_user_cmd}" ] || util::err "no command specified"
+
+    for _opt in ${_valid}; do
+        # exact match?
+        if [ "${_user_cmd}" = "${_opt}" ]; then
+            setvar "${_var}" "${_opt}"
+            return 0
+        fi
+
+        if echo "${_opt}" | grep -iqs "^${_user_cmd}"; then
+           [ -n "${_found}" ] && util::err "ambiguous command '${_user_cmd}'"
+
+           _found=1
+           _choice="${_opt}"
+        fi
+    done
+
+    [ -z "${_found}" ] && return 1
+    setvar "${_var}" "${_choice}"
 }

lib/vm-common

@@ -1,263 +0,0 @@
-#!/bin/sh
-#-------------------------------------------------------------------------+
-# Copyright (C) 2015 Matt Churchyard (churchers@gmail.com)
-# All rights reserved
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted providing that the following conditions
-# are met:
-# 1. Redistributions of source code must retain the above copyright
-#    notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions and the following disclaimer in the
-#    documentation and/or other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
-# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
-# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
-# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
-# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-# make sure we have the right environment
-#
-__setup(){
-    __load_module "vmm"
-    __load_module "nmdm"
-    __load_module "if_bridge"
-    __load_module "if_tap"
-
-    sysctl net.link.tap.up_on_open=1 >/dev/null 2>&1
-}
-
-# load a kernel module
-#
-# @param string _mod the module name
-#
-__load_module(){
-    local _mod="$1"
-    kldstat -qm ${_mod} >/dev/null 2>&1
-    if [ $? -ne 0 ]; then
-        kldload ${_mod} >/dev/null 2>&1
-        [ $? -ne 0 ] && __err "unable to load ${_mod}.ko!"
-    fi
-}
-
-# restart a local service
-# checks if service is running and either starts or restarts
-#
-# @param string _serv the name of the service
-#
-__restart_service(){
-    local _serv="$1"
-    local _cmd="restart"
-
-    # see if it's actually running
-    service ${_serv} status >/dev/null 2>&1
-    [ $? -ne 0 ] && _cmd="start"
-
-    service ${_serv} ${_cmd} >/dev/null 2>&1
-    [ $? -ne 0 ] && __warn "failed to ${_cmd} service ${_serv}"
-}
-
-# show version
-#
-__version(){
-    echo "vm-bhyve: Bhyve virtual machine management v${VERSION} (build ${VERSION_INT})"
-}
-
-# show version & usage information
-# we exit after running this
-#
-__usage(){
-    __version
-    cat << EOT
-Usage: vm ...
-    version
-    init
-    switch list
-    switch info [name]
-    switch create <name>
-    switch import <name> <bridge>
-    switch vlan <name> <vlan|0>
-    switch nat <name> <on|off>
-    switch add <name> <interface>
-    switch remove <name> <interface>
-    switch destroy <name>
-    list
-    info [name]
-    create [-t template] [-s size] <name>
-    install <name> <iso>
-    start <name>
-    stop <name> <...>
-    console <name> [com1|com2]
-    rename <name> <new-name>
-    add [-d device] [-t type] [-s size|switch] <name>
-    startall
-    stopall
-    reset <name>
-    poweroff <name>
-    configure <name>
-    destroy <name>
-    passthru
-    clone <name[@snapshot]> <new-name>
-    snapshot [-f] <name[@snapshot]>
-    rollback [-r] <name@snapshot>
-    iso [url]
-    image list
-    image create [-d description] <name>
-    image destroy <uuid>
-    image provision <uuid> <newname>
-EOT
-    exit 1
-}
-
-# error
-# display an error message and exit immediately
-#
-# @param string - the message to display
-#
-__err(){
-    echo "${0}: ERROR: $1"
-    exit 1
-}
-
-# warn
-# display warning, but do not exit
-#
-# @param string - the message to display
-#
-__warn(){
-    echo "${0}: WARNING: $1"
-}
-
-# log_rotate
-# simple rotation of log files
-# if we hit 1MB, which should cover a fair amount of history,
-# we move existing log and and create a new one.
-# one keep 1 previous file, as that should be enough
-#
-# @param string _type whether to rotate guest or main log
-#
-__log_rotate(){
-    local _type="$1"
-    local _lf="vm-bhyve.log"
-    local _file _size _guest
-
-    case "${_type}" in
-        guest)
-            _guest="$2"
-            _file="${vm_dir}/${_guest}/${_lf}"
-            ;;
-        system)
-            _file="${vm_dir}/${_lf}"
-            ;;
-    esac
-
-    if [ -e "${_file}" ]; then
-        _size=$(stat "${_file}" | cut -d' ' -f8)
-
-        if [ -n "${_size}" -a "${_size}" -ge 1048576 ]; then
-            unlink "${_file}.0.gz" >/dev/null 2>&1
-            mv "${_file}" "${_file}.0"
-            gzip "${_file}.0"
-        fi
-    fi
-}
-
-# log to file
-# writes the date and a message to the specified log
-# the global log is in $vm_dir/vm-bhyve.log
-# guest logs are $vm_dir/{guest}/vm-bhyve.log
-#
-# @param string _type=guest|system log to global vm-bhyve log or guest
-# @param optional string _guest if _type=guest, the guest name, otherwise do not provide at all
-# @param string _message the message to log
-#
-__log(){
-    local _type="$1"
-    local _lf="vm-bhyve.log"
-    local _guest _message _file _date
-
-    case "${_type}" in
-        guest)
-            _guest="$2"
-            _message="$3"
-            _file="${vm_dir}/${_guest}/${_lf}"
-            ;;
-        system)
-            _message="$2"
-            _file="${vm_dir}/${_lf}"
-            ;;
-    esac
-
-    echo "$(date +'%b %d %T'): ${_message}" >> "${_file}"
-}
-
-# write content to a file, and log what we
-# did to the guest log file
-# it's useful to be able to see what files vm-bhyve is creating
-# and the contents so we write that to the log.
-# The file is created in $vm_dir/{guest}
-#
-# @param string _type=write|appnd create file or append to it
-# @param string _guest the guest name
-# @param string _file the file name to write to
-# @param string _message the data to write
-#
-__log_and_write(){
-    local _type="$1"
-    local _guest="$2"
-    local _file="${vm_dir}/${_guest}/$3"
-    local _message="$4"
-
-    if [ "${_type}" = "write" ]; then
-        __log "guest" "${_guest}" "create file ${_file}"
-        echo "${_message}" > "${_file}"
-    else
-        echo "${_message}" >> "${_file}"
-    fi
-
-    __log "guest" "${_guest}" " -> ${_message}"
-}
-
-# confirm yes or no
-#
-# @param string _msh message to display
-# @return int success if confirmed
-#
-__confirm(){
-    local _msg="$1"
-    local _resp
-
-    while read -p "${_msg} (y/n)? " _resp; do
-        case "${_resp}" in
-            y*) return 0 ;;
-            n*) return 1 ;;
-        esac
-    done
-}
-
-# our own checkyesno copy
-# doesn't warn for unsupported values
-# also returns as 'yes' unless value is specifically no/off/false/0 
-#
-# @param _value the value to test
-# @return int 1 if set to "off/false/no/0", 0 otherwise
-#
-__checkyesno(){
-    local _value="$1"
-
-    [ -z "${_value}" ] && return 1
-
-    case "$_value" in
-        [Nn][Oo]|[Ff][Aa][Ll][Ss][Ee]|[Oo][Ff][Ff]|0)
-            return 1 ;;
-        *)  return 0 ;;		
-    esac
-}

lib/vm-config

@@ -1,6 +1,6 @@
 #!/bin/sh
 #-------------------------------------------------------------------------+
-# Copyright (C) 2015 Matt Churchyard (churchers@gmail.com)
+# Copyright (C) 2016 Matt Churchyard (churchers@gmail.com)
 # All rights reserved
 #
 # Redistribution and use in source and binary forms, with or without
@@ -34,13 +34,13 @@
 # @param string _file full path of the file to read
 # @modifies VM_CONFIG
 #
-__config_load(){
+config::load(){
     local _file="$1"
 
     # read config file
     # we kick out any lines that don't start with a letter,
     # scrap anything after a # character, and remove double-quotes
-    VM_CONFIG=$(grep '^[a-z]' "${_file}" 2>/dev/null | awk -F# '{print $1}' | tr -d '"')
+    VM_CONFIG=$(grep '^[a-z]' "${_file}" 2>/dev/null | awk -F# '{print $1}' | sed -e 's@ *$@@' | tr -d '"')
 }
 
 # get a configuration value from the current config file
@@ -50,7 +50,7 @@ __config_load(){
 # @param optional string _def default value to return if setting not found
 # @return true if setting found
 #
-__config_get(){
+config::get(){
     local _c_var="$1"
     local _c_key="$2"
     local _c_def="$3"
@@ -68,3 +68,132 @@ __config_get(){
     setvar "${_c_var}" "${_c_def}"
     return 1
 }
+
+# simple wrapper to check a config setting to see if it's 
+# set to yes/no true/false etc
+#
+# @param string _key the config key to check
+# @param string _def default value if config key doesn't exist
+# @return true(0) if set to anything other than no/false/off/0
+#
+config::yesno(){
+    local _key="$1"
+    local _def="$2"
+    local _value
+
+    config::get "_value" "${_key}" "${_def}"
+    util::yesno "${_value}"
+}
+
+# set a value in guest configuration file
+# we check for newline at the end as sysrc won't add it
+# and that will mess up the new key and the existing one
+# from the end of the file
+#
+# @param string _name guest name
+# @param string _key config key to set
+# @param string _value value
+# @param int _skip_newline_check skip the check for newline
+# @return true if sysrc successful
+#
+config::set(){
+    local _name="$1"
+    local _key="$2"
+    local _value="$3"
+    local _skip_newline_check="$4"
+    local _newline
+
+    if [ -z "${_skip_newline_check}" ]; then
+        _newline=$(tail -1 "${VM_DS_PATH}/${_name}/${_name}.conf" | wc -l | tr -d " ")
+        [ "${_newline}" -eq "0" ] && echo "" >> "${VM_DS_PATH}/${_name}/${_name}.conf"
+    fi
+
+    sysrc -inqf "${VM_DS_PATH}/${_name}/${_name}.conf" "${_key}=${_value}" >/dev/null 2>&1
+}
+
+# remove a value from guest config
+config::remove(){
+    local _name="$1"
+    local _key="$2"
+
+    sysrc -inxqf "${VM_DS_PATH}/${_name}/${_name}.conf" "${_key}" >/dev/null 2>&1
+}
+
+# load core configuration file
+#
+# @modifies VM_CORE_CONFIG VM_CONFIG_USER
+#
+config::core::load(){
+    VM_CONFIG_USER="console;compress;decompress;"
+
+    # check config file exists
+    # this is mainly for upgrades to make sure switch/datastore config are migrated
+    # DEPRECATED 1.3, remove after
+    if [ ! -e "${vm_dir}/.config/system.conf" ]; then
+        cat "${vm_dir}/.config/switch" > "${vm_dir}/.config/system.conf" 2>/dev/null
+        cat "${vm_dir}/.config/datastore" >> "${vm_dir}/.config/system.conf" 2>/dev/null
+    fi
+
+    VM_CORE_CONFIG=$(grep '^[a-z]' "${vm_dir}/.config/system.conf" 2>/dev/null | awk -F# '{print $1}' | sed -e 's@ *$@@' | tr -d '"')
+}
+
+# get a value from core config
+#
+# @param string _c_var variable name to put value into
+# @param string _c_key config key to look for
+# @param string _c_def default value if not value
+# @return 0 if found
+#
+config::core::get(){
+    local _c_var="$1"
+    local _c_key="$2"
+    local _c_def="$3"
+    local _c_line
+    local IFS=$'\n'
+
+    for _c_line in ${VM_CORE_CONFIG}; do
+        if [ "${_c_key}" = "${_c_line%%=*}" ]; then
+            setvar "${_c_var}" "${_c_line#*=}"
+            return 0
+        fi
+    done
+
+    # not found
+    setvar "${_c_var}" "${_c_def}"
+    return 1
+}
+
+# add a value to core configuration
+#
+# @param string _var variable to set
+# @param string _value new value
+# @param string _append non-empty to append to existing value
+#
+config::core::set(){
+    local _var="$1"
+    local _value="$2"
+    local _append="$3"
+
+    if [ -n "${_append}" ]; then
+        sysrc -inqf "${vm_dir}/.config/system.conf" "${_var}"+="${_value}" >/dev/null 2>&1
+    else
+        sysrc -inqf "${vm_dir}/.config/system.conf" "${_var}"="${_value}" >/dev/null 2>&1
+    fi
+}
+
+# remove a value from core configuration
+#
+# @param string _var variable to remove
+# @param string _value if non-empty we will try to remove just this value from setting
+# @return non-zero on error
+#
+config::core::remove(){
+    local _var="$1"
+    local _value="$2"
+
+    if [ -n "${_value}" ]; then
+        sysrc -inqf "${vm_dir}/.config/system.conf" "${_var}"-="${_value}" >/dev/null 2>&1
+    else
+        sysrc -inxqf "${vm_dir}/.config/system.conf" ${_var} >/dev/null 2>&1
+    fi
+}

lib/vm-datastore

@@ -0,0 +1,500 @@
+#!/bin/sh
+#-------------------------------------------------------------------------+
+# Copyright (C) 2016 Matt Churchyard (churchers@gmail.com)
+# All rights reserved
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted providing that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# 'vm datastore list'
+# show configured datastores
+#
+datastore::list(){
+    local _format="%-15s %-11s %-25s %s"
+    local _name _type _dataset _path _spec
+
+    # headings
+    printf "${_format}\n" "NAME" "TYPE" "PATH" "ZFS DATASET"
+
+    # add the default datastore
+    _name="default"
+    _path="${vm_dir}"
+
+    # get the type and path
+    if [ "${VM_ZFS}" ]; then
+        _type="zfs"
+        _dataset="${VM_ZFS_DATASET}"
+    else
+        _type="directory"
+        _dataset="-"
+    fi
+
+    printf "${_format}\n" "${_name}" "${_type}" "${_path}" "${_dataset}"
+
+    for _name in ${VM_DATASTORE_LIST}; do
+        [ "${_name}" = "default" ] && continue
+
+        config::core::get "_spec" "path_${_name}"
+        [ -z "${_spec}" ] && continue
+
+        if [ "${_spec%%:*}" = "zfs" ]; then
+            _type="zfs"
+            _dataset="${_spec#*:}"
+            datastore::__resolve_path "_path" "${_spec}"
+        elif [ "${_spec%%:*}" = "iso" ]; then
+            _type="iso"
+            _path="${_spec#*:}"
+            _dataset="-"
+        elif [ "${_spec%%:*}" = "img" ]; then
+            _type="img"
+            _path="${_spec#*:}"
+            _dataset="-"
+        else
+            _type="directory"
+            _path="${_spec}"
+            _dataset="-"
+        fi
+
+        printf "${_format}\n" "${_name}" "${_type}" "${_path}" "${_dataset}"
+    done
+}
+
+# 'vm datastore add'
+# create a new datastore
+#
+# we don't try and create directories or datasets.
+# the user should do that first
+#
+# @param string _name datastore name
+# @param string _spec specification (either /path or zfs:dataset)
+#
+datastore::add(){
+    local _name="$1"
+    local _spec="$2"
+    local _mount _num=0 _curr
+
+    [ -z "${_name}" -o -z "${_spec}" ] && util::usage
+    util::check_name "${_name}" || util::err "invalid datastore name - '${_name}'"
+
+    # check name not in use
+    for _curr in ${VM_DATASTORE_LIST}; do
+        [ "${_curr}" = "${_name}" ] && util::err "datstore '${_name}' already exists!"
+    done
+
+    # look for zfs
+    if [ "${_spec%%:*}" = "zfs" ]; then
+        # try to find mountpoint
+        _mount=$(mount | grep "^${_spec#*:} " |cut -d' ' -f3)
+        [ -z "${_mount}" ] && util::err "${_spec} doesn't seem to be a valid, mounted dataset"
+    else
+        # make sure it's a directory
+        [ ! -d "${_spec}" ] && util::err "${_spec} doesn't seem to be a valid directory"
+
+        _mount="${_spec}"
+    fi
+
+    # see if this is already our default datastore
+    [ "${_mount}" = "${vm_dir}" ] && util::err "specified path already exists as default datastore"
+
+    # save
+    config::core::set "datastore_list" "${_name}" "1"
+    config::core::set "path_${_name}" "${_spec}"
+    [ $? -eq 0 ] || util::err "error saving settings to configuration file"
+}
+
+# remove a datastore
+# we don't actually delete anything, just remove from config
+#
+# @param string _name name of dataset
+#
+datastore::remove(){
+    local _name="$1"
+    local _ds _found
+
+    [ "${_name}" = "default" ] && util::err "cannot remove default datastore"
+
+    for _ds in ${VM_DATASTORE_LIST}; do
+        [ "${_ds}" = "${_name}" ] && _found="1"
+    done
+
+    # found the dataset?
+    [ -z "${_found}" ] && util::err "unable to locate the specified dataset"
+
+    config::core::remove "datastore_list" "${_name}"
+    config::core::remove "path_${_name}"
+    [ $? -eq 0 ] || util::err "error removing settings from configuration file"
+}
+
+# get the filesystem path for the specified dataset spec
+# this can either be just a path, or ZFS dataset
+#
+# @param string _var variable to put path into
+# @param string _spec the path spec (either directory or zfs:dataset)
+# @return non-zero on error
+#
+datastore::__resolve_path(){
+    local _var="$1"
+    local _spec="$2"
+
+    if [ "${_spec%%:*}" = "zfs" ]; then
+        _mount=$(mount | grep "^${_spec#*:} " |cut -d' ' -f3)
+
+        if [ -n "${_mount}" ]; then
+            setvar "${_var}" "${_mount}"
+            return 0
+        fi
+    elif [ "${_spec%%:*}" = "iso" ] || [ "${_spec%%:*}" = "img" ]; then
+        setvar "${_var}" "${_spec#*:}"
+        return 0
+    else
+        if [ -d "${_spec}" ]; then
+            setvar "${_var}" "${_spec}"
+            return 0
+        fi
+    fi
+
+    setvar "${_var}" ""
+    return 1
+}
+
+# load list of datastores into VM_DATASTORE_LIST
+#
+# @modifies VM_DATASTORE_LIST
+#
+datastore::load(){
+    config::core::get "VM_DATASTORE_LIST" "datastore_list"
+    VM_DATASTORE_LIST="default${VM_DATASTORE_LIST:+ }${VM_DATASTORE_LIST}"
+}
+
+# init global settings for a vm
+# we take a guest name and try to find it in all
+# datastores. we don't allow duplicate names, and
+# if there is, we will just return the first found.
+#
+# @param string _guest guest name
+# @return non-zero on error
+# @modifies VM_DS_NAME VM_DS_PATH VM_DS_ZFS VM_DS_ZFS_DATASET
+#
+datastore::get_guest(){
+    local _guest="$1"
+    local _ds _spec _path _found _zfs _dataset
+
+    # look in default store
+    if [ -f "${vm_dir}/${_guest}/${_guest}.conf" ]; then
+        _found="1"
+        _ds="default"
+        _path="${vm_dir}"
+        _zfs="${VM_ZFS}"
+        _dataset="${VM_ZFS_DATASET}"
+    fi
+
+    # look on other datastores
+    if [ -z "${_found}" ]; then
+        for _ds in ${VM_DATASTORE_LIST}; do
+            [ "${_ds}" = "default" ] && continue
+
+            config::core::get "_spec" "path_${_ds}"
+            if [ "${_spec%%:*}" = "iso" ] || [ "${_spec%%:*}" = "img" ]; then
+                continue
+            fi
+
+            datastore::__resolve_path "_path" "${_spec}"
+
+            if [ -f "${_path}/${_guest}/${_guest}.conf" ]; then
+                [ "${_spec%%:*}" = "zfs" ] && _zfs="1" && _dataset="${_spec#*:}"
+
+                _found="1"
+                break
+            fi
+        done
+    fi
+    
+    # make sure we have a path
+    [ -z "${_found}" ] && return 1
+
+    # set variables
+    VM_DS_NAME="${_ds}"
+    VM_DS_PATH="${_path}"
+    VM_DS_ZFS="${_zfs}"
+    VM_DS_ZFS_DATASET="${_dataset}"
+}
+
+# get the path and details for a datastore
+# put into same variables as datastore_get_guest
+#
+# @param string _ds datastore name
+# @return non-zero on error
+# @modifies VM_DS_PATH VM_DS_ZFS VM_DS_ZFS_DATASET
+#
+datastore::get(){
+    local _ds="$1"
+    local _spec _path _zfs _dataset
+
+    # check for default
+    if [ "${_ds}" = "default" ]; then
+        VM_DS_NAME="default"
+        VM_DS_PATH="${vm_dir}"
+        VM_DS_ZFS="${VM_ZFS}"
+        VM_DS_ZFS_DATASET="${VM_ZFS_DATASET}"
+        return 0
+    fi
+
+    config::core::get "_spec" "path_${_ds}"
+    [ -z "${_spec}" ] && return 1
+
+    # skip iso and img stores
+    if [ "${_spec%%:*}" = "iso" ] || [ "${_spec%%:*}" = "img" ]; then
+        return 1
+    fi 
+
+    datastore::__resolve_path "_path" "${_spec}" || return 1
+    [ "${_spec%%:*}" = "zfs" ] && _zfs="1" && _dataset="${_spec#*:}"
+
+    # set variables
+    VM_DS_NAME="${_ds}"
+    VM_DS_PATH="${_path}"
+    VM_DS_ZFS="${_zfs}"
+    VM_DS_ZFS_DATASET="${_dataset}"
+}
+
+# get the path for an iso datastore
+#
+# @param string _ds datastore name
+#
+datastore::get_iso(){
+    local _ds="$1"
+
+    # default?
+    # we use the .iso subdir in that case
+    if [ "${_ds}" = "default" ]; then
+        VM_DS_PATH="${vm_dir}/.iso"
+        return 0
+    fi
+
+    config::core::get "_spec" "path_${_ds}"
+    [ -z "${_spec}" ] && return 1
+
+    # should be an iso ds
+    [ "${_spec%%:*}" = "iso" ] || return 1
+
+    datastore::__resolve_path "_path" "${_spec}" || return 1
+    VM_DS_PATH="${_path}"
+}
+
+# get the path for an img datastore
+#
+# @param string _ds datastore name
+#
+datastore::get_img(){
+    local _ds="$1"
+
+    # default?
+    # we use the .img subdir in that case
+    if [ "${_ds}" = "default" ]; then
+        VM_DS_PATH="${vm_dir}/.img"
+        return 0
+    fi
+
+    config::core::get "_spec" "path_${_ds}"
+    [ -z "${_spec}" ] && return 1
+
+    # should be an img ds
+    [ "${_spec%%:*}" = "img" ] || return 1
+
+    datastore::__resolve_path "_path" "${_spec}" || return 1
+    VM_DS_PATH="${_path}"
+}
+
+# add a datastore for iso files
+#
+# @param string _name the name of the datastore
+# @param string _path filesystem path
+#
+datastore::iso(){
+    local _name="$1"
+    local _path="$2"
+
+    [ -z "${_name}" -o -z "${_path}" ] && util::usage
+    util::check_name "${_name}" || util::err "invalid datastore name - '${_name}'"
+
+    # check name not in use
+    for _curr in ${VM_DATASTORE_LIST}; do
+        [ "${_curr}" = "${_name}" ] && util::err "datstore '${_name}' already exists!"
+    done
+
+    # make sure directory exists
+    [ ! -d "${_path}" ] && util::err "specified directory does not appear to be valid"
+
+    # save
+    config::core::set "datastore_list" "${_name}" "1"
+    config::core::set "path_${_name}" "iso:${_path}"
+    [ $? -eq 0 ] || util::err "error saving settings to configuration file"
+}
+
+# add a datastore for img files
+#
+# @param string _name the name of the datastore
+# @param string _path filesystem path
+#
+datastore::img(){
+    local _name="$1"
+    local _path="$2"
+
+    [ -z "${_name}" -o -z "${_path}" ] && util::usage
+    util::check_name "${_name}" || util::err "invalid datastore name - '${_name}'"
+
+    # check name not in use
+    for _curr in ${VM_DATASTORE_LIST}; do
+        [ "${_curr}" = "${_name}" ] && util::err "datstore '${_name}' already exists!"
+    done
+
+    # make sure directory exists
+    [ ! -d "${_path}" ] && util::err "specified directory does not appear to be valid"
+
+    # save
+    config::core::set "datastore_list" "${_name}" "1"
+    config::core::set "path_${_name}" "img:${_path}"
+    [ $? -eq 0 ] || util::err "error saving settings to configuration file"
+}
+
+# find an iso file by looking in the default location
+# and any "iso" datastores
+#
+# @param string _var variable name to put full iso path into
+# @param string _file iso filename to look for
+# @return int success if found
+#
+datastore::iso_find(){
+    local _var="$1"
+    local _file="$2"
+    local _ds _spec
+
+    # given a full path?
+    if [ -z "${_file%%/*}" ] && [ -r "${_file}" ]; then
+        setvar "${_var}" "${_file}"
+        return 0
+    fi
+
+    # file exists in current dir?
+    if [ -r "$(pwd)/${_file}" ]; then
+        setvar "${_var}" "$(pwd)/${_file}"
+        return 0
+    fi
+
+    # look in default store
+    if [ -r "${vm_dir}/.iso/${_file}" ]; then
+        setvar "${_var}" "${vm_dir}/.iso/${_file}"
+        return 0
+    fi
+
+    for _ds in ${VM_DATASTORE_LIST}; do
+            config::core::get "_spec" "path_${_ds}"
+            [ "${_spec%%:*}" != "iso" ] && continue
+
+            if [ -r "${_spec#*:}/${_file}" ]; then
+                setvar "${_var}" "${_spec#*:}/${_file}"
+                return 0
+            fi
+    done
+
+    return 1
+}
+
+# find an img file by looking in the default location
+# and any "img" datastores
+#
+# @param string _var variable name to put full img path into
+# @param string _file img filename to look for
+# @return int success if found
+#
+datastore::img_find(){
+    local _var="$1"
+    local _file="$2"
+    local _ds _spec
+
+    # given a full path?
+    if [ -z "${_file%%/*}" ] && [ -r "${_file}" ]; then
+        setvar "${_var}" "${_file}"
+        return 0
+    fi
+
+    # file exists in current dir?
+    if [ -r "$(pwd)/${_file}" ]; then
+        setvar "${_var}" "$(pwd)/${_file}"
+        return 0
+    fi
+
+    # look in default store
+    if [ -r "${vm_dir}/.img/${_file}" ]; then
+        setvar "${_var}" "${vm_dir}/.img/${_file}"
+        return 0
+    fi
+
+    for _ds in ${VM_DATASTORE_LIST}; do
+            config::core::get "_spec" "path_${_ds}"
+            [ "${_spec%%:*}" != "img" ] && continue
+
+            if [ -r "${_spec#*:}/${_file}" ]; then
+                setvar "${_var}" "${_spec#*:}/${_file}"
+                return 0
+            fi
+    done
+
+    return 1
+}
+
+# list iso files
+#
+datastore::iso_list(){
+    local _ds _spec _format="%-20s%s\n"
+
+    printf "${_format}" "DATASTORE" "FILENAME"
+
+    # look for default iso location
+    [ -d "${vm_dir}/.iso" ] && ls -1 "${vm_dir}/.iso" | awk '{printf "'${_format}'","default",$1}'
+
+    # look for iso datastores
+    for _ds in ${VM_DATASTORE_LIST}; do
+        config::core::get "_spec" "path_${_ds}"
+        [ "${_spec%%:*}" != "iso" ] && continue
+
+        [ -d "${_spec#*:}" ] && ls -1 ${_spec#*:} | awk '{printf "'${_format}'","'${_ds}'",$1}'
+    done
+}
+
+# list img files
+#
+datastore::img_list(){
+    local _ds _spec _format="%-20s%s\n"
+
+    printf "${_format}" "DATASTORE" "FILENAME"
+
+    # look for default img location
+    [ -d "${vm_dir}/.img" ] && ls -1 "${vm_dir}/.img" | awk '{printf "'${_format}'","default",$1}'
+
+    # look for img datastores
+    for _ds in ${VM_DATASTORE_LIST}; do
+        config::core::get "_spec" "path_${_ds}"
+        [ "${_spec%%:*}" != "img" ] && continue
+
+        [ -d "${_spec#*:}" ] && ls -1 ${_spec#*:} | awk '{printf "'${_format}'","'${_ds}'",$1}'
+    done
+}

lib/vm-guest

@@ -1,6 +1,6 @@
 #!/bin/sh
 #-------------------------------------------------------------------------+
-# Copyright (C) 2015 Matt Churchyard (churchers@gmail.com)
+# Copyright (C) 2016 Matt Churchyard (churchers@gmail.com)
 # All rights reserved
 #
 # Redistribution and use in source and binary forms, with or without
@@ -24,12 +24,12 @@
 # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-# __guest_load
+# guest::load
 # this function is responsible for doing any pre-load tasks for a guest.
 # for non uefi guests this normally means running bhyveload or grub-bhyve.
 # this function should return a non-zero value if there's a problem
 # or 0 on success.
-# As this is called from within the scope of __vm_run,
+# As this is called from within the scope of vm::run,
 # the following variables are already set (among others)
 #
 # _name: guest name
@@ -42,29 +42,43 @@
 # _bootdisk: full path to primary disk
 #
 # I've written append wrong as it just needs to be something other than 'write',
-# and is much more readable when all the __log* calls line up
+# and is much more readable when all the util::log* calls line up
 #
 # @param optional string _iso set to the boot iso on install, or not given for normal run
 # @return int 0=success, 15=vm-bhyve error (see log), other=bhyveload|grub-bhyve error code
 #
-__guest_load(){
+guest::load(){
     local _iso="$1"
-    local _args _command _timeout _grub_opt
+    local _args _command _timeout _grub_opt _bsd_loader _custom_args
+
+    # require a boot disk
+    if [ -z "${_bootdisk}" ]; then
+        util::log "guest" "${_name}" "fatal; non-uefi loaders require a boot disk device"
+        return 15
+    fi
 
     # all loaders have same console and wired memory options
-    [ -z "${VM_FOREGROUND}" ] && _args="-c ${_com}"
+    [ -z "${VM_OPT_FOREGROUND}" ] && _args="-c ${_com}"
     [ "${_wiredmem}" = "1" ] && _args="${_args}${_args:+ }-S"
 
     # get timeout
-    __config_get "_timeout" "loader_timeout" "3"
+    config::get "_timeout" "loader_timeout" "3"
 
     case "${_loader}" in
         bhyveload)
             _command="bhyveload"
-            _args="${_args}${_args:+ }-m ${_memory} -e autoboot_delay=${_timeout}"
+            _args="${_args}${_args:+ }-m ${_memory} -e smbios.system.uuid=${_uuid} -e autoboot_delay=${_timeout} -e bhyve_vm_name=${_name}"
+
+            # look for custom bhyveload arguments
+            config::get "_custom_args" "bhyveload_args"
+            [ -n "${_custom_args}" ] && _args="${_args} ${_custom_args}"
+
+            # have a custom guest loader specified?
+            config::get "_bsd_loader" "bhyveload_loader"
+            [ -n "${_bsd_loader}" ] && _args="${_args} -l ${_bsd_loader}"
 
             if [ -n "${_iso}" ]; then
-                _args="${_args} -d ${vm_dir}/.iso/${_iso}"
+                _args="${_args} -d ${_iso}"
             else
                 _args="${_args} -d ${_bootdisk}"
             fi
@@ -74,56 +88,75 @@ __guest_load(){
 
             # check we have grub-bhyve
             if [ $? -ne 0 ]; then
-                __log "guest" "${_name}" "grub requested but sysutils/grub2-bhyve not installed?"
+                util::log "guest" "${_name}" "fatal; grub requested but sysutils/grub2-bhyve not installed?"
                 return 15
             fi
 
             # add device map path and memory
-            _args="${_args}${_args:+ }-m ${vm_dir}/${_name}/device.map -M ${_memory}"
+            _args="${_args}${_args:+ }-m ${VM_DS_PATH}/${_name}/device.map -M ${_memory}"
 
             if [ -n "${_iso}" ]; then
                 _root="cd0"
-                __log_and_write "write" "${_name}" "device.map" "(hd0) ${_bootdisk}"
-                __log_and_write "appnd" "${_name}" "device.map" "(cd0) ${vm_dir}/.iso/${_iso}"
+                util::log_and_write "write" "${_name}" "device.map" "(cd0) ${_iso}"
+                util::log_and_write "appnd" "${_name}" "device.map" "(hd0) ${_bootdisk}"
+                guest::__map_all_disks
 
                 # if we have local grub config, we need to point grub-bhyve at the host.
                 # if not, just use defaults
-                if __guest_write_config "install"; then
-                    _args="${_args} -r host -d ${vm_dir}/${_name}"
+                if guest::__write_config "install"; then
+                    _args="${_args} -r host -d ${VM_DS_PATH}/${_name}"
                 else
                     _args="${_args} -r ${_root}"
                 fi
             else
                 _root="hd0,1"
-                __log_and_write "write" "${_name}" "device.map" "(hd0) ${_bootdisk}"
+                util::log_and_write "write" "${_name}" "device.map" "(hd0) ${_bootdisk}"
+                guest::__map_all_disks
 
-                __config_get "_grub_opt" "grub_run_partition"
+                config::get "_grub_opt" "grub_run_partition"
                 [ -n "${_grub_opt}" ] && _root="hd0,${_grub_opt}"
 
                 # if we have local config, point grub-bhyve at it
                 # otherwise we use defaults, or directory and file specified by user
-                if __guest_write_config "run"; then
-                    _args="${_args} -r host -d ${vm_dir}/${_name}"
+                if guest::__write_config "run"; then
+                    _args="${_args} -r host -d ${VM_DS_PATH}/${_name}"
                 else
                     _args="${_args} -r ${_root}"
 
-                    __config_get "_grub_opt" "grub_run_dir"
+                    config::get "_grub_opt" "grub_run_dir"
                     [ -n "${_grub_opt}" ] && _args="${_args} -d ${_grub_opt}"
-                    __config_get "_grub_opt" "grub_run_file"
+                    config::get "_grub_opt" "grub_run_file"
                     [ -n "${_grub_opt}" ] && _args="${_args} -g ${_grub_opt}"
                 fi
             fi
             ;;
         *)
-            __log "guest" "${_name}" "unsupported loader - '${_loader}'"
+            util::log "guest" "${_name}" "unsupported loader - '${_loader}'"
             return 15
             ;;
     esac
 
     # run the command
-    __log "guest" "${_name}" "${_command} ${_args} ${_name}"
+    util::log "guest" "${_name}" "${_command} ${_args} ${_name}"
     ${_command} ${_args} ${_name}
-    return $?
+}
+
+# Add all extra/non-boot disks to the device.map file
+# Some users may need to access additional disks from the loader
+#
+guest::__map_all_disks(){
+    local _disk _dev _path _num=1
+
+    config::get "_disk" "disk${_num}_name"
+
+    while [ -n "${_disk}" ]; do
+        config::get "_dev" "disk${_num}_dev"
+        vm::get_disk_path "_path" "${_name}" "${_disk}" "${_dev}"
+        util::log_and_write "appnd" "${_name}" "device.map" "(hd${_num}) ${_path}"
+
+        _num=$((_num + 1))
+        config::get "_disk" "disk${_num}_name"
+    done
 }
 
 # See if the user has configured grub commands.
@@ -133,31 +166,31 @@ __guest_load(){
 # @param string _type=install|run which commands to load
 # @return int true (0) if commands were loaded
 #
-__guest_write_config(){
+guest::__write_config(){
     local _type="$1"
     local _command _num=0
 
     # make sure original boot command file grub.cmd is gone
     # we've switched to grub.cfg now as this is the 
     # default for grub-bhyve and makes one less option needed
-    rm "${vm_dir}/${_name}/grub.*" >/dev/null 2>&1
+    rm "${VM_DS_PATH}/${_name}/grub.*" >/dev/null 2>&1
 
-    __config_get "_command" "grub_${_type}${_num}"
+    config::get "_command" "grub_${_type}${_num}"
     [ -z "${_command}" ] && return 1
 
-    __log_and_write "write" "${_name}" "grub.cfg" "timeout=${_timeout}"
-    __log_and_write "appnd" "${_name}" "grub.cfg" "menuentry '${_name} (bhyve ${_type})' {"
-    __log_and_write "appnd" "${_name}" "grub.cfg" "    root=${_root}"
+    util::log_and_write "write" "${_name}" "grub.cfg" "timeout=${_timeout}"
+    util::log_and_write "appnd" "${_name}" "grub.cfg" "menuentry '${_name} (bhyve ${_type})' {"
+    util::log_and_write "appnd" "${_name}" "grub.cfg" "    root=${_root}"
 
     while [ -n "${_command}" ]; do
         # we don't need boot command anymore
-        [ "${_command}" != "boot" ] && __log_and_write "appnd" "${_name}" "grub.cfg" "    ${_command}"
+        [ "${_command}" != "boot" ] && util::log_and_write "appnd" "${_name}" "grub.cfg" "    ${_command}"
 
         _num=$((_num + 1))
-        __config_get "_command" "grub_${_type}${_num}"
+        config::get "_command" "grub_${_type}${_num}"
     done
 
-    __log_and_write "appnd" "${_name}" "grub.cfg" "}"
+    util::log_and_write "appnd" "${_name}" "grub.cfg" "}"
 
     return 0
 }

lib/vm-info

@@ -1,6 +1,6 @@
 #!/bin/sh
 #-------------------------------------------------------------------------+
-# Copyright (C) 2015 Matt Churchyard (churchers@gmail.com)
+# Copyright (C) 2016 Matt Churchyard (churchers@gmail.com)
 # All rights reserved
 #
 # Redistribution and use in source and binary forms, with or without
@@ -27,45 +27,66 @@
 # 'vm info'
 # display a wealth of information about all guests, or one specified
 #
-# @param optional string _name name of the guest to display
+# @param optional string[multiple] _name name of the guest to display
 #
-__vm_info(){
+info::guest(){
     local _name="$1"
-    local _bridge_list=$(ifconfig | grep ^bridge | cut -d: -f1)
+    local _bridge_list=$(ifconfig -g vm-switch)
+    local _ds
 
-    __vm_running_load
+    vm::running_load
 
+    # see if guest name(s) provided
     if [ -n "${_name}" ]; then
-        __vm_info_guest "${_name}"
+        while [ -n "${_name}" ]; do
+            datastore::get_guest "${_name}" || util::err "unable to locate virtual machine '${_name}'"
+            info::guest_show "${_name}"
+
+            shift
+            _name="$1"
+        done
+
         exit
     fi
 
-    ls -1 "${vm_dir}" | \
-    while read _name; do
-        [ -e "${vm_dir}/${_name}/${_name}.conf" ] && __vm_info_guest "${_name}"
+    # show all guests from all datastores
+    for _ds in ${VM_DATASTORE_LIST}; do
+        datastore::get "${_ds}" || continue
+
+        ls -1 "${VM_DS_PATH}" 2>/dev/null | \
+        while read _name; do
+            [ -e "${VM_DS_PATH}/${_name}/${_name}.conf" ] && info::guest_show "${_name}"
+        done
     done
 }
 
 # 'vm switch info'
 # display config of each virtual switch as well as stats and connected guests
 #
-# @param optional string _switch name of switch to display
+# @param optional string[multiple] _switch name of switch to display
 #
-__vm_info_switch(){
+info::switch(){
     local _switch="$1"
-    local _conf="${vm_dir}/.config/switch"
     local _list
 
-    __config_load "${_conf}"
-    __config_get "_list" "switch_list"
+    # load config file manually using non-core function
+    # this means we can share the config_output function with guest
+    config::load "${vm_dir}/.config/system.conf"
+    config::get "_list" "switch_list"
 
     if [ -n "${_switch}" ]; then
-        __vm_info_switch_show "${_switch}"
+        while [ -n "${_switch}" ]; do
+            info::switch_show "${_switch}"
+
+            shift
+            _switch="$1"
+        done
+
         exit
     fi
 
     for _switch in ${_list}; do
-        __vm_info_switch_show "${_switch}"
+        info::switch_show "${_switch}"
     done
 }
 
@@ -73,51 +94,47 @@ __vm_info_switch(){
 #
 # @param string _switch the name of the switch
 #
-__vm_info_switch_show(){
+info::switch_show(){
     local _switch="$1"
-    local _bridge _vale _id
+    local _type _bridge _vale _netgraph _id
     local _INDENT="  "
 
     [ -z "${_switch}" ] && return 1
 
-    __config_get "_bridge" "bridge_${_switch}"
-    __config_get "_vale" "vale_${_switch}"
+    config::get "_type" "type_${_switch}" "standard"
+    config::get "_bridge" "bridge_${_switch}"
+    config::get "_vale" "vale_${_switch}"
+    config::get "_netgraph" "netgraph_${_switch}"
+
 
     echo "------------------------"
     echo "Virtual Switch: ${_switch}"
     echo "------------------------"
 
-    if __checkyesno "${_vale}"; then
-        echo "${_INDENT}type: vale"
-        __switch_vale_id "_id" "${_switch}"
-    elif [ -n "${_bridge}" ]; then
-        echo "${_INDENT}type: manual"
-        _id="${_bridge}"
-    else
-        echo "${_INDENT}type: auto"
-        __switch_get_ident "_bridge" "${_switch}"
-        _id="${_bridge}"
-    fi
+    echo "${_INDENT}type: ${_type}"
+    switch::id "_id" "${_switch}"
+
+    # we don't have a bridge for vale and netgraph switches
+    [ "${_type}" != "vale" ] && [ "${_type}" != "netgraph" ] && _bridge="${_id}"
 
     echo "${_INDENT}ident: ${_id:--}"
 
-    __vm_info_output_config "vlan_${_switch}" "vlan"
-    __vm_info_output_config "nat_${_switch}" "nat"
-    __vm_info_output_config "ports_${_switch}" "physical-ports"
+    info::__output_config "vlan_${_switch}" "vlan"
+    info::__output_config "ports_${_switch}" "physical-ports"
 
     if [ -n "${_bridge}" ]; then
         _stats=$(netstat -biI "${_bridge}" |grep '<Link#' | tail -n1 | awk '{ for (i=NF; i>1; i--) printf("%s ",$i); print $1; }' | awk '{print $5,$2}')
 
         if [ -n "${_stats}" ]; then
-            _b_in=$(__vm_info_bytes_human "${_stats%% *}")
-            _b_out=$(__vm_info_bytes_human "${_stats##* }")
+            _b_in=$(info::__bytes_human "${_stats%% *}")
+            _b_out=$(info::__bytes_human "${_stats##* }")
 
             echo "${_INDENT}bytes-in: ${_stats%% *} (${_b_in})"
             echo "${_INDENT}bytes-out: ${_stats##* } (${_b_out})"
         fi
 
         # show guest ports
-        __vm_info_switch_ports
+        info::switch_ports
     fi
 
     echo ""
@@ -125,12 +142,12 @@ __vm_info_switch_show(){
 
 # get all guest ports for current bridge
 #
-__vm_info_switch_ports(){
+info::switch_ports(){
     local _port_list=$(ifconfig "${_bridge}" |grep 'member: tap' |awk '{print $2}')
     local _port _guest
 
     for _port in ${_port_list}; do
-        _guest=$(ifconfig "${_port}" |grep 'description: vmnet-' |awk '{print $2}' |cut -d'-' -f2)
+        _guest=$(ifconfig "${_port}" |grep 'description: vmnet' |awk '{print $2}' |cut -d'/' -f2)
 
         echo ""
         echo "${_INDENT}virtual-port"
@@ -143,21 +160,21 @@ __vm_info_switch_ports(){
 #
 # @param string _name name of the guest to display
 #
-__vm_info_guest(){
+info::guest_show(){
     local _name="$1"
-    local _conf="${vm_dir}/${_name}/${_name}.conf"
+    local _conf="${VM_DS_PATH}/${_name}/${_name}.conf"
     local _INDENT="  "
     local _RUN="0"
-    local _res_mem _b_res_mem _global_run _port
+    local _res_mem _b_res_mem _global_run _port _opt _pid
 
     [ -z "${_name}" ] && return 1
     [ ! -f "${_conf}" ] && return 1
 
-    __config_load "${_conf}"
+    config::load "${_conf}"
 
     # check local and global runstate
     [ -e "/dev/vmm/${_name}" ] && _RUN="1"
-    __vm_running_check "_global_run" "${_name}"
+    vm::running_check "_global_run" "_pid" "${_name}"
     _global_run=$(echo "${_global_run}" | tr '[:upper:]' '[:lower:]')
 
     echo "------------------------"
@@ -165,69 +182,133 @@ __vm_info_guest(){
     echo "------------------------"
 
     echo "${_INDENT}state: ${_global_run}"
+    echo "${_INDENT}datastore: ${VM_DS_NAME}"
 
     # basic guest configuration
-    __vm_info_output_config "guest"
-    __vm_info_output_config "loader" "" "default"
-    __vm_info_output_config "uuid" "" "auto"
-    __vm_info_output_config "uefi" "" "no"
-    __vm_info_output_config "cpu"
-    __vm_info_output_config "memory"
+    info::__output_config "loader" "" "none"
+    info::__output_config "uuid" "" "auto"
+    info::__output_config "cpu"
+
+    # check for a cpu topology
+    config::get "_opt" "cpu_sockets"
+
+    if [ -n "${_opt}" ]; then
+        echo -n "${_INDENT}cpu-topology: sockets=${_opt}"
+        config::get "_opt" "cpu_cores"
+        [ -n "${_opt}" ] && echo -n ", cores=${_opt}"
+        config::get "_opt" "cpu_threads"
+        [ -n "${_opt}" ] && echo -n ", threads=${_opt}"
+        echo ""
+    fi
+
+    info::__output_config "memory"
 
     # running system details
     if [ "${_RUN}" = "1" ]; then
         _res_mem=$(bhyvectl --get-stats --vm="${_name}" |grep 'Resident memory' |awk '{print $3}')
 
         if [ -n "${_res_mem}" ]; then
-            _b_res_mem=$(__vm_info_bytes_human "${_res_mem}")
+            _b_res_mem=$(info::__bytes_human "${_res_mem}")
             echo "${_INDENT}memory-resident: ${_res_mem} (${_b_res_mem})"
         fi
 
         # show com ports
-        if [ -e "${vm_dir}/${_name}/console" ]; then
+        if [ -e "${VM_DS_PATH}/${_name}/console" ]; then
             echo ""
             echo "${_INDENT}console-ports"
 
-            cat "${vm_dir}/${_name}/console" | \
+            cat "${VM_DS_PATH}/${_name}/console" | \
             while read _port; do
                 echo "${_INDENT}${_INDENT}${_port%%=*}: ${_port##*=}"
             done
+
+            # virtio-console devices
+            info::guest_vtcon
         fi
     fi
 
     # network interfaces
-    __vm_info_networking
+    info::guest_networking
 
     # disks
-    __vm_info_disks
+    info::guest_disks
+
+    # zfs data
+    info::guest_zfs
 
     echo ""
 }
 
+# display any virtio consoles
+#
+info::guest_vtcon(){
+    local _INDENT="    "
+    local _console _num=0
+
+    config::get "_console" "virt_console0"
+    [ -z "${_console}" ] && return 0
+
+    echo ""
+
+    while [ -n "${_console}" -a ${_num} -lt 16 ]; do
+        # if set to "yes/on/1", just use the console number as port name
+        case "${_console}" in
+            [Yy][Ee][Ss]|[Oo][Nn]|1) _console="${_num}" ;;
+        esac
+
+        echo "${_INDENT}vtcon${_num}: ${VM_DS_PATH}/${_name}/vtcon.${_console}"
+
+        _num=$((_num + 1))
+        config::get "_console" "virt_console${_num}"        
+    done
+}
+
+# display zfs snapshot/origin data
+#
+info::guest_zfs(){
+   local _INDENT="    "
+   local _data
+
+   [ -z "${VM_DS_ZFS}" ] && return 1
+
+   _data=$(zfs list -o name,used,creation -s creation -rHt snapshot "${VM_DS_ZFS_DATASET}/${_name}" | sed "s/^/${_INDENT}/")
+
+   if [ -n "${_data}" ]; then
+       echo ""
+       echo "  snapshots"
+       echo "${_data}"
+   fi
+
+   _data=$(zfs get -Ho value origin "${VM_DS_ZFS_DATASET}/${_name}")
+   [ "${_data}" = "-" ] && return 0
+
+   echo ""
+   echo "  clone-origin"
+   echo "    ${_data}"
+}
+
 # display disks
 #
-__vm_info_disks(){
+info::guest_disks(){
     local _num=0
     local _disk _type _dev _path _size _b_size _used _b_used
     local _INDENT="    "
 
-    while [ 1 ]; do
-        __config_get "_disk" "disk${_num}_name"
-        __config_get "_type" "disk${_num}_type"
-        __config_get "_dev" "disk${_num}_dev"
+    while true; do
+        config::get "_disk" "disk${_num}_name"
+        config::get "_type" "disk${_num}_type"
+        config::get "_dev" "disk${_num}_dev" "file"
         [ -z "${_disk}" -o -z "${_type}" ] && break
 
-        : ${_dev:=file}
-
-        __vm_get_disk_path "_path" "${_name}" "${_disk}" "${_dev}"
+        vm::get_disk_path "_path" "${_name}" "${_disk}" "${_dev}"
 
         echo ""
         echo "  virtual-disk"
         echo "${_INDENT}number: ${_num}"
 
-        __vm_info_output_config "disk${_num}_dev" "device-type" "file"
-        __vm_info_output_config "disk${_num}_type" "emulation"
-        __vm_info_output_config "disk${_num}_opts" "options"
+        info::__output_config "disk${_num}_dev" "device-type" "file"
+        info::__output_config "disk${_num}_type" "emulation"
+        info::__output_config "disk${_num}_opts" "options"
 
         echo "${_INDENT}system-path: ${_path:--}"
 
@@ -237,7 +318,7 @@ __vm_info_disks(){
         if [ -n "${_path}" ]; then
             case "${_dev}" in
                 file)
-                    _size=$(stat "${_path}" | cut -d' ' -f8)
+                    _size=$(stat -f%z "${_path}")
                     _used=$(du "${_path}" | awk '{print $1}')
                     _used=$((_used * 1024))
                     ;;
@@ -245,11 +326,14 @@ __vm_info_disks(){
                     _size=$(zfs get -Hp volsize "${_path#/dev/zvol/}" |cut -f3)
                     _used=$(zfs get -Hp refer "${_path#/dev/zvol/}" |cut -f3)
                     ;;
+                iscsi)
+                    _size=$(sysctl -b kern.geom.conftxt | awk "/ ${_path#/dev/} /{print \$4}")
+                    _used=${_size}
             esac
 
             if [ -n "${_size}" -a -n "${_used}" ]; then
-                _b_size=$(__vm_info_bytes_human "${_size}")
-                _b_used=$(__vm_info_bytes_human "${_used}")
+                _b_size=$(info::__bytes_human "${_size}")
+                _b_used=$(info::__bytes_human "${_used}")
                 echo "${_INDENT}bytes-size: ${_size} (${_b_size})"
                 echo "${_INDENT}bytes-used: ${_used} (${_b_used})"
             fi
@@ -261,13 +345,13 @@ __vm_info_disks(){
 
 # display networking configuration
 #
-__vm_info_networking(){
+info::guest_networking(){
     local _num=0
     local _int _id _tag _switch _stats _b_in _b_out
     local _INDENT="    "
 
-    while [ 1 ]; do
-        __config_get "_int" "network${_num}_type"
+    while true; do
+        config::get "_int" "network${_num}_type"
         [ -z "${_int}" ] && break
 
         echo ""
@@ -275,20 +359,20 @@ __vm_info_networking(){
         echo "${_INDENT}number: ${_num}"
 
         # basic interface config
-        __vm_info_output_config "network${_num}_type" "emulation"
-        __vm_info_output_config "network${_num}_switch" "virtual-switch"
-        __vm_info_output_config "network${_num}_mac" "fixed-mac-address"
-        __vm_info_output_config "network${_num}_device" "fixed-device"
+        info::__output_config "network${_num}_type" "emulation"
+        info::__output_config "network${_num}_switch" "virtual-switch"
+        info::__output_config "network${_num}_mac" "fixed-mac-address"
+        info::__output_config "network${_num}_device" "fixed-device"
 
         # if running, try to get some more interface details
         if [ "${_RUN}" = "1" ]; then
-            __config_get "_switch" "network${_num}_switch"
+            config::get "_switch" "network${_num}_switch"
 
-            _int=$(ifconfig | grep -B1 "vmnet-${_name}-${_num}-" | head -n1 | cut -d' ' -f1,6)
+            _int=$(ifconfig | grep -B1 "vmnet/${_name}/${_num}/" | head -n1 | cut -d' ' -f1,6)
             _id=${_int%%:*}
-            _tag=$(ifconfig | grep "vmnet-${_name}-${_num}-" | cut -d' ' -f2)
+            _tag=$(ifconfig | grep "vmnet/${_name}/${_num}/" | cut -d' ' -f2)
 
-            __vm_info_find_bridge "_bridge" "${_id}"
+            info::__find_bridge "_bridge" "${_id}"
 
             echo "${_INDENT}active-device: ${_id:--}"
             echo "${_INDENT}desc: ${_tag:--}"
@@ -296,9 +380,9 @@ __vm_info_networking(){
             echo "${_INDENT}bridge: ${_bridge:--}"
 
             if [ -n "${_id}" ]; then
-                _stats=$(netstat -biI "${_id}" | tail -n1 | awk '{ for (i=NF; i>1; i--) printf("%s ",$i); print $1; }' | awk '{print $2,$5}')
-                _b_in=$(__vm_info_bytes_human "${_stats%% *}")
-                _b_out=$(__vm_info_bytes_human "${_stats##* }")
+                _stats=$(netstat -biI "${_id}" |grep '<Link#' |tail -n1 |awk '{ for (i=NF; i>1; i--) printf("%s ",$i); print $1; }' |awk '{print $2,$5}')
+                _b_in=$(info::__bytes_human "${_stats%% *}")
+                _b_out=$(info::__bytes_human "${_stats##* }")
 
                 echo "${_INDENT}bytes-in: ${_stats%% *} (${_b_in})"
                 echo "${_INDENT}bytes-out: ${_stats##* } (${_b_out})"
@@ -316,27 +400,27 @@ __vm_info_networking(){
 # @param optional string _title title to display instead of using option name
 # @param optional string _default default value to display if not -
 #
-__vm_info_output_config(){
+info::__output_config(){
     local _option="$1"
     local _title="$2"
     local _default="$3"
     local _var
 
-    __config_get "_var" "${_option}" "${_default:--}"
+    config::get "_var" "${_option}" "${_default:--}"
     [ -z "${_title}" ] && _title="${_option}"
 
     echo "${_INDENT}${_title}: ${_var}"
 }
 
 # try and find the bridge an interface is a member of.
-# we do this rather than just use __switch_get_ident as
+# we do this rather than just use switch::id as
 # this should be able to locate the bridge even for devices
 # that have been bridged manually and have no switch name configured
 #
 # @param string _var variable to put value into
 # @param string _interface interface to look for
 #
-__vm_info_find_bridge(){
+info::__find_bridge(){
     local _var="$1"
     local _interface="$2"
     local _br _found
@@ -360,11 +444,14 @@ __vm_info_find_bridge(){
 #
 # @param int _val the value to convert
 #
-__vm_info_bytes_human(){
+info::__bytes_human(){
     local _val="$1" _int _ext
-    local _num=1
+    local _dec="$2"
+    local _num="$3"
 
+    : ${_dec:=3}
     : ${_val:=0}
+    : ${_num:=1}
     _int=${_val%%.*}
 
     while [ ${_int} -ge 1024 -a ${_num} -lt 5 ]; do
@@ -381,6 +468,36 @@ __vm_info_bytes_human(){
         5) _ext="T" ;;
     esac
 
-    export LC_NUMERIC="C"
-    printf "%.3f%s" "${_val}" "${_ext}"
+    export LC_ALL="C"
+    printf "%.${_dec}f%s" "${_val}" "${_ext}"
+}
+
+info::__find_iscsi() {
+    local _var="$1"
+    # _target format: [iqn.*]unique_name[/N] where N is the lun# and defaults
+    # to 0. The address before the unique_name can be omitted so long as
+    # the unique_name is sufficient to select only one output of iscsictl -L.
+    local _target="$2"
+    local _lun _col _found
+
+    # If no lun is specified, assume /0
+    _lun=${_target##*/}
+    [ "${_lun}" = "${_target}" ] && _lun=0
+    _target=${_target%/*}
+
+    _found=$(iscsictl -L -w 10 | grep ${_target} | wc -l)
+    if [ "${_found}" -ne 1 ]; then
+        setvar "${_var}" ""
+        util::err "Unable to locate unique iSCSI device ${_target}"
+    fi
+
+    # _col to be the column of iscsictl -L we want
+    _col=$((_lun + 4))
+    _found=$(iscsictl -L | awk "/${_target}/{print \$${_col}}")
+    if echo "${_found}" | egrep -q '^da[0-9]+$'; then
+        setvar "${_var}" /dev/"${_found}"
+        return 0
+    fi
+
+    util::err "Unable to locate iSCSI device ${_target}/${_lun}"
 }

lib/vm-migration

@@ -0,0 +1,278 @@
+#!/bin/sh
+#-------------------------------------------------------------------------+
+# Copyright (C) 2021 Matt Churchyard (churchers@gmail.com)
+# All rights reserved
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted providing that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# vm migrate ...
+#
+# @param string name the guest to send
+# @param string host host to send guest to
+#
+migration::run(){
+    local _name
+    local _ds="default"
+    local _start="1"
+    local _renaming="0"
+    local _config _opt _stage _inc _triple _rdataset _pid _exists _rname _running
+    local _snap1 _snap2 _snap3 _destroy
+    local _count=0
+
+    while getopts cn12txr:d:i: _opt; do
+        case $_opt in
+            c) _config="1" ;;
+            r) _rname="${OPTARG}" ;;
+            n) _start="" ;;
+            i) _inc="${OPTARG}" ;;
+            1) _stage="1" ;;
+            2) _stage="2" ;;
+            t) _triple="1" ;;
+            x) _destroy="1" ;;
+            d) _ds="${OPTARG}" ;;
+        esac
+    done
+
+    # get the name and host
+    shift $((OPTIND -1))
+    _name="$1"
+    _host="$2"
+
+    # do we want to output our config?
+    # sender uses the config option to pull config from the recieve end
+    if [ -n "${_config}" ]; then
+        migration::__check_config "${_ds}"
+        exit
+    fi
+
+    # basic checks
+    [ -z "${_name}" -o -z "${_host}" ] && util::usage
+    datastore::get_guest "${_name}" || util:err "unable to locate guest - '${_name}'"
+    [ -z "${VM_DS_ZFS}" ] && util:err "the source datastore must be ZFS to support migration"
+    [ -n "${_stage}" -a -n "${_triple}" ] && util::err "single stage and triple stage are mutually exclusive"
+    [ "${_stage}" = "2" -a -z "${_inc}" ] && util::err "source snapshot must be given when running stage 2"
+
+    if [ -n "${_rname}" ]; then
+        util::check_name "${_rname}" || util::err "invalid virtual machine name - '${_rname}'"
+        _renaming="1"
+    else
+        _rname="${_name}"
+    fi
+
+    # check guest can be sent
+    config::load "${VM_DS_PATH}/${_name}/${_name}.conf"
+    migration::__check_compat
+
+    # check running state
+    vm::confirm_stopped "${_name}" "1" >/dev/null 2>&1
+    _state=$?
+    [ ${_state} -eq 2 ] && util::err "guest is powered up on another host"
+    [ ${_state} -eq 1 ] && _running="1"
+
+    # try to get pid
+    if [ -n "${_running}" ]; then
+        _pid=$(pgrep -fx "bhyve: ${_name}")
+        [ -z "${_pid}" ] && util::err "guest seems to be running but can't find its pid"
+    fi
+
+    # try to get remote config
+    _rdataset=$(ssh "${_host}" vm migrate -cd "${_ds}" 2>/dev/null)
+    [ $? = "1" -o -z "${_rdataset}" ] && util::err "unable to get config from ${_host}"
+
+    echo "Attempting to send ${_name} to ${_host}"
+    echo "  * remote dataset ${_rdataset}/${_rname}"
+    [ -n "${_running}" ] && echo "  * source guest is powered on (#${_pid})"
+
+    # STAGE 1
+    # we send a full snapshot of the guest
+    if [ -z "${_stage}" -o "${_stage}" = "1" ]; then
+        _snap1="$(date +'%Y%m%d%H%M%S-s1')"
+        echo "  * stage 1: taking snapshot ${_snap1}"
+        zfs snapshot -r "${VM_DS_ZFS_DATASET}/${_name}@${_snap1}" >/dev/null 2>&1
+        [ $? -eq 0 ] || util::err_inline "error taking local snapshot"
+
+        # send this snapshot
+        migrate::__send "1" "${_snap1}" "${_inc}"
+    fi
+
+    # STAGE 1B
+    # do it again in triple mode
+    # for a big guest, hopefully not too much changed during full send
+    # this will therefore complete fairly quick, leaving very few changes for stage 2
+    if [ -n "${_triple}" ]; then
+        _snap2="$(date +'%Y%m%d%H%M%S-s1b')"
+        echo "  * stage 1b: taking snapshot ${_snap2}"
+        zfs snapshot -r "${VM_DS_ZFS_DATASET}/${_name}@${_snap2}" >/dev/null 2>&1
+        [ $? -eq 0 ] || util::err_inline "error taking local snapshot"
+
+        # send this snapshot
+        migrate::__send "1b" "${_snap2}" "${_snap1}"
+    fi
+
+    # only running first stage
+    if [ "${_stage}" = "1" ]; then
+        echo "  * done"
+        exit
+    fi
+
+    # if it's running we now need to stop it
+    if [ -n "${_running}" ]; then
+        echo -n "  * stage 2: attempting to stop guest"
+
+        kill ${_pid} >/dev/null 2>&1
+
+        while [ ${_count} -lt 60 ]; do
+            sleep 2
+            kill -0 ${_pid} >/dev/null 2>&1 || break
+            echo -n "."
+            _count=$((_count + 1))
+        done
+
+        echo ""
+    fi
+
+    # has it stopped?
+    kill -0 ${_pid} >/dev/null 2>&1 && util:err_inline "failed to stop guest"
+    echo "  * stage 2: guest powered off"
+
+    # only needed if running or specifically doing a stage 2
+    if [ -n "${_running}" -o "${_stage}" = "2" ]; then
+        _snap3="$(date +'%Y%m%d%H%M%S-s2')"
+        echo "  * stage 2: taking snapshot ${_snap3}"
+        zfs snapshot -r "${VM_DS_ZFS_DATASET}/${_name}@${_snap3}" >/dev/null 2>&1
+        [ $? -eq 0 ] || util::err_inline "error taking local snapshot"
+
+        # send this snapshot
+        if [ "${_triple}" = "1" ]; then
+            migrate::__send "2" "${_snap3}" "${_snap2}"
+        elif [ "${_stage}" = "2" ]; then
+            migrate::__send "2" "${_snap3}" "${_inc}"
+        else
+            migrate::__send "2" "${_snap3}" "${_snap1}"
+        fi
+    fi
+
+    # do we need to rename?
+    [ "${_renaming}" = "1" ] && migrate::__rename_config
+
+    # start
+    if [ -n "${_start}" -a -n "${_running}" ]; then
+        echo "  * attempting to start ${_rname} on ${_host}"
+        ssh ${_host} vm start ${_rname}
+    fi
+
+    if [ -n "${_destroy}" ]; then
+        echo "  * removing source guest"
+        zfs destroy -r "${VM_DS_ZFS_DATASET}/${_name}"
+    else
+        echo "  * removing snapshots"
+        [ -n "${_snap1}" ] && zfs destroy "${VM_DS_ZFS_DATASET}/${_name}@${_snap1}" >/dev/null 2>&1
+        [ -n "${_snap2}" ] && zfs destroy "${VM_DS_ZFS_DATASET}/${_name}@${_snap2}" >/dev/null 2>&1
+        [ -n "${_snap3}" ] && zfs destroy "${VM_DS_ZFS_DATASET}/${_name}@${_snap3}" >/dev/null 2>&1
+    fi
+
+    echo "  * done"
+}
+
+# updates the config file for a renamed guest
+# god knows why I didn't just use "guest.conf"
+#
+migrate::__rename_config(){
+    local _path
+
+    # we need the mount path first
+    _path=$(ssh "${_host}" mount | grep "^${_rdataset} " | cut -wf3)
+
+    if [ $? -ne 0 -o -z "${_path}" ]; then
+        echo "  ! failed to find remote datastore path. guest may not start"
+        return 1
+    fi
+
+    # make sure it's mounted on remote
+    ssh "${_host}" zfs mount "${_rdataset}/${_rname}" >/dev/null 2>&1
+
+    echo "  * renaming configuration file to ${_rname}.conf"
+    ssh "${_host}" mv "${_path}/${_rname}/${_name}.conf" "${_path}/${_rname}/${_rname}.conf" >/dev/null 2>1
+
+    if [ $? -ne 0 ]; then
+        echo "  ! failed to find rename remote configuration file. guest may not start"
+        return 1
+    fi
+}
+
+migrate::__send(){
+    local _stage="$1"
+    local _snap="$2"
+    local _inc="$3"
+
+    # are we sending incremental?
+    if [ -n "${_inc}" ]; then
+        echo "  * stage ${_stage}: sending ${VM_DS_ZFS_DATASET}/${_name}@${_snap} (incremental source ${_inc})"
+        zfs send -Ri "${_inc}" "${VM_DS_ZFS_DATASET}/${_name}@${_snap}" | ssh ${_host} zfs recv "${_rdataset}/${_rname}"
+    else
+        echo "  * stage ${_stage}: sending ${VM_DS_ZFS_DATASET}/${_name}@${_snap}"
+        zfs send -R "${VM_DS_ZFS_DATASET}/${_name}@${_snap}" | ssh ${_host} zfs recv "${_rdataset}/${_rname}"
+    fi
+
+    [ $? -eq 0 ] || util::err_inline "error detected while sending snapshot"
+    echo "  * stage ${_stage}: snapshot sent"
+}
+
+# currently just outputs zfs path or error if datastore isn't zfs
+# in future may also return some data we can use to verify compat, etc
+#
+# @param string _ds the datastore to get details of
+#
+migration::__check_config(){
+    local _ds="$1"
+
+    datastore::get "${_ds}"
+    [ -z "${VM_DS_ZFS}" ] && exit 1
+
+    # output the datastore dataset
+    # sender needs this to do a zfs recv
+    echo "${VM_DS_ZFS_DATASET}"
+}
+
+# see if a guest can be migrated.
+# there are a few guest settings that are likely to
+# cause the guest to break if it's moved to another host
+#
+migration::__check_compat(){
+    local _setting _err _num=0
+
+    # check pass through
+    config::get "_setting" "passthru0"
+    [ -n "${_setting}" ] && _err="pci pass-through enabled"
+
+    # check for custom disks
+    # file/zvol are under guest dataset and should go across ok
+    # custom disks could be anywhere
+    while true; do
+        config::get "_setting" "disk${_num}_type"
+        [ -z "${_setting}" ] && break
+        [ "${_setting}" = "custom" ] && _err="custom disk(s) configured" && break
+        _num=$((_num + 1))
+    done
+
+    [ -n "${_err}" ] && util::err "migration is not supported for this guest (${_err})"
+}

lib/vm-rctl

@@ -28,35 +28,39 @@
 # set limits to virtual machine
 # this is the background process
 #
-__rctl_set_limits(){
+rctl::set(){
     local _pcpu _rbps _wbps _riops _wiops
-    local _pid
+    local _pid _pri
 
     # get limit settings
-    __config_get "_pcpu" "limit_pcpu"
-    __config_get "_rbps" "limit_rbps"
-    __config_get "_wbps" "limit_wbps"
-    __config_get "_riops" "limit_riops"
-    __config_get "_wiops" "limit_wiops"
+    config::get "_pri" "priority"
+    config::get "_pcpu" "limit_pcpu"
+    config::get "_rbps" "limit_rbps"
+    config::get "_wbps" "limit_wbps"
+    config::get "_riops" "limit_riops"
+    config::get "_wiops" "limit_wiops"
+
+    # wait till bhyve starts and get pid
+    sleep 1
+    _pid=$(pgrep -fx "bhyve[: ].* ${_name}")
+    [ -z "${_pid}" ] && return 1
+
+    # check for a priority
+    [ -n "${_pri}" ] && renice ${_pri} ${_pid} >/dev/null 2>&1
 
     # return if there are no limits
     [ -z "${_pcpu}${_rbps}${_wbps}${_riops}${_wiops}" ] && return 1
 
-    # wait till bhyve starts and get pid
-    sleep 1
-    _pid=$(pgrep -fx "bhyve: ${_name}")
-    [ -z "${_pid}" ] && return 1
-
     # see if rctl works
     /usr/bin/rctl >/dev/null 2>&1
     [ $? -ne 0 ] && \
-        __log "guest" "${_name}" "RCTL support requested but RCTL not available" && return 1
+        util::log "guest" "${_name}" "RCTL support requested but RCTL not available" && return 1
 
-    __log "guest" "${_name}" "applying rctl limits"
+    util::log "guest" "${_name}" "applying rctl limits"
 
     if [ -n "${_pcpu}" ]; then
         /usr/bin/rctl -a process:${_pid}:pcpu:deny=${_pcpu} >/dev/null 2>&1
-        [ $? -eq 0 ] && __log "guest" "${_name}" " pcpu=${_pcpu}"
+        [ $? -eq 0 ] && util::log "guest" "${_name}" " pcpu=${_pcpu}"
     fi
 
     # at this point we can return if < FreeBSD 11
@@ -64,21 +68,21 @@ __rctl_set_limits(){
 
     if [ -n "${_rbps}" ]; then
         /usr/bin/rctl -a process:${_pid}:readbps:throttle=${_rbps} >/dev/null 2>&1
-        [ $? -eq 0 ] && __log "guest" " readbps=${_rbps}"
+        [ $? -eq 0 ] && util::log "guest" " readbps=${_rbps}"
     fi
 
     if [ -n "${_wbps}" ]; then
         /usr/bin/rctl -a process:${_pid}:writebps:throttle=${_wbps} >/dev/null 2>&1
-        [ $? -eq 0 ] && __log "guest" " writebps=${_wbps}"
+        [ $? -eq 0 ] && util::log "guest" " writebps=${_wbps}"
     fi
 
     if [ -n "${_riops}" ]; then
         /usr/bin/rctl -a process:${_pid}:readiops:throttle=${_riops} >/dev/null 2>&1
-        [ $? -eq 0 ] && __log "guest" " readiops=${_riops}"
+        [ $? -eq 0 ] && util::log "guest" " readiops=${_riops}"
     fi
 
     if [ -n "${_wiops}" ]; then
         /usr/bin/rctl -a process:${_pid}:writeiops:throttle=${_wiops} >/dev/null 2>&1
-        [ $? -eq 0 ] && __log "guest" " writeiops=${_wiops}"
+        [ $? -eq 0 ] && util::log "guest" " writeiops=${_wiops}"
     fi
 }

lib/vm-switch

@@ -1,6 +1,6 @@
 #!/bin/sh
 #-------------------------------------------------------------------------+
-# Copyright (C) 2015 Matt Churchyard (churchers@gmail.com)
+# Copyright (C) 2016 Matt Churchyard (churchers@gmail.com)
 # All rights reserved
 #
 # Redistribution and use in source and binary forms, with or without
@@ -24,572 +24,444 @@
 # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
+# switch libraries
+#
+. "${LIB}/vm-switch-netgraph"
+. "${LIB}/vm-switch-manual"
+. "${LIB}/vm-switch-standard"
+. "${LIB}/vm-switch-vale"
+. "${LIB}/vm-switch-vxlan"
+
 # create switches from rc list on init
 # this should run once per boot to make sure switches from the
 # configuration file have bridge interfaces. If any new switches are
-# created, the bridge is done at the same time
+# created, the create function takes care of setting them up
 #
-__switch_init(){
-    local _switchlist _switch _id
-    local _nat _havenat=0 _bridge _vale
+switch::init(){
+    local _switchlist _switch _type
 
-    __config_load "${vm_dir}/.config/switch"
-    __config_get "_switchlist" "switch_list"
+    config::core::get "_switchlist" "switch_list"
 
-    # create bridges for each switch if they don't already exist
     if [ -n "${_switchlist}" ]; then
         for _switch in ${_switchlist}; do
-            # do nothing if it's a vale switch
-            __config_get "_vale" "vale_${_switch}"
-            __checkyesno "${_vale}" && continue
+            # get the switch type
+            config::core::get "_type" "type_${_switch}"
 
-            # do nothing if the bridge already exists
-            __switch_get_ident "_id" "${_switch}"
-            [ -n "${_id}" ] && continue
-
-            __config_get "_nat" "nat_${_switch}"
-            __config_get "_bridge" "bridge_${_switch}"
-
-            if [ -n "${_bridge}" ]; then
-                ifconfig "${_bridge}" description "vm-${_switch}" up >/dev/null 2>&1
-            else
-                _id=$(ifconfig bridge create)
-                [ $? -ne 0 ] && __err "failed to create bridge interface"
-                ifconfig "${_id}" description "vm-${_switch}" up
-
-                # add all member interfaces
-                __switch_add_allmembers "${_switch}" "${_id}"
-            fi
-
-            [ -n "${_nat}" ] &&_havenat="1"
+            case "${_type}" in
+                vxlan)    switch::vxlan::init "${_switch}" ;;
+                manual)   switch::manual::init "${_switch}" ;;
+                vale)     ;;
+                netgraph) ;;
+                *)        switch::standard::init "${_switch}" ;;
+            esac
         done
     fi
-
-    # only load dnsmasq/pf if we have a nat switch
-    [ "${_havenat}" = "1" ] && __switch_nat_init
 }
 
-# list switches, currently just from stored configuration
+# list switches configured
 #
-__switch_list(){
-    local _switchlist _portlist _switch _port
-    local _vlan _nat _bridge _type _vale
-    local _id _format="%-15s %-10s %-11s %-9s %-12s %s\n"
+switch::list(){
+    local _switchlist _switch _type
+    local _id _format="%s^%s^%s^%s^%s^%s^%s^%s\n"
 
-    __config_load "${vm_dir}/.config/switch"
-    __config_get "_switchlist" "switch_list"
+    config::core::get "_switchlist" "switch_list"
 
-    printf "${_format}" "NAME" "TYPE" "IDENT" "VLAN" "NAT" "PORTS"
+    {
+        printf "${_format}" "NAME" "TYPE" "IFACE" "ADDRESS" "PRIVATE" "MTU" "VLAN" "PORTS"
 
-    for _switch in ${_switchlist}; do
-        __switch_get_ident "_id" "${_switch}"
-        [ -z "${_id}" ] && _id="-"
+        if [ -n "${_switchlist}" ]; then
+            for _switch in ${_switchlist}; do
+                # get the switch type
+                config::core::get "_type" "type_${_switch}"
 
-        __config_get "_portlist" "ports_${_switch}"
-        __config_get "_vlan" "vlan_${_switch}"
-        __config_get "_nat" "nat_${_switch}"
-        __config_get "_bridge" "bridge_${_switch}"
-        __config_get "_vale" "vale_${_switch}"
-
-        if __checkyesno "${_vale}"; then
-            _type="vale"
-            __switch_vale_id "_id" "${_switch}"
-            : ${_portlist:=n/a}
-            : ${_vlan:=n/a}
-            : ${_nat:=n/a}
-        elif [ -z "${_bridge}" ]; then
-            _type="auto"
-            : ${_portlist:=-}
-            : ${_vlan:=-}
-            : ${_nat:=-}
-        else
-            _type="manual"
-            _portlist="n/a"
-            _vlan="n/a"
-            _nat="n/a"
+                case "${_type}" in
+                    netgraph) switch::netgraph::show "${_switch}" "${_format}" ;;
+                    vale)     switch::vale::show "${_switch}" "${_format}" ;;
+                    vxlan)    switch::vxlan::show "${_switch}" "${_format}" ;;
+                    manual)   switch::manual::show "${_switch}" "${_format}" ;;
+                    *)        switch::standard::show "${_switch}" "${_format}" ;;
+                esac
+            done
         fi
-
-        printf "${_format}" "${_switch}" "${_type}" "${_id}" "${_vlan}" "${_nat}" "${_portlist}"
-    done
+    } | column -ts^
 }
 
-# 'vm switch create'
 # create a new virtual switch
 #
 # @param string _switch name of the switch to create
 #
-__switch_create(){
-    local _switch="$1"
-    local _id _curr_list _curr
+switch::create(){
+    local _switch
+    local _type="standard"
+    local _list _curr _vlan _if _bridge _addr _mtu _priv
 
-    __vm_check_name "${_switch}" || __err "invalid switch name - '${_name}'"
-
-    _curr_list=$(sysrc -inqf "${vm_dir}/.config/switch" switch_list)
-
-    for _curr in $_curr_list; do
-        [ "${_switch}" = "${_curr}" ] && __err "switch ${_switch} already exists"
+    # process options
+    while getopts t:i:n:b:a:m:p _opt; do
+        case ${_opt} in
+            t) _type="${OPTARG}" ;;
+            i) _if="${OPTARG}" ;;
+            n) _vlan="${OPTARG}" ;;
+            b) _bridge="${OPTARG}" ;;
+            a) _addr="${OPTARG}" ;;
+            m) _mtu="${OPTARG}" ;;
+            p) _priv="yes" ;;
+            *) util::usage ;;
+        esac
     done
 
-    __rc_append_string ".config/switch" "switch_list" "${_switch}"
+    shift $((OPTIND - 1))
+    _switch="$1"
 
-    _id=$(ifconfig bridge create)
-    [ $? -ne 0 ] && __err "failed to create bridge interface"
-    ifconfig "${_id}" description "vm-${_switch}" up
-}
+    # check for a valid switch name
+    util::check_name "${_switch}" || util::err "invalid switch name - '${_switch}'"
 
-# 'vm switch import'
-# use an existing manually configured bridge
-#
-# @param string _switch name of the switch to create
-# @param string _bridge name of the bridge interface to import
-#
-__switch_import(){
-    local _switch="$1"
-    local _bridge="$2"
-    local _exists
+    # make sure it's not an existing name
+    config::core::get "_list" "switch_list"
 
-    [ -z "${_switch}" -o -z "${_bridge}" ] && __usage
-    __vm_check_name "${_switch}" || __err "invalid switch name - '${_name}'"
-
-    _curr_list=$(sysrc -inqf "${vm_dir}/.config/switch" switch_list)
-
-    for _curr in $_curr_list; do
-        [ "${_switch}" = "${_curr}" ] && __err "switch ${_switch} already exists"
+    for _curr in ${_list}; do
+        [ "${_switch}" = "${_curr}" ] && util::err "switch ${_switch} already exists"
     done
 
-    _exists=$(ifconfig | grep "^${_bridge}: ")
-    [ -z "${_exists}" ] && __err "${_bridge} does not appear to be a valid existing bridge"
-
-    __rc_append_string ".config/switch" "switch_list" "${_switch}"
-    sysrc -inqf "${vm_dir}/.config/switch" "bridge_${_switch}"="${_bridge}" >/dev/null 2>&1
-
-    # attach our description to bridge
-    # this will allow vm-bhyve to find it and attach guests
-    ifconfig "${_bridge}" description "vm-${_switch}" up
-}
-
-# 'vm switch destroy'
-# remove a virtual switch
-#
-# @param string _switch name of the switch
-#
-__switch_remove(){
-    local _switch="$1"
-    local _id _nat _bridge _vale
-
-    __config_load "${vm_dir}/.config/switch"
-    __config_get "_bridge" "bridge_${_switch}"
-    __config_get "_nat" "nat_${_switch}"
-    __config_get "_vale" "vale_${_switch}"
-
-    # if manual, leave it there and just remove from our config
-    if [ -n "${_bridge}" ]; then
-        ifconfig "${_bridge}" description "" >/dev/null 2>&1
-        __rc_splice_string ".config/switch" "switch_list" "${_switch}"
-        sysrc -inxqf "${vm_dir}/.config/switch" "bridge_${_switch}"
-        exit 0
-    fi	
-
-    if [ -z "${_vale}" ]; then
-        __switch_get_ident "_id" "${_switch}"
-        __switch_remove_allmembers "${_switch}" "${_id}"
-    fi
-
-    __rc_splice_string ".config/switch" "switch_list" "${_switch}"
-    sysrc -inxqf "${vm_dir}/.config/switch" "ports_${_switch}" "vlan_${_switch}" \
-        "nat_${_switch}" "vale_${_switch}" >/dev/null 2>&1
-
-    if [ -z "${_vale}" ]; then
-        if [ -n "${_id}" ]; then
-            ifconfig "${_id}" destroy >/dev/null 2>&1
-            [ $? -ne 0 ] && __warn "removed configuration but failed to remove bridge"
-        else
-            __warn "removed configuration but failed to remove bridge"
-        fi
-
-        # reset nat if this switch had nat
-        [ -n "${_nat}" ] && __switch_nat_init
-    fi
-}
-
-# 'vm switch vlan'
-# set vlan number for a switch
-# all interfaces will be removed, vlan interfaces created, then re-added
-#
-# @param string _switch name of the switch
-# @param int _vlan vlan number (or 0 to switch vlan off)
-#
-__switch_vlan(){
-    local _switch="$1"
-    local _vlan="$2"
-
-    [ -z "${_switch}" -o -z "${_vlan}" ] && __usage
-    __switch_auto "${_switch}"
-
-    echo "${_vlan}" | egrep -qs '^[0-9]{1,4}$'
-    [ $? -ne 0 ] && __err "invalid vlan number"
-    [ ${_vlan} -ge 4095 ] && __err "invalid vlan number"
-
-    # we need to remove everything and re-add as raw interfaces will
-    # change to vlan or visa-versa
-    __switch_remove_allmembers "${_switch}"
-
-    if [ "${_vlan}" = "0" ]; then
-        sysrc -inxqf "${vm_dir}/.config/switch" "vlan_${_switch}" >/dev/null 2>&1
-    else
-        sysrc -inqf "${vm_dir}/.config/switch" "vlan_${_switch}"="${_vlan}" >/dev/null 2>&1
-    fi
-
-    # put interfaces back in
-    __switch_add_allmembers "${_switch}"
-}
-
-# physically add switch member interfaces
-# this will add all configured interfaces to the bridge
-# if vlan specified, each real interface will be assigned to a vlan interface
-#
-# @private
-# @param string _switch the switch to configure
-# @param optional string _id switch id (eg bridge0) if already known
-#
-__switch_add_allmembers(){
-    local _switch="$1"
-    local _id="$2"
-    local _portlist _port
-
-    if [ -z "${_id}" ]; then
-        __switch_get_ident "_id" "${_switch}"
-        [ -z "${_id}" ] && __err "failed to get switch id while adding members"
-    fi
-
-    _portlist=$(sysrc -inqf "${vm_dir}/.config/switch" "ports_${_switch}")
-
-    if [ -n "${_portlist}" ]; then
-        for _port in ${_portlist}; do
-            __switch_configure_port "${_switch}" "${_id}" "${_port}"
-        done
-    fi
-}
-
-# physically remove all configured members from a switch
-#
-# @private
-# @param string _switch name of the switch
-# @param optional string _id switch id if already known
-#
-__switch_remove_allmembers(){
-    local _switch="$1"
-    local _id="$2"
-    local _portlist _port
-
-    if [ -z "${_id}" ]; then
-        __switch_get_ident "_id" "${_switch}"
-        [ -z "${_id}" ] && __err "failed to get switch id while removing members"
-    fi
-
-    _portlist=$(sysrc -inqf "${vm_dir}/.config/switch" "ports_${_switch}")
-
-    if [ -n "${_portlist}" ]; then
-        for _port in ${_portlist}; do
-            __switch_unconfigure_port "${_switch}" "${_id}" "${_port}"
-        done
-    fi
-}
-
-# physically adds a port to a virtual switch
-# if vlan is configured, we need to create the relevant vlan interface first,
-# then add that to the bridge
-#
-# @private
-# @param string _switch the switch to add port to
-# @param string _id the switch id
-# @param string _member name of the member interface to add
-#
-__switch_configure_port(){
-    local _switch="$1"
-    local _id="$2"
-    local _member="$3"
-    local _vlan _vid
-
-    _vlan=$(sysrc -inqf "${vm_dir}/.config/switch" "vlan_${_switch}")
-
+    # check vlan number
     if [ -n "${_vlan}" ]; then
-        __switch_get_ident "_vid" "vlan-${_member}-${_vlan}"
-
-        if [ -z "${_vid}" ]; then
-            _vid=$(ifconfig vlan create)
-            [ $? -ne 0 ] && __err "failed to create vlan interface"
-            ifconfig "${_vid}" vlandev "${_member}" vlan "${_vlan}" description "vm-vlan-${_member}-${_vlan}"
-        fi
-
-        ifconfig ${_id} addm ${_vid} >/dev/null 2>&1
-    else
-        ifconfig ${_id} addm ${_member} >/dev/null 2>&1
+        echo "${_vlan}" | egrep -qs '^[0-9]{1,4}$'
+        [ $? -eq 0 ] || util::err "invalid vlan number"
+        [ ${_vlan} -ge 4095 ] && util::err "invalid vlan number"
     fi
 
-    [ $? -ne 0 ] && __err "failed to add member to the virtual switch"
-}
-
-# 'vm switch add'
-# configure a member interface to a "switch"
-# add the interface to bridge and update switch configuration
-#
-# @param string _switch name of the switch
-# @param string _member name of the interface to add
-#
-__switch_add_member(){
-    local _switch="$1"
-    local _member="$2"
-    local _id
-
-    [ -z "${_switch}" -o -z "${_member}" ] && __usage
-
-    __switch_auto "${_switch}"
-    __switch_get_ident "_id" "${_switch}"
-    [ -z "${_id}" ] && __err "unable to locate virtual switch ${_id}"
-
-    __switch_configure_port "${_switch}" "${_id}" "${_member}"
-    __rc_append_string ".config/switch" "ports_${_switch}" "${_member}"
-}
-
-# physically remove a port from a virtual switch / bridge
-#
-# @private
-# @param string _switch the switch to update
-# @param string _id the switch id
-# @param string _member the interface to remove
-#
-__switch_unconfigure_port(){
-    local _switch="$1"
-    local _id="$2"
-    local _member="$3"
-    local _id _vlan _vid _usage
-
-    _vlan=$(sysrc -inqf "${vm_dir}/.config/switch" "vlan_${_switch}")
-
-    if [ -n "${_vlan}" ]; then
-        __switch_get_ident "_vid" "vlan-${_member}-${_vlan}"
-        [ -z "${_vid}" ] && __err "unable to find relevent vlan interface for ${_member}"
-
-        ifconfig ${_id} deletem ${_vid} >/dev/null 2>&1
-    else
-        ifconfig ${_id} deletem ${_member} >/dev/null 2>&1
+    # check address
+    if [ -n "${_addr}" ]; then
+        echo "${_addr}" | egrep -qs '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\/[0-9]{1,2}$'
+        [ $? -eq 0 ] || util::err "address must be supplied in CIDR notation (a.b.c.d/prefix-len)"
     fi
 
-    [ $? -ne 0 ] && __err "failed to remove member from the virtual switch"
-
-    # it's possible a vlan interface may be assigned to multiple switches
-    # we want to remove the vlan interface if possible, but not if it's
-    # still assigned to another switch
-    if [ -n "${_vlan}" -a -n "${_vid}" ]; then
-        _usage=$(ifconfig -a |grep "member: vm-vlan-${_member}-${_vlan}\$")
-        [ -z "${_usage}" ] && ifconfig "${_vid}" destroy >/dev/null 2>&1
+    # check mtu
+    if [ -n "${_mtu}" ]; then
+        echo "${_mtu}" | egrep -qs '^[0-9]{3,4}$'
+        [ $? -eq 0 ] || util::err "invalid mtu"
+        [ ${_mtu} -gt 9000 ] && util::err "invalid mtu"
     fi
+
+    # check switch type
+    case "${_type}" in
+        standard) switch::standard::create ;;
+        manual)   switch::manual::create ;;
+        netgraph) switch::netgraph::create ;;
+        vale)     switch::vale::create ;;
+        vxlan)    switch::vxlan::create ;;
+        *)        util::err "invalid switch type - '${_type}'" ;;
+    esac
 }
 
-# 'vm switch remove'
-# remove a member interface from a switch configuration
-# update configuration then remove device from bridge
+# destroy a switch
+# remove from configuration and unload any interfaces we created
 #
-# @param string _switch the switch to update
-# @param string _member the interface to remove
+# @param string _switch name of the switch to remove
 #
-__switch_remove_member(){
+switch::remove(){
     local _switch="$1"
-    local _member="$2"
-    local _id
+    local _type
 
-    [ -z "${_switch}" -o -z "${_member}" ] && __usage
+    [ -z "${_switch}" ] && util::usage
 
-    __switch_auto "${_switch}"
-    __switch_get_ident "_id" "${_switch}"
-    [ -z "${_id}" ] && __err "unable to locate virtual switch ${_id}"
+    # get the type of switch
+    config::core::get "_type" "type_${_switch}"
 
-    __rc_splice_string ".config/switch" "ports_${_switch}" "${_member}"
-    __switch_unconfigure_port "${_switch}" "${_id}" "${_member}"
-}
-
-# 'vm switch nat'
-# configure nat on a switch
-# this function just deals with the vm-bhyve configuration
-#
-# @param string _switch the name of the switch
-# @param string _nat=on|off whether to switch it on or off
-#
-__switch_nat(){
-    local _switch="$1"
-    local _nat="$2"
-
-    [ -z "${_switch}" ] && __usage
-    __switch_auto "${_switch}"
-
-    case "${_nat}" in
-        off)
-            sysrc -inxqf "${vm_dir}/.config/switch" "nat_${_switch}"
-            ;;
-        on)
-            if ! checkyesno pf_enable; then
-                __err "pf needs to be enabled for nat functionality"
-            fi
-
-            sysrc -inqf "${vm_dir}/.config/switch" "nat_${_switch}=yes" >/dev/null 2>&1
-            [ $? -ne 0 ] && __err "failed to store nat configuration"
-
-            echo "******"
-            echo "  NAT has been enabled on the specified switch"
-            echo "  A sample dnsmasq configuration has been created in /usr/local/etc/dnsmasq.conf.bhyve"
-            echo "  To enable DHCP on this switch, please install the dnsmasq confguration or merge with your existing."
-            echo "******"
-            ;;
-        *)
-            __err "last option should either be 'on' or 'off' to enable/disable nat functionality"
-            ;;
+    case "${_type}" in
+        standard) switch::standard::remove "${_switch}" ;;
+        manual)   switch::manual::remove "${_switch}" ;;
+        netgraph) switch::netgraph::remove "${_switch}" ;;
+        vale)     switch::vale::remove "${_switch}" ;;
+        vxlan)    switch::vxlan::remove "${_switch}" ;;
+        *)        util::err "unable to remove switch of unknown type" ;;
     esac
 
-    # reset nat configuration
-    __switch_nat_init
-}
-
-# set the system up for nat usage
-# do the actual nat work
-# we completely take over dnsmasq and assign it to the required interfaces
-# /etc/pf.conf gets an include statement added pointing to a file in
-# out .config directory. This file contains a nat rule for each nat switch
-#
-# @private
-#
-__switch_nat_init(){
-    local _pf_rules="${vm_dir}/.config/pf-nat.conf"
-    local _havenat=0
-    local _grep _switchlist _nat _net24 _if
-    local _gw _bnum
-
-    # get default gateway
-    _gw=$(netstat -4rn | grep default | awk '{print $4}')
-
-    # basic dnsmasq settings
-    echo "# vm-bhyve dhcp" > /usr/local/etc/dnsmasq.conf.bhyve
-    echo "port=0" >> /usr/local/etc/dnsmasq.conf.bhyve
-    echo "domain-needed" >> /usr/local/etc/dnsmasq.conf.bhyve
-    echo "no-resolv" >> /usr/local/etc/dnsmasq.conf.bhyve
-    echo "except-interface=lo0" >> /usr/local/etc/dnsmasq.conf.bhyve
-    echo "bind-interfaces" >> /usr/local/etc/dnsmasq.conf.bhyve
-    echo "local-service" >> /usr/local/etc/dnsmasq.conf.bhyve
-    echo "dhcp-authoritative" >> /usr/local/etc/dnsmasq.conf.bhyve
-
-    # reset our pf config and create /etc/pf.conf if needed
-    echo "# vm-bhyve nat" > "${vm_dir}/.config/pf-nat.conf"
-    [ ! -e "/etc/pf.conf" ] && touch /etc/pf.conf
-
-    # only add our include statement to /etc/pf.conf if it's not already in there somwhere
-    _grep=$(grep "${_pf_rules}" /etc/pf.conf)
-    [ -z "${_grep}" ] && echo "include \"${_pf_rules}\"" >> /etc/pf.conf
-
-    _switchlist=$(sysrc -inqf "${vm_dir}/.config/switch" switch_list)
-
-    # add each nat switch to dnsmasq.conf and .config/pf-nat.conf
-    for _switch in ${_switchlist}; do
-        _nat=$(sysrc -inqf "${vm_dir}/.config/switch" "nat_${_switch}")
-        __switch_get_ident "_id" "${_switch}"
-
-        if [ "${_nat}" = "yes" -a -n "${_id}" ]; then
-            _bnum=$(echo "${_id}" |awk -F'bridge' '{print $2}')
-            _net24="172.16.${_bnum}"
-
-            echo "nat on ${_gw} from {${_net24}.0/24} to any -> (${_gw})" >> "${vm_dir}/.config/pf-nat.conf"
-            echo "" >> /usr/local/etc/dnsmasq.conf.bhyve
-            echo "interface=${_id}" >> /usr/local/etc/dnsmasq.conf.bhyve
-            echo "dhcp-range=${_net24}.10,${_net24}.254" >> /usr/local/etc/dnsmasq.conf.bhyve
-
-            # make sure interface has an ip
-            # this doesn't get removed when nat disabled but not a major issue
-            ifconfig "${_id}" ${_net24}.1/24
-
-            _havenat="1"
-        fi
-    done
-
-    # make sure forwarding enabled if we have any nat
-    [ "${_havenat}" = "1" ] && sysctl net.inet.ip.forwarding=1 >/dev/null 2>&1
-
-    # restart services regardless
-    # still need to restart if _havenat=0, in case we've just removed last nat switch
-    __restart_service "pf"
-}
-
-# check if a switch is a vale switch
-# if so, we don't need to actually create anything.
-# interfaces are created on demand when starting guests,
-# and vale switches are dynamic
-#
-# @return int 0 is switch is vale
-#
-__switch_is_vale(){
-    local _switch="$1"
-    local _is_vale=$(sysrc -inqf "${vm_dir}/.config/switch" "vale_${_switch}")
-
-    [ -z "${_is_vale}" ] && return 1
-
-    __checkyesno "${_is_vale}"
-}
-
-# gets a unique port name for a vale interface
-# we need to make sure vale switch name is the same
-# for all interfaces on the same switch, but port is 
-# different
-# 
-# @param string _var name of variable to put port name into
-# @param string _switch the name of the switch
-# @param string _port unique port identifier (usually mac address)
-#
-__switch_vale_id(){
-    local _var="$1"
-    local _switch="$2"
-    local _port="$3"
-    local _id_s _id_p
-
-    # get a switch id
-    _id_s=$(md5 -qs "${_switch}" | awk '{print substr($0,0,5)}')
-
-    # given port?
-    if [ -n "${_port}" ]; then
-        _id_p=$(md5 -qs "${_port}" | awk '{print substr($0,0,5)}')
-        setvar "${_var}" "vale${_id_s}:${_id_p}"
+    # remove all configuration if there's no error
+    if [ $? -eq 0 ]; then
+        config::core::remove "switch_list" "${_switch}"
+        config::core::remove "ports_${_switch} vlan_${_switch} nat_${_switch} type_${_switch}"
+        config::core::remove "addr_${_switch} private_${_switch} mtu_${_switch}"
     else
-        setvar "${_var}" "vale${_id_s}"
+        util::err "failed to remove virtual switch"
     fi
+
+    # make sure the exit status indicates success,
+    # even if config::core::remove did not
+    return 0
 }
 
-# produce an error if the listed switch is not managed by vm-bhyve.
-# we allow users to import existing bridges which they configure themselves.
-# we don't allow vm-bhyve to mess with these, it's all up to the user
-# also don't mess with vale switches
+# add a new interface to a switch
 #
-# @param string _switch the name of the switch
+# @param string _switch name of the switch
+# @param string _if the interface to add
 #
-__switch_auto(){
+switch::add_member(){
     local _switch="$1"
-    local _bridge _vale
+    local _if="$2"
+    local _type
 
-    _bridge=$(sysrc -inqf "${vm_dir}/.config/switch" "bridge_${_switch}")
-    _vale=$(sysrc -inqf "${vm_dir}/.config/switch" "vale_${_switch}")
-    [ -n "${_bridge}" ] && __err "this is a manual switch that is managed outside of vm-bhyve"
-    [ -n "${_vale}" ] && __err "this command is not currently supported on vale switches"
+    [ -z "${_switch}" -o -z "${_if}" ] && util::usage
+
+    # get the type of switch
+    config::core::get "_type" "type_${_switch}"
+
+    case "${_type}" in
+        standard) switch::standard::add_member "${_switch}" "${_if}" ;;
+        manual)   switch::manual::add_member "${_switch}" "${_if}" ;;
+        netgraph) switch::netgraph::add_member "${_switch}" "${_if}" ;;
+        vale)     switch::vale::add_member "${_switch}" "${_if}" ;;
+        vxlan)    switch::vxlan::add_member "${_switch}" "${_if}" ;;
+        *)        util::err "unable to configure switch of unknown type" ;;
+    esac
 }
 
-# get the interface name for a switch
-# the id will be the name of the bridge interface (eg. "bridge0")
-# ifconfig will have "description: vm-switchname" directly below the first interface line
+# remove a member interface from a virtual switch
 #
-# @param string _var variable to put id into
-# @param string _switch the switch to look for
+# @param string _switch name of the switch
+# @param string _if the interface to remove
 #
-__switch_get_ident(){
+switch::remove_member(){
+    local _switch="$1"
+    local _if="$2"
+    local _type
+
+    [ -z "${_switch}" -o -z "${_if}" ] && util::usage
+
+    # get the type of switch
+    config::core::get "_type" "type_${_switch}"
+
+    case "${_type}" in
+        standard) switch::standard::remove_member "${_switch}" "${_if}" ;;
+        manual)   switch::manual::remove_member "${_switch}" "${_if}" ;;
+        netgraph) switch::netgraph::remove_member "${_switch}" "${_if}" ;;
+        vale)     switch::vale::remove_member "${_switch}" "${_if}" ;;
+        vxlan)    switch::vxlan::remove_member "${_switch}" "${_if}" ;;
+        *)        util::err "unable to configure switch of unknown type" ;;
+    esac
+}
+
+# change the vlan number on a virtual switch
+#
+# @param string _switch name of the switch
+# @param int _vlan the vlan number (0 to turn vlan off)
+#
+switch::vlan(){
+    local _switch="$1"
+    local _vlan="$2"
+    local _id _type
+
+    [ -z "${_switch}" -o -z "${_vlan}" ] && util::usage
+
+    switch::id "_id" "${_switch}"
+    switch::type "_type" "${_switch}"
+    [ -z "${_id}" ] && util::err "unable to locate specified virtual switch"
+
+    echo "${_vlan}" | egrep -qs '^[0-9]{1,4}$'
+    [ $? -eq 0 ] || util::err "invalid vlan number"
+    [ ${_vlan} -ge 4095 ] && util::err "invalid vlan number"
+
+    case "${_type}" in
+        standard) switch::standard::vlan "${_switch}" "${_vlan}" ;;
+        manual)   switch::manual::vlan "${_switch}" "${_vlan}" ;;
+        netgraph) switch::netgraph::vlan "${_switch}" "${_vlan}" ;;
+        vale)     switch::vale::vlan "${_switch}" "${_vlan}" ;;
+        vxlan)    switch::vxlan::vlan "${_switch}" "${_vlan}" ;;
+        *)        util::err "unable to configure switch of unknown type" ;;
+    esac
+}
+
+# enable or diable private flag on a switch
+# note that we don't update existing interfaces; this 
+# makes things easy for us and any guests booted after
+# will get the new setting
+#
+# @param string _switch the switch to update
+# @param string _priv on,yes|off,no
+#
+switch::private(){
+    local _switch="$1"
+    local _priv="$2"
+    local _type
+
+    # try to get switch type
+    [ -z "${_switch}" -o -z "${_priv}" ] && util::usage
+    switch::type "_type" "${_switch}" || util::err "specified switch does not appear to be valid"
+
+    case "${_type}" in
+        standard|manual|vxlan)
+            if util::yesno "${_priv}"; then
+                config::core::set "private_${_switch}" "yes"
+            else
+                config::core::set "private_${_switch}" "no"
+            fi
+            ;;
+        netgraph)
+            util::err "unable to configure private mode on netgraph switches"
+            ;;
+        vale)
+            util::err "unable to configure private mode on vale switches"
+            ;;
+        *)
+            util::err "unable to configure switch of unknown type"
+            ;;
+    esac
+}
+
+# enable or disable nat functionality on a virtual switch
+#
+# @param string _switch name of the switch
+# @param string _nat on|off
+#
+switch::nat(){
+    util::warn "internal nat support is currently disabled"
+    util::warn "please add an address to the virtual switch and configure your firewall for NAT manually"
+}
+
+# set or remove ip address from a virtual switch
+#
+# @param string _switch name of the switch
+# @param string _addr the ip address to add (or "none" to remove
+#
+switch::address(){
+    local _switch="$1"
+    local _addr="$2"
+    local _id _type
+
+    [ -z "${_switch}" ] && util::usage
+
+    switch::id "_id" "${_switch}"
+    switch::type "_type" "${_switch}"
+    [ -z "${_id}" ] && util::err "unable to locate specified virtual switch"
+
+    # check address
+    if [ -n "${_addr}" ] && [ "${_addr}" != "none" ]; then
+        echo "${_addr}" | egrep -qs '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\/[0-9]{1,2}$'
+        [ $? -eq 0 ] || util::err "address must be supplied in CIDR notation (a.b.c.d/prefix-len)"
+    fi
+
+    case "${_type}" in
+        standard) switch::standard::address "${_switch}" "${_addr}" ;;
+        manual)   ;&
+        netgraph) ;&
+        vale)     ;&
+        vxlan)    util::err "feature not currently supported on switches of this type" ;;
+        *)        util::err "unable to configure switch of unknown type" ;;
+    esac
+}
+
+# return the type for a switch
+#
+# @param string _var variable to put type into
+# @param string _switch the switch name
+#
+switch::type(){
     local _var="$1"
     local _switch="$2"
-    local _c_id
 
-    # search ifconfig for our switch id, and pull bridge interface name from preceeding line
-    _c_id=$(ifconfig -a | grep -B 1 "vm-${_switch}\$" | head -n 1 | awk -F: '{print $1}')
-    setvar "${_var}" "${_c_id}"
+    [ -z "${_switch}" ] && return 1
+    config::core::get "${_var}" "type_${_switch}" "standard"
+}
+
+# check if a switch is configured for private members
+#
+# @param string _switch switch name
+# @return succes (0) if it's private
+#
+switch::is_private(){
+    local _switch="$1"
+    local _priv
+
+    config::core::get "_priv" "private_${_switch}"
+    util::yesno "${_priv}"
+}
+
+# get the bridge id for a virtual switch
+#
+# @param string _var variable to put name into
+# @param string _switch the name of the switch
+#
+switch::id(){
+    local _var="$1"
+    local _switch="$2"
+    local _type
+
+    [ -z "${_switch}" ] && return 1
+
+    # get switch type
+    config::core::get "_type" "type_${_switch}"
+
+    case "${_type}" in
+        vale)     switch::vale::id "${_var}" "${_switch}" ;;
+        netgraph) switch::netgraph::id "${_var}" "${_switch}" ;;
+        manual)   switch::manual::id "${_var}" "${_switch}" ;;
+        *)        switch::standard::id "${_var}" "${_switch}" ;;
+    esac
+}
+
+# get a virt interface id for a port/switch
+#
+# @param string _var variable name to put result into
+# @param string _switch switch name to get id for
+#
+switch::__viid(){
+    local _hash=$(md5 -qs "${2}" | cut -c1-5)
+    setvar "$1" "viid-${_hash}@"
+}
+
+# retrieve interface name, given a switch name
+# we convert to viid then look for the matching group
+#
+# @param string _var variable to put interface name into
+# @param string _switch the switch name
+#
+switch::find(){
+    local _var="$1"
+    local _switch="$2"
+    local _viid _name
+
+    switch::__viid "_viid" "${_switch}"
+    _name=$(ifconfig -g "${_viid}" 2>/dev/null)
+    [ -n "${_name}" ] && setvar "${_var}" "${_name}"
+}
+
+# mark an interface with a unique viid
+# i say unique, its 5 chars from an md5 hash which
+# should be enough for half a dozen switches
+#
+# @parem string _switch switch name
+# @param string _iface interface to mark
+#
+switch::set_viid(){
+    local _switch="$1"
+    local _iface="$2"
+    local _viid
+
+    switch::__viid "_viid" "${_switch}"
+    ifconfig "${_iface}" group "${_viid}" >/dev/null 2>&1
+}
+
+# create a network interface for a guest
+# relies heavily on variables set in the main vm::run function
+#
+# @modifies _func _devices
+#
+switch::provision(){
+    local _switch _mac _type
+
+    config::get "_switch" "network${_num}_switch"
+    config::get "_mac" "network${_num}_mac"
+
+    # set a static mac if we don't have one
+    [ -z "${_mac}" ] && vm::generate_static_mac
+
+    switch::type "_type" "${_switch}"
+
+    case "${_type}" in
+        vale)     switch::vale::provision ;;
+        netgraph) switch::netgraph::provision ;;
+        standard) ;&
+        manual)   ;&
+        vxlan)    switch::standard::provision ;;
+        *)        util::warn "unable to configure interface ${_num}" ;;
+    esac
 }

lib/vm-switch-manual

@@ -0,0 +1,142 @@
+#!/bin/sh
+#-------------------------------------------------------------------------+
+# Copyright (C) 2016 Matt Churchyard (churchers@gmail.com)
+# All rights reserved
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted providing that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# for a manual switch the bridge interface should already exist
+# we just assign our description so it's visible in ifconfig
+#
+# @param string _name name of the switch
+#
+switch::manual::init(){
+    local _name="$1"
+    local _bridge="$2"
+
+    # get bridge
+    if [ -z "${_bridge}" ]; then
+        config::core::get "_bridge" "bridge_${_name}"
+        [ -z "${_bridge}" ] && return 1
+    fi
+
+    # don't rename custom bridges nor set a description. 
+    # manual interfaces are fully configured using rc.conf.
+    ifconfig "${_bridge}" group vm-switch up >/dev/null 2>&1
+    switch::set_viid "${_name}" "${_bridge}"
+}
+
+# show the configuration details for a manual switch
+#
+# @param string _name the switch name
+# @param string _format output format
+#
+switch::manual::show(){
+    local _name="$1"
+    local _format="$2"
+    local _bridge _priv
+
+    config::core::get "_bridge" "bridge_${_name}"
+    config::core::get "_priv" "private_${_name}" "no"
+    printf "${_format}" "${_name}" "manual" "${_bridge}" "n/a" "${_priv}" "n/a" "n/a" "n/a"
+}
+
+# create a manual switch
+# we just assign our switch name to the existing bridge
+#
+switch::manual::create(){
+
+    # we need to have a bridge
+    [ -z "${_bridge}" ] && util::err "you must specify a bridge to import when creating a manual switch"
+
+    # check we can find this bridge on the system
+    ifconfig "${_bridge}" >/dev/null 2>&1
+    [ $? -eq 0 ] || util::err "${_bridge} does not appear to be a valid existing bridge"
+
+    # store configuration
+    config::core::set "switch_list" "${_switch}" "1"
+    config::core::set "type_${_switch}" "manual"
+    config::core::set "bridge_${_switch}" "${_bridge}"
+
+    [ -n "${_priv}" ] && config::core::set "private_${_switch}" "${_priv}"
+
+    # import
+    switch::manual::init "${_switch}" "${_bridge}"
+}
+
+# remove a manual switch
+#
+# @param string _switch the name of the switch
+#
+switch::manual::remove(){
+    local _switch="$1"
+    local _id
+
+    switch::manual::id "_id" "${_switch}"
+    [ -z "${_id}" ] && return 0
+
+    # try to remove our description
+    # viid stays but it's not worth the extra hassle to remove that
+    ifconfig ${_id} -descr -group vm-switch >/dev/null 2>&1
+}
+
+# add a new interface to this switch
+# manual switches should be managed by the user
+# using rc.conf, hence "manual"
+#
+# @param string _switch name of the switch
+# @param string _if the interface to add
+#
+switch::manual::add_member(){
+    util::err "manual switches and member interfaces should be configured using /etc/rc.conf"
+}
+
+# remove an interface
+# manual switches should be managed by the user
+# using rc.conf, hence "manual"
+#
+# @param string _switch name of the switch
+# @param string _if the interface to remove
+#
+switch::manual::remove_member(){
+    util::err "manual switches and member interfaces should be configured using /etc/rc.conf"
+}
+
+# set vlan id
+#
+# @param string _switch name of switch
+# @param int _vlan vlan id to set
+#
+switch::manual::vlan(){
+    util::err "manual switches and member interfaces should be configured using /etc/rc.conf"
+}
+
+# get id for a manual switch
+# in this case we need to return the bridge name
+#
+# @param string _var variable to put id into
+# @param string _switch switch name
+# @return 0 if switch id found
+#
+switch::manual::id(){
+    config::core::get "$1" "bridge_$2"
+}

lib/vm-switch-netgraph

@@ -0,0 +1,119 @@
+#!/bin/sh
+#-------------------------------------------------------------------------+
+# Copyright (C) 2021 Benoit Chesneau (bchesneau@gmail.com)
+# All rights reserved
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted providing that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+
+# show the configuration details for a netgraph switch
+#
+# @param string _name the switch name
+# @param string _format output format
+#
+switch::netgraph::show(){
+    local _name="$1"
+    local _format="$2"
+    local _id
+
+    switch::netgraph::id "_id" "${_name}"
+    printf "${_format}" "${_name}" "netraph" "${_id}" "n/a" "n/a" "n/a" "n/a" "n/a"
+}
+
+# create a netgraph switch
+#
+# @param string _switch the name of the switch
+#
+switch::netgraph::create(){
+    config::core::set "switch_list" "${_switch}" "1"
+    config::core::set "type_${_switch}" "netgraph"
+}
+
+# remove a netgraph switch
+#
+switch::netgraph::remove(){ }
+
+# add a new interface to this switch
+# at the moment we require the user to manually
+# set up any netgraph switches
+#
+# @param string _switch name of the switch
+# @param string _if the interface to add
+#
+switch::netgraph::add_member(){
+    util::err "physical interfaces must be added to the netgraph switch manually"
+}
+
+# remove an interface
+#
+# @param string _switch name of the switch
+# @param string _if the interface to remove
+#
+switch::netgraph::remove_member(){
+    util::err "physical interfaces must be removed from the netgraph switch manually"
+}
+
+# set vlan id
+#
+# @param string _switch name of switch
+# @param int _vlan vlan id to set
+#
+switch::netgraph::vlan(){
+    util::err "vlan support is not currently implemented for netgraph switches"
+}
+# gets a unique linkname name for a ng_bridge interface
+# we need to make sure the link is unique and the last one
+#
+# @param string _var name of variable to put port name into
+# @param string _switch the name of the switch
+#
+switch::netgraph::id(){
+    local _var="$1"
+    local _switch="$2"
+
+    # Create a new interface to the bridge
+    num=2
+    while ngctl msg "${_switch}:" getstats $num > /dev/null 2>&1
+    do
+        num=$(( $num + 1 ))
+    done
+    setvar "${_var}" "netgraph,path=${_switch}:,peerhook=link$num"
+}
+
+# create a netgraph interface for a guest
+# relies heavily on variables set in the main vm::run function
+#
+# @modifies _func _devices
+# @return 1 if we don't get a tap device
+#
+switch::netgraph::provision(){
+    local _ngid
+
+    # create a netgraph peer
+    switch::netgraph::id "_ngid" "${_switch}"
+
+    util::log "guest" "${_name}" "adding netgraph interface ${_ngid} (${_switch})"
+    _devices="${_devices} -s ${_bus}:${_slot}:${_func},${_emulation},${_ngid}"
+    [ -n "${_mac}" ] && _devices="${_devices},mac=${_mac}"
+
+     _func=$((_func + 1))
+}

lib/vm-switch-standard

@@ -0,0 +1,405 @@
+#!/bin/sh
+#-------------------------------------------------------------------------+
+# Copyright (C) 2016 Matt Churchyard (churchers@gmail.com)
+# All rights reserved
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted providing that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# creaate bridge interface for a standard switch
+#
+# @param string _name name of the switch
+#
+switch::standard::init(){
+    local _name="$1"
+    local _id _addr _mtu _len _ifconf
+
+    # see if it already exists
+    switch::standard::id "_id" "${_name}" && return 0
+
+    # get the length of the switch name
+    # it's useful for other utilities to use switch name as interface name
+    # as it's static. can't do that if it's > 12 chars
+    _len=$(echo -n "${_name}" | wc -m)
+
+    if [ ${_len} -le 12 ]; then
+      _ifconf="name vm-${_name}"
+    else
+      _ifconf="descr vm/${_name}"
+    fi
+
+    # create a bridge for this switch
+    _id=$(ifconfig bridge create ${_ifconf} group vm-switch up 2>/dev/null)
+    [ $? -eq 0 ] || util::err "failed to create bridge interface for switch ${_name}"
+
+    switch::set_viid "${_name}" "${_id}"
+
+    # randomise mac if feature is available
+    [ ${VERSION_BSD} -ge 1102000 ] && ifconfig "${_id}" link random
+
+    # try to set ip address
+    config::core::get "_addr" "addr_${_name}"
+    [ -n "${_addr}" ] && ifconfig "${_id}" inet ${_addr} 2>/dev/null
+    config::core::get "_addr" "addr6_${_name}"
+    [ -n "${_addr}" ] && ifconfig "${_id}" inet6 ${_addr} 2>/dev/null
+
+    # custom mtu?
+    config::core::get "_mtu" "mtu_${_name}"
+    [ -n "${_mtu}" ] && ifconfig "${_id}" mtu ${_mtu}
+
+    # add member interfaces
+    switch::standard::__add_members "${_name}" "${_id}"
+}
+
+# show the configuration details for a switch
+#
+# @param string _name the switch name
+# @param string _format output format
+#
+switch::standard::show(){
+    local _name="$1"
+    local _format="$2"
+    local _id _vlan _ports _addr _mtu _priv
+
+    switch::find "_id" "${_name}"
+    config::core::get "_ports" "ports_${_name}"
+    config::core::get "_vlan" "vlan_${_name}"
+    config::core::get "_addr" "addr_${_name}"
+    config::core::get "_mtu" "mtu_${_name}"
+    config::core::get "_priv" "private_${_name}" "no"
+
+    printf "${_format}" "${_name}" "standard" "${_id:--}" "${_addr:--}" "${_priv}" "${_mtu:--}" \
+      "${_vlan:--}" "${_ports:--}"
+}
+
+# create a standard virtual switch
+#
+switch::standard::create(){
+
+    # store configuration
+    config::core::set "switch_list" "${_switch}" "1"
+    config::core::set "type_${_switch}" "standard"
+
+    [ -n "${_if}" ] && config::core::set "ports_${_switch}" "${_if}"
+    [ -n "${_vlan}" ] && config::core::set "vlan_${_switch}" "${_vlan}"
+    [ -n "${_addr}" ] && config::core::set "addr_${_switch}" "${_addr}"
+    [ -n "${_priv}" ] && config::core::set "private_${_switch}" "${_priv}"
+    [ -n "${_mtu}" ] && config::core::set "mtu_${_switch}" "${_mtu}"
+
+    config::core::load
+    switch::standard::init "${_switch}"
+}
+
+# destroy a standard switch
+#
+# @param string _switch name of the switch to destroy
+#
+switch::standard::remove(){
+    local _switch="$1"
+    local _id
+
+    # get the bridge id
+    switch::standard::id "_id" "${_switch}"
+    [ $? -eq 0 ] || return 1
+
+    # remove all member interfaces
+    switch::standard::__remove_members "${_switch}" "${_id}"
+
+    # destroy the bridge
+    ifconfig "${_id}" destroy >/dev/null 2>&1
+}
+
+# add a new interface to this switch
+#
+# @param string _switch name of the switch
+# @param string _if the interface to add
+#
+switch::standard::add_member(){
+    local _switch="$1"
+    local _if="$2"
+    local _id _vlan _mtu
+    
+    switch::standard::id "_id" "${_switch}" || util::err "unable to locate switch id"
+    config::core::get "_vlan" "vlan_${_switch}"
+    config::core::get "_mtu" "mtu_${_switch}"
+    switch::standard::__configure_port "${_switch}" "${_id}" "${_if}" "${_vlan}" "${_mtu}"
+    config::core::set "ports_${_switch}" "${_if}" "1"
+}
+
+# remove a member interface from this switch
+#
+# @param string _switch name of the switch
+# @param string _if the interface to remove
+#
+switch::standard::remove_member(){
+    local _switch="$1"
+    local _if="$2"
+    local _id _vlan
+
+    switch::standard::id "_id" "${_switch}" || util::err "unable to locate switch id"
+    config::core::remove "ports_${_switch}" "${_if}"
+    config::core::get "_vlan" "vlan_${_switch}"
+    switch::standard::__unconfigure_port "${_switch}" "${_id}" "${_if}" "${_vlan}"
+}
+
+# set vlan id
+#
+# @param string _switch name of switch
+# @param int _vlan vlan id to set
+#
+switch::standard::vlan(){
+    local _switch="$1"
+    local _vlan="$2"
+    local _id
+
+    switch::standard::id "_id" "${_switch}" || util::err "unable to locate switch id"
+    switch::standard::__remove_members "${_switch}" "${_id}"
+
+    # update configuration
+    if [ "${_vlan}" = "0" ]; then
+        config::core::remove "vlan_${_switch}"
+    else
+        config::core::set "vlan_${_switch}" "${_vlan}"
+    fi
+
+    config::core::load
+    switch::standard::__add_members "${_switch}" "${_id}"
+}
+
+# set or remove ip address
+#
+# @param string _swtich name of the switch
+# @param string _addr address or "none"
+# @scope _id switch if from parent switch::address
+#
+switch::standard::address(){
+    local _switch="$1"
+    local _addr="$2"
+    local _curr
+
+    if [ "${_addr}" = "none" ]; then
+
+        config::core::get "_curr" "addr_${_switch}"
+        [ $? -eq 0 ] || util::err "unable to locate an existing address for this switch"
+
+        config::core::remove "addr_${_switch}"
+        ifconfig "${_id}" "${_curr}" delete
+    else
+
+        config::core::set "addr_${_switch}" "${_addr}"
+        ifconfig "${_id}" "${_addr}"
+    fi
+}
+
+# add all member interfaces to a switch
+#
+# @param string _switch the name of the switch
+# @param string _id interface id for the switch
+#
+switch::standard::__add_members(){
+    local _switch="$1"
+    local _id="$2"
+    local _ports _vlan _port _mtu
+
+    # get the id if not provided
+    if [ -z "${_id}" ]; then
+        switch::standard::id "_id" "${_switch}" || util:err "failed to get switch id while adding members"
+    fi
+
+    config::core::get "_ports" "ports_${_switch}"
+    config::core::get "_vlan" "vlan_${_switch}"
+    config::core::get "_mtu" "mtu_${_switch}"
+
+    if [ -n "${_ports}" ]; then
+        for _port in ${_ports}; do
+            switch::standard::__configure_port "${_switch}" "${_id}" "${_port}" "${_vlan}" "${_mtu}"
+        done
+    fi
+}
+
+# remove member interfaces from a switch
+#
+# @param string _switch the name of the switch
+# @param string _id bridge id if already known
+#
+switch::standard::__remove_members(){
+    local _switch="$1"
+    local _id="$2"
+    local _ports _port _vlan
+
+    # get id if not given to us
+    if [ -z "${_id}" ]; then
+        switch::standard::id "_id" "${_switch}"
+        [ $? -eq 0 ] || util::err "failed to get switch id while removing members"
+    fi
+
+    # get full port list
+    config::core::get "_ports" "ports_${_switch}"
+    config::core::get "_vlan" "vlan_${_switch}"
+
+    if [ -n "${_ports}" ]; then
+        for _port in ${_ports}; do
+           switch::standard::__unconfigure_port "${_switch}" "${_id}" "${_port}" "${_vlan}"
+        done
+    fi
+}
+
+# configure a local port for our bridge
+#
+# @param string _switch the switch to add port to
+# @param string _id the bridge id of the switch
+# @param string _port the interface to add 
+# @param int _vlan vlan number if assigned to this switch
+# @param int _mtu custom mtu to use for this port
+#
+switch::standard::__configure_port(){
+    local _switch="$1"
+    local _id="$2"
+    local _port="$3"
+    local _vlan="$4"
+    local _mtu="$5"
+    local _viface _vname
+
+    # try to set mtu of port?
+    [ -n "${_mtu}" ] && ifconfig "${_port}" mtu ${_mtu} >/dev/null 2>&1
+
+    # vlan enabled?
+    if [ -n "${_vlan}" ]; then
+
+        # see if vlan interface already exists
+        _vname="${_port}.${_vlan}"
+        switch::standard::id "_viface" "${_vname}"
+
+        # create if needed
+        if [ $? -ne 0 ]; then
+            # use our id as the interface name here.
+            # it should always be a valid name and interface.vlan-id is much easier to understand in ifconfig
+            # than a bunch of vlanX interfaces
+            _viface=$(ifconfig vlan create vlandev "${_port}" vlan "${_vlan}" descr "vm-vlan/${_switch}/${_vname}" name "${_vname}" group vm-vlan up 2>/dev/null)
+            [ $? -eq 0 ] || util::err "failed to create vlan interface for port ${_port} on switch ${_switch}"
+        fi
+
+        switch::set_viid "${_vname}" "${_viface}"
+        ifconfig ${_id} addm ${_viface} >/dev/null 2>&1
+    else
+        # add to bridge, nice and simple :)
+        ifconfig ${_id} addm ${_port} >/dev/null 2>&1
+    fi
+
+    [ $? -eq 0 ] || util::err "failed to add member ${_port} to the virtual switch ${_switch}"
+}
+
+# unconfigure a local port
+#
+# @param string _switch the switch to remove port from
+# @param string _id the bridge id of the switch
+# @param string _port the interface to remove
+# @param string _vlan vlan number if assigned to this switch
+#
+switch::standard::__unconfigure_port(){
+    local _switch="$1"
+    local _id="$2"
+    local _port="$3"
+    local _vlan="$4"
+    local _vid
+
+    if [ -n "${_vlan}" ]; then
+        # get vlan interface
+        switch::standard::id "_vid" "${_port}.${_vlan}"
+
+        # remove the vlan interface, it will be removed from bridge automatically
+        [ $? -eq 0 ] && ifconfig ${_vid} destroy >/dev/null 2>&1
+    else
+        ifconfig ${_id} deletem ${_port} >/dev/null 2>&1
+    fi
+}
+
+# get the id for a standard switch
+# this returns the associated bridge name
+#
+# @param string _var variable to put id into
+# @param string _switch the switch to look for
+# @return 0 on success
+#
+switch::standard::id(){
+    switch::find "$1" "$2"
+}
+
+# creates a standard tap interface for a guest
+# relies heavily on variables set in the main vm::run function
+#
+# @modifies _func _devices
+# @return 1 if we don't get a tap device
+#
+switch::standard::provision(){
+    local _tap _custom_tap _sid _mtu _member_type _iname
+
+    config::get "_custom_tap" "network${_num}_device"
+    config::get "_iname" "network${_num}_name"
+
+    # create interface
+    if [ -n "${_custom_tap}" ]; then
+        _tap="${_custom_tap}"
+    elif [ -n "${_iname}" ]; then
+        _tap=$(ifconfig tap create name "${_iname}")
+    else
+        _tap=$(ifconfig tap create)
+    fi
+
+    [ -z "${_tap}" ] && return 1;
+
+    util::log "guest" "${_name}" "initialising network device ${_tap}"
+    ifconfig "${_tap}" descr "vmnet/${_name}/${_num}/${_switch:-custom}" group vm-port >/dev/null 2>&1
+
+    if [ -n "${_switch}" ]; then
+        switch::id "_sid" "${_switch}"
+
+        # should this be a span member?
+        _member_type="addm"
+        config::yesno "network${_num}_span" && _member_type="span"
+
+        if [ -n "${_sid}" ]; then
+            _mtu=$(ifconfig "${_sid}" | head -n1 | awk '{print $NF}')
+
+            if [ "${_mtu}" != "1500" ]; then
+                util::log "guest" "${_name}" "setting mtu of ${_tap} to ${_mtu}"
+                ifconfig "${_tap}" mtu "${_mtu}" >/dev/null 2>&1
+            fi
+
+            util::log "guest" "${_name}" "adding ${_tap} -> ${_sid} (${_switch} ${_member_type})"
+            ifconfig "${_sid}" "${_member_type}" "${_tap}" >/dev/null 2>&1 || util::log "guest" "${_name}" "failed to add ${_tap} to ${_sid}"
+
+            util::log "guest" "${_name}" "bring up ${_tap} -> ${_sid} (${_switch} ${_member_type})"
+            ifconfig "${_tap}" up >/dev/null 2>&1 || util::log "guest" "${_name}" "failed to bring up ${_tap} in ${_sid}"
+
+            # set private if configured
+            switch::is_private "${_switch}" && ifconfig "${_sid}" "private" "${_tap}" >/dev/null 2>&1
+        else
+            util::log "guest" "${_name}" "failed to find virtual switch '${_switch}'"
+        fi
+    fi
+
+    _devices="${_devices} -s ${_bus}:${_slot}:${_func},${_emulation},${_tap}"
+    [ -n "${_mac}" ] && _devices="${_devices},mac=${_mac}"
+
+    _func=$((_func + 1))
+    [ -z "${_custom_tap}" ] && _taplist="${_taplist}${_taplist:+ }${_tap}"
+}

lib/vm-switch-vale

@@ -0,0 +1,128 @@
+#!/bin/sh
+#-------------------------------------------------------------------------+
+# Copyright (C) 2016 Matt Churchyard (churchers@gmail.com)
+# All rights reserved
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted providing that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# show the configuration details for a vale switch
+#
+# @param string _name the switch name
+# @param string _format output format
+#
+switch::vale::show(){
+    local _name="$1"
+    local _format="$2"
+    local _id
+
+    switch::vale::id "_id" "${_name}"
+    printf "${_format}" "${_name}" "vale" "${_id}" "n/a" "n/a" "n/a" "n/a" "n/a"
+}
+
+# create a vale switch
+#
+# @param string _switch the name of the switch
+#
+switch::vale::create(){
+
+    config::core::set "switch_list" "${_switch}" "1"
+    config::core::set "type_${_switch}" "vale"
+}
+
+# remove a vale switch
+#
+switch::vale::remove(){ }
+
+# add a new interface to this switch
+# at the moment we require the user to manually
+# set up any vale switches
+#
+# @param string _switch name of the switch
+# @param string _if the interface to add
+#
+switch::vale::add_member(){
+    util::err "physical interfaces must be added to the vale switch manually"
+}
+
+# remove an interface
+#
+# @param string _switch name of the switch
+# @param string _if the interface to remove
+#
+switch::vale::remove_member(){
+    util::err "physical interfaces must be removed from the vale switch manually"
+}
+
+# set vlan id
+#
+# @param string _switch name of switch
+# @param int _vlan vlan id to set
+#
+switch::vale::vlan(){
+    util::err "vlan support is not currently implemented for vale switches"
+}
+
+# gets a unique port name for a vale interface
+# we need to make sure vale switch name is the same
+# for all interfaces on the same switch, but port is
+# different
+#
+# @param string _var name of variable to put port name into
+# @param string _switch the name of the switch
+# @param string _port unique port identifier (usually mac address)
+#
+switch::vale::id(){
+    local _var="$1"
+    local _switch="$2"
+    local _port="$3"
+    local _id_s _id_p
+
+    # get a switch id
+    _id_s=$(md5 -qs "${_switch}" | cut -c1-5)
+
+    # given port?
+    if [ -n "${_port}" ]; then
+        _id_p=$(md5 -qs "${_port}" | cut -c1-5)
+        setvar "${_var}" "vale${_id_s}:${_id_p}"
+    else
+        setvar "${_var}" "vale${_id_s}"
+    fi
+}
+
+# create a vale interface for a guest
+# relies heavily on variables set in the main vm::run function
+#
+# @modifies _func _devices
+# @return 1 if we don't get a tap device
+#
+switch::vale::provision(){
+    local _vale_id
+
+    # create a vale port id
+    switch::vale::id "_vale_id" "${_switch}" "${_mac}"
+
+    util::log "guest" "${_name}" "adding vale interface ${_tap} (${_switch})"
+    _devices="${_devices} -s ${_bus}:${_slot}:${_func},${_emulation},${_vale_id}"
+    [ -n "${_mac}" ] && _devices="${_devices},mac=${_mac}"
+
+    _func=$((_func + 1))
+}

lib/vm-switch-vxlan

@@ -0,0 +1,204 @@
+#!/bin/sh
+#-------------------------------------------------------------------------+
+# Copyright (C) 2016 Matt Churchyard (churchers@gmail.com)
+# All rights reserved
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted providing that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# create a vxlan switch
+# we create a bridge and then add the vxlan interface to it
+#
+# @param string _name name of the switch
+#
+switch::vxlan::init(){
+    local _name="$1"
+    local _id _vlan _if _maddr _addr _ifconf
+
+    # see if the bridge already exists
+    switch::standard::id "_id" "${_name}" && return 0
+
+    # need a vlan id and interface
+    config::core::get "_vlan" "vlan_${_name}"
+    config::core::get "_if" "ports_${_name}"
+    [ -z "${_vlan}" -o -z "${_if}" ] && return 1
+
+    # get local address for this interface
+    _local=$(ifconfig ${_if} | grep "inet " | cut -d" " -f 2)
+    [ -z "${_local}" ] && return 1
+
+    # come up with an ip address for multicast data
+    switch::vxlan::__multicast "_maddr" "${_name}"
+
+    # create the vxlan interface
+    ifconfig "vxlan${_vlan}" create vxlanid "${_vlan}" vxlanlocal "${_local}" vxlangroup "${_maddr}" \
+      vxlandev "${_if}" descr "vm-vxlan/${_switch}" group vm-vlan up >/dev/null 2>&1
+    [ $? -eq 0 ] || return 1
+
+    # get the length of the switch name
+    # it's useful for other utilities to use switch name as interface name
+    # as it's static. can't do that if it's > 12 chars
+    _len=$(echo -n "${_name}" | wc -m)
+
+    if [ ${_len} -le 12 ]; then
+      _ifconf="name vm-${_name}"
+    else
+      _ifconf="descr vm/${_name}"
+    fi
+
+    # create a bridge for this switch
+    _id=$(ifconfig bridge create ${_ifconf} group vm-switch up 2>/dev/null)
+    [ $? -eq 0 ] || util::err "failed to create bridge interface for switch ${_name}"
+
+    switch::set_viid "${_name}" "${_id}"
+
+    # randomise mac if feature is available
+    [ ${VERSION_BSD} -ge 1102000 ] && ifconfig "${_id}" link random
+
+    # bridge vxlan to our guest switch
+    # static route traffic for this multicast address via our specified interface
+    ifconfig "${_id}" addm "vxlan${_vlan}"
+    route add -net ${_maddr}/32 -iface ${_if} >/dev/null 2>&1
+
+    # custom address for bridge?
+    config::core::get "_addr" "addr_${_name}"
+    [ -n "${_addr}" ] && ifconfig "${_id}" inet ${_addr}
+}
+
+# show the configuration details for a vxlan switch
+#
+# @param string _name the switch name
+# @param string _format output format
+#
+switch::vxlan::show(){
+    local _name="$1"
+    local _format="$2"
+    local _id _vlan _port _addr _priv
+
+    switch::standard::id "_id" "${_name}"
+    config::core::get "_vlan" "vlan_${_name}"
+    config::core::get "_port" "ports_${_name}"
+    config::core::get "_addr" "addr_${_name}"
+    config::core::get "_priv" "private_${_name}" "no"
+
+    printf "${_format}" "${_name}" "vxlan" "${_id:--}" "${_addr:--}" "${_priv}" "n/a" "${_vlan}" "${_port}"
+}
+
+# create a vxlan switch
+#
+switch::vxlan::create(){
+
+    # we must have an interface and vlan to use
+    [ -z "${_if}" -o -z "${_vlan}" ] && util::err "vxlan switches must be created with an interface and vlan id specified"
+
+    # store configuration
+    config::core::set "switch_list" "${_switch}" "1"
+    config::core::set "type_${_switch}" "vxlan"
+    config::core::set "vlan_${_switch}" "${_vlan}"
+    config::core::set "ports_${_switch}" "${_if}"
+
+    [ -n "${_addr}" ] && config::core::set "addr_${_switch}" "${_addr}"
+    [ -n "${_priv}" ] && config::core::set "private_${_switch}" "${_priv}"
+
+    config::core::load
+    switch::vxlan::init "${_switch}"
+}
+
+# destroy a vxlan switch
+#
+# @param string _switch the switch to remove
+#
+switch::vxlan::remove(){
+    local _switch="$1"
+    local _id _vlan _maddr
+    
+    # try to get guest bridge and vxlan id
+    switch::standard::id "_id" "${_switch}"
+    [ $? -eq 0 ] || return 1
+
+    config::core::get "_vlan" "vlan_${_switch}"
+    [ -z "${_vlan}" ] && return 1
+
+    # get the multicast address we used for this switch
+    # and try to remove any route we may have added
+    switch::vxlan::__multicast "_maddr" "${_switch}"
+    route del -net "${_maddr}/32" >/dev/null 2>&1
+
+    # destroy the guest bridge
+    ifconfig ${_id} destroy >/dev/null 2>&1
+    [ $? -eq 0 ] || return 1
+
+    # destroy the vxlan
+    ifconfig "vxlan${_vlan}" destroy >/dev/null 2>&1
+}
+
+# add a new interface to this switch
+# we only allow a single physical interface for
+# vxlan switches. this must be set at creation time
+# so this just reports an error
+#
+# @param string _switch name of the vxlan switch
+# @param string _if the interface to add
+#
+switch::vxlan::add_member(){
+    util::err "vxlan interface must be configured at creation time"
+}
+
+# remove an interface from a switch
+# we don't support this here
+#
+# @param string _switch name of the switch
+# @param string _if the interface to remove
+#
+switch::vxlan::remove_member(){
+    util::err "vxlan interface must be configured at creation time"
+}
+
+# set vlan id
+#
+# @param string _switch name of switch
+# @param int _vlan vlan id to set
+#
+switch::vxlan::vlan(){
+    util::err "vxlan id can only be set at creation time"
+}
+
+# get the multicast address for a vxlan switch
+#
+# @param string _var variable to put address into
+# @param string _switch the switch name
+#
+switch::vxlan::__multicast(){
+    local _var="$1"
+    local _switch="$2"
+    local _hash _l_addr _octet _pos
+
+    # come up with an ip address for multicast data
+    _hash=$(md5 -qs "${_switch}")
+    _l_addr="239"
+
+    for _pos in 1-2 3-4 5-6; do
+        _octet=$(printf ".%d" "0x`echo "${_hash}"| cut -c ${_pos}`")
+        _l_addr="${_l_addr}${_octet}"
+    done
+
+    setvar "${_var}" "${_l_addr}"
+}

lib/vm-util

@@ -0,0 +1,405 @@
+#!/bin/sh
+#-------------------------------------------------------------------------+
+# Copyright (C) 2016 Matt Churchyard (churchers@gmail.com)
+# All rights reserved
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted providing that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# make sure we have the right environment
+#
+util::setup(){
+    util::load_module "nmdm"
+    util::load_module "if_bridge"
+
+    # tap(4) & tun(4) were unified in r347241, this is closest ABI bump
+    if [ `uname -K` -ge 1300029 ]; then
+        util::load_module "if_tuntap"
+    else
+        util::load_module "if_tap"
+    fi
+
+    sysctl net.link.tap.up_on_open=1 >/dev/null 2>&1
+
+    # do we have the default template example, but no default in our .templates?
+    # if so, get a copy, this at least allows a simple "vm create" to work out of the box
+    if [ -e "/usr/local/share/examples/vm-bhyve/default.conf" -a ! -e "${vm_dir}/.templates/default.conf" ]; then
+        cp "/usr/local/share/examples/vm-bhyve/default.conf" "${vm_dir}/.templates/" >/dev/null 2>&1
+    fi
+}
+
+# load a kernel module
+#
+# @param string _mod the module name
+#
+util::load_module(){
+    local _mod="$1"
+    kldstat -qm ${_mod} >/dev/null 2>&1
+    if [ $? -ne 0 ]; then
+        kldload ${_mod} >/dev/null 2>&1
+        [ $? -eq 0 ] || util::err "unable to load ${_mod}.ko!"
+    fi
+}
+
+# check if system have bhyve support
+# look for sysctls set by the vmm module. I can't get confirmation that this
+# is a valid way to check vmm is working, even though it seems reasonable.
+# the vm_disable_host_checks="yes" rc settings allows bypassing all this
+# if your system should be supported but these checks break.
+# 
+# @modifies VM_NO_UG
+#
+util::check_bhyve_support(){
+
+    # almost all our functionality requires access to things only root can do
+    [ `id -u` -ne 0 ] && util::err "virtual machines can only be managed by root"
+
+    # try to load the vmm module
+    # we do this here to make sure the sysctls exist, and before disable_host_checks
+    # as we want this loaded anyway. FreeBSD version check removed as the
+    # module won't exist on systems too old for bhyve
+    util::load_module "vmm"
+
+    # don't check if user wants to bypass host checks
+    util::yesno "$vm_disable_host_checks" && return 0
+
+    # check sysctls
+    # these only work for intel
+    # for AMD we give up trying to check for the time being
+    sysctl hw.model |grep Intel >/dev/null 2>&1
+
+    if [ $? -eq 0 ]; then
+        [ "`sysctl -n hw.vmm.vmx.initialized 2>/dev/null`" != "1" ] && util::err "kernel vmm not initialised (no VT-x / AMD SVM cpu support?)"
+        [ "`sysctl -n hw.vmm.vmx.cap.unrestricted_guest 2>/dev/null`" != "1" ] && VM_NO_UG="1"
+    fi
+}
+
+# check for passthru support
+# following neel@ wiki we search for DMAR acpi table for vt-d
+# and we check sysctl if amdvi is present and enabled
+#
+# @return success if host has vt-d or amdvi
+#
+util::check_bhyve_iommu(){
+    local _vtd
+    local _amdvi
+
+    # don't check if user wants to bypass host checks
+    # think this check is fairly solid but there's probably someone somewhere
+    # with iommu support that our tests fail for.
+    util::yesno "$vm_disable_host_checks" && return 0
+
+    _vtd=$(acpidump -t |grep DMAR)
+    _amdvi=$(sysctl hw |grep 'vmm.amdvi.enable: 1')
+    [ -z "${_vtd}" -a -z "${_amdvi}" ] && return 1
+
+    return 0
+}
+
+# restart a local service
+# checks if service is running and either starts or restarts
+#
+# @param string _serv the name of the service
+#
+util::restart_service(){
+    local _serv="$1"
+    local _cmd="restart"
+
+    # see if it's actually running
+    service ${_serv} status >/dev/null 2>&1
+    [ $? -ne 0 ] && _cmd="start"
+
+    service ${_serv} ${_cmd} >/dev/null 2>&1
+    [ $? -ne 0 ] && util::warn "failed to ${_cmd} service ${_serv}"
+}
+
+# show version
+#
+util::version(){
+    echo "vm-bhyve: Bhyve virtual machine management v${VERSION} (rev. ${VERSION_INT})"
+}
+
+# show version & usage information
+# we exit after running this
+#
+util::usage(){
+    util::version
+    cat << EOT
+Usage: vm ...
+    version
+    init
+    set [setting=value] [...]
+    get [all|setting] [...]
+    switch list
+    switch info [name] [...]
+    switch create [-t type] [-i interface] [-n vlan-id] [-m mtu] [-a address/prefix-len] [-b bridge] [-p] <name>
+    switch vlan <name> <vlan|0>
+    switch nat <name> <on|off>
+    switch private <name> <on|off>
+    switch add <name> <interface>
+    switch remove <name> <interface>
+    switch destroy <name>
+    datastore list
+    datastore add <name> <spec>
+    datastore remove <name>
+    datastore add <name> <path>
+    list [-r]
+    info [name] [...]
+    create [-d datastore] [-t template] [-s size] [-m memory] [-c vCPUs] [-i vm-image] [-C -k pubkeys] <name>
+    install [-fi] <name> <iso>
+    start [-fi] <name> [...]
+    stop <name> [...]
+    restart <name>
+    console <name> [com1|com2]
+    configure <name>
+    rename <name> <new-name>
+    add [-d device] [-t type] [-s size|switch] <name>
+    startall
+    stopall
+    reset  [-f] <name>
+    poweroff [-f] <name>
+    destroy [-f] <name>
+    passthru
+    clone <name[@snapshot]> <new-name>
+    snapshot [-f] <name[@snapshot]>
+    rollback [-r] <name@snapshot>
+    iso [url]
+    img [url]
+    image list
+    image create [-d description] [-u] <name>
+    image destroy <uuid>
+    image provision [-d datastore] <uuid> <newname>
+EOT
+    exit 1
+}
+
+# err
+# display an error message and exit immediately
+#
+# @param string - the message to display
+#
+util::err(){
+    echo "${0}: ERROR: $1" >&2
+    exit 1
+}
+
+# err_inline
+# display an error inline with informational output
+#
+# @param string - message to display
+#
+util::err_inline(){
+    echo "  ! $1"
+    exit 1
+}
+
+# warn
+# display warning, but do not exit
+#
+# @param string - the message to display
+#
+util::warn(){
+    echo "${0}: WARNING: $1" >&2
+}
+
+# log_rotate
+# simple rotation of log files
+# if we hit 1MB, which should cover a fair amount of history,
+# we move existing log and and create a new one.
+# one keep 1 previous file, as that should be enough
+#
+# @param string _type whether to rotate guest or main log
+#
+util::log_rotate(){
+    local _type="$1"
+    local _lf="vm-bhyve.log"
+    local _file _size _guest
+
+    case "${_type}" in
+        guest)
+            _guest="$2"
+            _file="${VM_DS_PATH}/${_guest}/${_lf}"
+            ;;
+        system)
+            _file="${vm_dir}/${_lf}"
+            ;;
+    esac
+
+    if [ -e "${_file}" ]; then
+        _size=$(stat -f %z "${_file}")
+
+        if [ -n "${_size}" -a "${_size}" -ge 1048576 ]; then
+            unlink "${_file}.0.gz" >/dev/null 2>&1
+            mv "${_file}" "${_file}.0"
+            gzip "${_file}.0"
+        fi
+    fi
+}
+
+# log to file
+# writes the date and a message to the specified log
+# the global log is in $vm_dir/vm-bhyve.log
+# guest logs are $vm_dir/{guest}/vm-bhyve.log
+#
+# @param string _type=guest|system log to global vm-bhyve log or guest
+# @param optional string _guest if _type=guest, the guest name, otherwise do not provide at all
+# @param string _message the message to log
+#
+util::log(){
+    local _type="$1"
+    local _lf="vm-bhyve.log"
+    local _guest _message _file _date
+
+    case "${_type}" in
+        guest)
+            _guest="$2"
+            _file="${VM_DS_PATH}/${_guest}/${_lf}"
+            shift 2
+            ;;
+        system)
+            _file="${vm_dir}/${_lf}"
+            shift 1
+            ;;
+    esac
+
+    while [ -n "$1" ]; do
+      echo "$(date +'%b %d %T'): $1" >> "${_file}"
+      shift
+    done
+}
+
+# write content to a file, and log what we
+# did to the guest log file
+# it's useful to be able to see what files vm-bhyve is creating
+# and the contents so we write that to the log.
+# The file is created in $vm_dir/{guest}
+#
+# @param string _type=write|appnd create file or append to it
+# @param string _guest the guest name
+# @param string _file the file name to write to
+# @param string _message the data to write
+#
+util::log_and_write(){
+    local _type="$1"
+    local _guest="$2"
+    local _file="${VM_DS_PATH}/${_guest}/$3"
+    local _message="$4"
+
+    if [ "${_type}" = "write" ]; then
+        util::log "guest" "${_guest}" "create file ${_file}"
+        echo "${_message}" > "${_file}"
+    else
+        echo "${_message}" >> "${_file}"
+    fi
+
+    util::log "guest" "${_guest}" " -> ${_message}"
+}
+
+# confirm yes or no
+#
+# @param string _msh message to display
+# @return int success if confirmed
+#
+util::confirm(){
+    local _msg="$1"
+    local _resp
+
+    while read -p "${_msg} (y/n)? " _resp; do
+        case "${_resp}" in
+            y*) return 0 ;;
+            n*) return 1 ;;
+        esac
+    done
+}
+
+# our own checkyesno copy
+# doesn't warn for unsupported values
+# also returns as 'yes' unless value is specifically no/off/false/0 
+#
+# @param _value the value to test
+# @return int 1 if set to "off/false/no/0", 0 otherwise
+#
+util::yesno(){
+    local _value="$1"
+
+    [ -z "${_value}" ] && return 1
+
+    case "$_value" in
+        [Nn][Oo]|[Ff][Aa][Ll][Ss][Ee]|[Oo][Ff][Ff]|0)
+            return 1 ;;
+        *)  return 0 ;;		
+    esac
+}
+
+# 'vm check name'
+# check name of virtual machine
+#
+# @param _name name to check
+# @param _maxlen=30(229 on 13+) maximum name length (NOTE should be 2 less than desired)
+# @return int 0 if name is valid
+#
+util::check_name(){
+    local _name="$1"
+    local _maxlen="$2"
+
+    if [ -z "${_maxlen}" ]; then
+        if [ ${VERSION_BSD} -ge 1300000 ]; then
+            : ${_maxlen:=229}
+        else
+            : ${_maxlen:=30}
+        fi
+    fi
+
+    echo "${_name}" | egrep -iqs "^[a-z0-9][.a-z0-9_-]{0,${_maxlen}}[a-z0-9]\$"
+}
+
+# check if the specified string is a valid core configuration
+# setting that the user can change
+#
+# @param string the setting name to look for
+#
+util::valid_config_setting(){
+    echo "${VM_CONFIG_USER}" | grep -iqs "${1};"
+}
+
+# __getpid
+# get a process id
+#
+# @param string _var variable to put pid into
+# @param string _proc process to look for
+#
+util::getpid(){
+    local _var="$1"
+    local _proc="$2"
+    local _ret
+
+    _ret=$(pgrep -f "${_proc}")
+    [ $? -eq 0 ] || return 1
+    setvar "${_var}" "${_ret}"
+}
+
+util::get_part(){
+    local _var="$1"
+    local _data="$2"
+    local _num="$3"
+
+    setvar "${_var}" $(echo "${_data}" |cut -w -f${_num})
+}

lib/vm-zfs

@@ -1,6 +1,6 @@
 #!/bin/sh
 #-------------------------------------------------------------------------+
-# Copyright (C) 2015 Matt Churchyard (churchers@gmail.com)
+# Copyright (C) 2016 Matt Churchyard (churchers@gmail.com)
 # All rights reserved
 #
 # Redistribution and use in source and binary forms, with or without
@@ -41,16 +41,15 @@
 #
 # @modifies VM_ZFS VM_ZFS_DATASET vm_dir
 #
-__zfs_init(){
-    local _zfs
+zfs::init(){
 
     # check for zfs storage location
     # user should specify "zfs:pool/dataset" if they want ZFS support
-    if [ ${vm_dir%%:*} = "zfs" ]; then
+    if [ "${vm_dir%%:*}" = "zfs" ]; then
 
         # check zfs running
         kldstat -qm zfs >/dev/null 2>&1
-        [ $? -ne 0 ] && __err "ZFS support requested but ZFS not available"
+        [ $? -eq 0 ] || util::err "ZFS support requested but ZFS not available"
 
         # global zfs details
         VM_ZFS="1"
@@ -59,36 +58,36 @@ __zfs_init(){
         # update vm_dir
         # this makes sure it exists, confirms it's mounted & gets correct path in one go
         vm_dir=$(mount | grep "^${VM_ZFS_DATASET} " |cut -d' ' -f3)
-        [ -z "${vm_dir}" ] && __err "unable to locate mountpoint for ZFS dataset ${VM_ZFS_DATASET}"
+        [ -z "${vm_dir}" ] && util::err "unable to locate mountpoint for ZFS dataset ${VM_ZFS_DATASET}"
     fi
 }
 
 # make a new dataset
 # this is always called when creating a new vm, but will do nothing
 #
-# @param string _name name of the dataset (under VM_ZFS_DATASET) to create
+# @param string _name name of the dataset to create
 #
-__zfs_make_dataset(){
+zfs::make_dataset(){
     local _name="$1"
     local _opts="$2"
 
-    if [ -n "${_name}" -a "${VM_ZFS}" = "1" ]; then
-        __zfs_format_options "_opts" "${_opts}"
-        zfs create ${_opts} "${VM_ZFS_DATASET}/${_name}"
-        [ $? -ne 0 ] && __err "failed to create new ZFS dataset ${VM_ZFS_DATASET}/${_name}"
+    if [ -n "${_name}" -a "${VM_DS_ZFS}" = "1" ]; then
+        zfs::__format_options "_opts" "${_opts}"
+        zfs create ${_opts} "${_name}"
+        [ $? -eq 0 ] || util::err "failed to create new ZFS dataset ${_name}"
     fi
 }
 
 # destroy a dataset
 #
-# @param string _name name of the dataset to destroy (under VM_ZFS_DATASET)
+# @param string _name name of the dataset to destroy
 #
-__zfs_destroy_dataset(){
+zfs::destroy_dataset(){
     local _name="$1"
 
-    if [ -n "${_name}" -a "${VM_ZFS}" = "1" ]; then
-        zfs destroy -rf "${VM_ZFS_DATASET}/${_name}" >/dev/null 2>&1
-        [ $? -ne 0 ] && __err "failed to destroy ZFS dataset ${VM_ZFS_DATASET}/${_name}"
+    if [ -n "${_name}" -a "${VM_DS_ZFS}" = "1" ]; then
+        zfs destroy -rf "${_name}" >/dev/null 2>&1
+        [ $? -eq 0 ] || util::err "failed to destroy ZFS dataset ${_name}"
     fi
 }
 
@@ -99,35 +98,35 @@ __zfs_destroy_dataset(){
 # @param string _old the name of the dataset to rename
 # @param string _new the new name
 #
-__zfs_rename_dataset(){
+zfs::rename_dataset(){
     local _old="$1"
     local _new="$2"
 
-    if [ -n "${_old}" -a -n "${_new}" -a "${VM_ZFS}" = "1" ]; then
-        zfs rename "${VM_ZFS_DATASET}/${_old}" "${VM_ZFS_DATASET}/${_new}" >/dev/null 2>&1
-        [ $? -ne 0 ] && __err "failed to rename ZFS dataset ${VM_ZFS_DATASET}/${_old}"
+    if [ -n "${_old}" -a -n "${_new}" -a "${VM_DS_ZFS}" = "1" ]; then
+        zfs rename "${VM_DS_ZFS_DATASET}/${_old}" "${VM_DS_ZFS_DATASET}/${_new}" >/dev/null 2>&1
+        [ $? -eq 0 ] || util::err "failed to rename ZFS dataset ${VM_DS_ZFS_DATASET}/${_old}"
     fi
 }
 
 # make a zvol for a guest disk image
 #
-# @param string _name name of the guest (will be a dataset under VM_ZFS_DATASET)
+# @param string _name name of the zvol to create
 # @param string _size how big to create the dataset
 # @param int _sparse=0 set to 1 for a sparse zvol
 #
-__zfs_make_zvol(){
+zfs::make_zvol(){
     local _name="$1"
     local _size="$2"
     local _sparse="$3"
     local _user_opts="$4"
     local _opt="-V"
 
-    [ ! "${VM_ZFS}" = "1" ] && __err "cannot use ZVOL storage unless ZFS support is enabled"
+    [ ! "${VM_DS_ZFS}" = "1" ] && util::err "cannot use ZVOL storage unless ZFS support is enabled"
     [ "${_sparse}" = "1" ] && _opt="-sV"
 
-    __zfs_format_options "_user_opts" "${_user_opts}"
-    zfs create ${_opt} ${_size} -o volmode=dev ${_user_opts} "${VM_ZFS_DATASET}/${_name}"
-    [ $? -ne 0 ] && __err "failed to create new ZVOL ${VM_ZFS_DATASET}/${_name}"
+    zfs::__format_options "_user_opts" "${_user_opts}"
+    zfs create ${_opt} ${_size} -o volmode=dev ${_user_opts} "${_name}"
+    [ $? -eq 0 ] || util::err "failed to create new ZVOL ${_name}"
 }
 
 # format options for zfs commands
@@ -136,7 +135,7 @@ __zfs_make_zvol(){
 #
 # @modifies $_val
 #
-__zfs_format_options(){
+zfs::__format_options(){
     local _val="$1"
     local _c_opts="$2"
 
@@ -158,43 +157,55 @@ __zfs_format_options(){
 # @param flag (-f) force snapshot if guest is running
 # @param string _name the name of the guest to snapshot
 #
-__zfs_snapshot(){
-    local _name _snap _opt _force _snap_exists
+zfs::snapshot(){
+    local _name _snap
 
-    while getopts f _opt; do
-        case $_opt in
-            f)
-                _force=1
-                ;;
-            *)
-                __usage
-                ;;
-        esac
-    done
-
-    shift $((OPTIND - 1))
+    cmd::parse_args "$@"
+    shift $?
     _name="$1"
 
     # try to get snapshot name
     # we support normal zfs syntax for this
-    _snap_exists=$(echo "${_name}" | grep "@")
+    echo "${_name}" | grep -qs "@"
 
-    if [ -n "${_snap_exists}" ]; then
+    if [ $? -eq 0 ]; then
         _snap=${_name##*@}
         _name=${_name%%@*}
     fi
 
-    [ -z "${_name}" ] && __usage
-    [ ! "${VM_ZFS}" = "1" ] && __err "cannot snapshot guests unless ZFS support is enabled"
-    [ ! -e "${vm_dir}/${_name}/${_name}.conf" ] && __err "${_name} does not appear to be an existing virtual machine"
+    [ -z "${_name}" ] && util::usage
+    datastore::get_guest "${_name}" || util::err "${_name} does not appear to be an existing virtual machine"
+    [ ! "${VM_DS_ZFS}" = "1" ] && util::err "cannot snapshot guests on non-zfs datastores"
     [ -z "${_snap}" ] && _snap=$(date +"%Y-%m-%d-%H:%M:%S")
 
-    if ! __vm_confirm_stopped "${_name}" >/dev/null; then
-        [ -z "${_force}" ] && __err "${_name} must be powered off first (use -f to override)"
+    if ! vm::confirm_stopped "${_name}" >/dev/null; then
+        [ -z "${VM_OPT_FORCE}" ] && util::err "${_name} must be powered off first (use -f to override)"
     fi
 
-    zfs snapshot -r ${VM_ZFS_DATASET}/${_name}@${_snap}
-    [ $? -ne 0 ] && __err "failed to create recursive snapshot of virtual machine"
+    zfs snapshot -r ${VM_DS_ZFS_DATASET}/${_name}@${_snap}
+    [ $? -eq 0 ] || util::err "failed to create recursive snapshot of virtual machine"
+}
+
+# try to remove a snapshot
+#
+# @param string _name the guest name and snapshot (guest@snap)
+# @return true if successful
+#
+zfs::remove_snapshot(){
+    local _name="$1"
+    local _snap
+
+    # split name and snapshot
+    _snap=${_name##*@}
+    _name=${_name%%@*}
+
+    # try to load guest
+    datastore::get_guest "${_name}" || util::err "${_name} does not appear to be an existing virtual machine"
+    [ ! "${VM_DS_ZFS}" = "1" ] && util::err "cannot snapshot guests on non-zfs datastores"
+
+    # remove
+    zfs destroy -r ${VM_DS_ZFS_DATASET}/${_name}@${_snap}
+    [ $? -eq 0 ] || util::err "failed to remove snapshot ${VM_DS_ZFS_DATASET}/${_name}@${_snap}"
 }
 
 # 'vm rollback'
@@ -207,36 +218,33 @@ __zfs_snapshot(){
 # @param flag (-r) force deletion of more recent snapshots
 # @param string _name name of the guest
 #
-__zfs_rollback(){
+zfs::rollback(){
     local _name _snap _opt _force _fs _snap_exists
 
     while getopts r _opt; do
         case $_opt in
-            r)
-                _force="-r"
-                ;;
-            *)
-                __usage
-                ;;
+            r) _force="-r" ;;
+            *) util::usage ;;
         esac
     done
 
     shift $((OPTIND - 1))
 
     _snap_exists=$(echo "${1}" | grep "@")
-    [ -z "${_snap_exists}" ] && __err "a snapshot name must be provided in guest@snapshot format"
+    [ -z "${_snap_exists}" ] && util::err "a snapshot name must be provided in guest@snapshot format"
 
     _name="${1%%@*}"
     _snap="${1##*@}"
 
-    [ -z "${_name}" -o -z "${_snap}" ] && __usage
-    [ ! "${VM_ZFS}" = "1" ] && __err "cannot rollback guests unless ZFS support is enabled"
-    [ ! -e "${vm_dir}/${_name}/${_name}.conf" ] && __err "${_name} does not appear to be an existing virtual machine"
+    [ -z "${_name}" -o -z "${_snap}" ] && util::usage
 
-    __vm_confirm_stopped "${_name}" || exit 1
+    datastore::get_guest "${_name}" || util::err "${_name} does not appear to be an existing virtual machine"
+    [ ! "${VM_DS_ZFS}" = "1" ] && util::err "cannot rollback guests on non-zfs datastores"
+
+    vm::confirm_stopped "${_name}" || exit 1
 
     # list all datasets and zvols under guest
-    zfs list -o name -rHt filesystem,volume ${VM_ZFS_DATASET}/${_name} | \
+    zfs list -o name -rHt filesystem,volume ${VM_DS_ZFS_DATASET}/${_name} | \
     while read _fs; do
         zfs rollback ${_force} ${_fs}@${_snap}
         [ $? -ne 0 ] && exit $?
@@ -250,16 +258,16 @@ __zfs_rollback(){
 # @param string _old the guest to clone
 # @param string _new name of the new guest
 #
-__zfs_clone(){
+zfs::clone(){
     local _old="$1"
     local _name="$2"
     local _fs _newfs _snap _snap_exists _fs_list _entry
     local _num=0 _error=0
     local _uuid=$(uuidgen)
 
-    [ -z "${_old}" -o -z "${_name}" ] && __usage
-    [ ! "${VM_ZFS}" = "1" ] && __err "cannot clone guests unless ZFS support is enabled"
-    [ -d "${vm_dir}/${_name}" ] && __err "directory ${vm_dir}/${_name} already exists"
+    # check args and make sure new guest doesn't already exist
+    [ -z "${_old}" -o -z "${_name}" ] && util::usage
+    datastore::get_guest "${_name}" && util::err "new guest already exists in ${VM_DS_PATH}/${_name}"
 
     # try to get snapshot name
     # we support normal zfs syntax for this
@@ -270,50 +278,48 @@ __zfs_clone(){
         _old=${_old%%@*}
     fi
 
-    [ ! -e "${vm_dir}/${_old}/${_old}.conf" ] && __err "${_old} does not appear to be an existing virtual machine"
+    # make sure old guest exists
+    datastore::get_guest "${_old}" || util::err "${_old} does not appear to be an existing virtual machine"
+    [ ! "${VM_DS_ZFS}" = "1" ] && util::err "cannot clone guests on non-zfs datastores"
 
     # get list of datasets to copy
-    _fs_list=$(zfs list -rHo name -t filesystem,volume "${VM_ZFS_DATASET}/${_old}")
-    [ $? -ne 0 ] && __err "unable to list datasets for ${VM_ZFS_DATASET}/${_old}"
+    _fs_list=$(zfs list -rHo name -t filesystem,volume "${VM_DS_ZFS_DATASET}/${_old}")
+    [ $? -eq 0 ] || util::err "unable to list datasets for ${VM_DS_ZFS_DATASET}/${_old}"
 
     # generate a short uuid and create snapshot if no custom snap given
     if [ -z "${_snap}" ]; then
-        __vm_confirm_stopped "${_old}" || exit 1
+        vm::confirm_stopped "${_old}" || exit 1
         _snap=$(echo "${_uuid}" |awk -F- '{print $1}')
 
-        zfs snapshot -r "${VM_ZFS_DATASET}/${_old}@${_snap}"
-        [ $? -ne 0 ] && __err "failed to create snapshot ${VM_ZFS_DATASET}/${_old}@${_snap}"
+        zfs snapshot -r "${VM_DS_ZFS_DATASET}/${_old}@${_snap}"
+        [ $? -eq 0 ] || util::err "failed to create snapshot ${VM_DS_ZFS_DATASET}/${_old}@${_snap}"
     else
         for _fs in ${_fs_list}; do
             zfs get creation "${_fs}@${_snap}" >/dev/null 2>&1
-            [ $? -ne 0 ] && __err "snapshot ${_fs}@${_snap} doesn't seem to exist"
+            [ $? -eq 0 ] || util::err "snapshot ${_fs}@${_snap} doesn't seem to exist"
         done
     fi
 
     # clone
     for _fs in ${_fs_list}; do
-        _newfs=$(echo "${_fs}" | sed "s@${VM_ZFS_DATASET}/${_old}@${VM_ZFS_DATASET}/${_name}@")
+        _newfs=$(echo "${_fs}" | sed "s@${VM_DS_ZFS_DATASET}/${_old}@${VM_DS_ZFS_DATASET}/${_name}@")
 
         zfs clone "${_fs}@${_snap}" "${_newfs}"
-        [ $? -ne 0 ] && __err "error while cloning dataset ${_fs}@${_snap}"
+        [ $? -eq 0 ] || util::err "error while cloning dataset ${_fs}@${_snap}"
     done
 
     # update new guest files
-    mv "${vm_dir}/${_name}/${_old}.conf" "${vm_dir}/${_name}/${_name}.conf"
-    unlink "${vm_dir}/${_name}/vm-bhyve.log" >/dev/null 2>&1
+    unlink "${VM_DS_PATH}/${_name}/vm-bhyve.log" >/dev/null 2>&1
+    mv "${VM_DS_PATH}/${_name}/${_old}.conf" "${VM_DS_PATH}/${_name}/${_name}.conf" >/dev/null 2>&1
+    if [ $? -ne 0 ]; then
+        echo "Unable to rename configuration file to ${_name}.conf"
+        echo "This will need to be renamed manually"
+        echo "Please also remove any uuid or mac address settings, these will be regenerated automatically"
+        exit 1
+    fi
 
-    # update mac addresses and uuid
-    __config_load "${vm_dir}/${_name}/${_name}.conf"
-    __vm_config_set "${_name}" "uuid" "${_uuid}"
-
-    # find all network interfaces and change mac
-    while [ 1 ]; do
-        __config_get "_entry" "network${_num}_type"
-        [ -z "${_entry}" ] && break
-
-        __vm_generate_static_mac
-        _num=$((_num + 1))
-    done
+    # generalise the clone
+    vm::generalise "${_name}"
 }
 
 # 'vm image create'
@@ -324,18 +330,16 @@ __zfs_clone(){
 # @param optional string (-d) description of the image
 # @param string _name name of guest to take image of
 #
-__zfs_image_create(){
+zfs::image_create(){
     local _name _opt _desc
-    local _uuid _snap _date
+    local _uuid _snap _date _no_compress _filename
+    local _compress _decompress
 
-    while getopts d: _opt ; do
+    while getopts d:u _opt ; do
         case $_opt in
-            d)
-                _desc=${OPTARG}
-                ;;
-            *)
-                __usage
-                ;;
+            d) _desc=${OPTARG} ;;
+            u) _no_compress="1" ;;
+            *) util::usage ;;
         esac
     done
 
@@ -346,31 +350,54 @@ __zfs_image_create(){
     _date=$(date)
 
     [ -z "${_desc}" ] && _desc="No description provided"
-    [ ! -e "${vm_dir}/${_name}" ] && __err "${_name} does not appear to be a valid virtual machine"
+
+    datastore::get_guest "${_name}" || util::err "${_name} does not appear to be a valid virtual machine"
+    [ -z "${VM_ZFS}" -o -z "${VM_DS_ZFS}" ] && util::err "this command is only supported on zfs datastores"
 
     # create the image dataset if we don't have it
     if [ ! -e "${vm_dir}/images" ]; then
         zfs create "${VM_ZFS_DATASET}/images" >/dev/null 2>&1
-        [ $? -ne 0 ] && __err "failed to create image store ${VM_ZFS_DATASET}/images"
+        [ $? -eq 0 ] || util::err "failed to create image store ${VM_ZFS_DATASET}/images"
     fi
 
     # try to snapshot
-    zfs snapshot -r "${VM_ZFS_DATASET}/${_name}@${_snap}" >/dev/null 2>&1
-    [ $? -ne 0 ] && __err "failed to create snapshot of source dataset ${VM_ZFS_DATASET}/${_name}@${_snap}"
+    zfs snapshot -r "${VM_DS_ZFS_DATASET}/${_name}@${_snap}" >/dev/null 2>&1
+    [ $? -eq 0 ] || util::err "failed to create snapshot of source dataset ${VM_DS_ZFS_DATASET}/${_name}@${_snap}"
 
     # copy source
-    echo "Creating a compressed image, this may take some time..."
-    zfs send -R "${VM_ZFS_DATASET}/${_name}@${_snap}" | xz > "${vm_dir}/images/${_uuid}.zfs.xz"
+    if [ -n "${_no_compress}" ]; then
+        _filename="${_uuid}.zfs"
+
+        echo "Creating guest image, this may take some time..."
+        zfs send -R "${VM_DS_ZFS_DATASET}/${_name}@${_snap}" > "${vm_dir}/images/${_filename}"
+    else
+        _filename="${_uuid}.zfs.z"
+
+        config::core::get "_compress" "compress"
+        config::core::get "_decompress" "decompress"
+
+        # use defaults if either of these settings are missing
+        # no point using user defined compress if we don't know how to decompress
+        if [ "${_compress}" = "" -o "${_decompress}" = "" ]; then
+            _compress="xz -T0"
+            _decompress="xz -d"
+        fi
+
+        echo "Creating a compressed image, this may take some time..."
+        zfs send -R "${VM_DS_ZFS_DATASET}/${_name}@${_snap}" | ${_compress} > "${vm_dir}/images/${_filename}"
+    fi
+
     [ $? -ne 0 ] && exit 1
 
     # done with the source snapshot
-    zfs destroy ${VM_ZFS_DATASET}/${_name}@${_snap}
+    zfs destroy -r ${VM_DS_ZFS_DATASET}/${_name}@${_snap}
 
     # create a description file
     sysrc -inqf "${vm_dir}/images/${_uuid}.manifest" "description=${_desc}" >/dev/null 2>&1
     sysrc -inqf "${vm_dir}/images/${_uuid}.manifest" "created=${_date}" >/dev/null 2>&1
     sysrc -inqf "${vm_dir}/images/${_uuid}.manifest" "name=${_name}" >/dev/null 2>&1
-    sysrc -inqf "${vm_dir}/images/${_uuid}.manifest" "filename=${_uuid}.zfs.xz" >/dev/null 2>&1
+    sysrc -inqf "${vm_dir}/images/${_uuid}.manifest" "filename=${_filename}" >/dev/null 2>&1
+    sysrc -inqf "${vm_dir}/images/${_uuid}.manifest" "decompress=${_decompress}" >/dev/null 2>&1
 
     echo "Image of ${_name} created with UUID ${_uuid}"
 }
@@ -381,73 +408,94 @@ __zfs_image_create(){
 # @param string _uuid the uuid of the image to use
 # @param string _name name of the new guest
 #
-__zfs_image_provision(){
-    local _uuid="$1"
-    local _name="$2"
-    local _file _oldname _entry _num=0
+zfs::image_provision(){
+    local _uuid _name _file _oldname _entry _num=0 _type
+    local _datastore="default" _decompress
 
-    [ -z "${_uuid}" -o -z "${_name}" ] && __usage
-    [ ! -e "${vm_dir}/images/${_uuid}.manifest" ] && __err "unable to locate image with uuid ${_uuid}"
-    [ -e "${vm_dir}/${_name}" ] && __err "directory ${vm_dir}/${_name} already exists"
+    while getopts d: _opt ; do
+        case $_opt in
+            d) _datastore="${OPTARG}" ;;
+            *) util::usage ;;
+        esac
+    done
+
+    shift $((OPTIND - 1))
+    _uuid="$1"
+    _name="$2"
+
+    [ -z "${_uuid}" -o -z "${_name}" ] && util::usage
+    [ ! -e "${vm_dir}/images/${_uuid}.manifest" ] && util::err "unable to locate image with uuid ${_uuid}"
+    datastore::get_guest "${_name}" && util::err "new guest already exists in ${VM_DS_PATH}/${_name}"
 
     # get the data filename
     _file=$(sysrc -inqf "${vm_dir}/images/${_uuid}.manifest" filename)
+    _type=${_file##*.}
+
     _oldname=$(sysrc -inqf "${vm_dir}/images/${_uuid}.manifest" name)
-    [ -z "${_file}" -o -z "${_oldname}" ] && __err "unable to locate required details from the specified image manifest"
-    [ ! -e "${vm_dir}/images/${_file}" ] && __err "image data file does not exist: ${vm_dir}/images/${_file}"
+    [ -z "${_file}" -o -z "${_oldname}" ] && util::err "unable to locate required details from the specified image manifest"
+    [ ! -e "${vm_dir}/images/${_file}" ] && util::err "image data file does not exist: ${vm_dir}/images/${_file}"
+
+    # get the datastore to create on
+    datastore::get "${_datastore}" || util::err "unable to locate datastore '${_datastore}'"
 
     # try to recieve
-    echo "Unpacking compressed image, this may take some time..."
-    cat "${vm_dir}/images/${_file}" | xz -d | zfs recv "${VM_ZFS_DATASET}/${_name}"
-    [ $? -ne 0 ] && __err "errors occured while trying to unpackage the image file"
+    echo "Unpacking guest image, this may take some time..."
+
+    # check format of image
+    case ${_type} in
+        zfs) cat "${vm_dir}/images/${_file}" | zfs recv "${VM_DS_ZFS_DATASET}/${_name}" ;;
+        xz)  xz -dc "${vm_dir}/images/${_file}" 2>/dev/null | zfs recv "${VM_DS_ZFS_DATASET}/${_name}" ;;
+        z)   _decompress=$(sysrc -inqf "${vm_dir}/images/${_uuid}.manifest" decompress)
+             [ -z "${_decompress}" ] && util::err "unable to locate decompression configuration"
+             ${_decompress} <"${vm_dir}/images/${_file}" 2>/dev/null | zfs recv "${VM_DS_ZFS_DATASET}/${_name}" ;;
+        *)   util::err "unsupported guest image type - '${_type}'" ;;
+    esac
+
+    # error unpacking?
+    [ $? -eq 0 ] || util::err "errors occured while trying to unpackage the image file"
 
     # remove the original snapshot
-    zfs destroy "${VM_ZFS_DATASET}/${_name}@${_uuid}" >/dev/null 2>&1
+    zfs destroy -r "${VM_DS_ZFS_DATASET}/${_name}@${_uuid%%-*}" >/dev/null 2>&1
 
     # rename the guest configuration file
-    mv "${vm_dir}/${_name}/${_oldname}.conf" "${vm_dir}/${_name}/${_name}.conf" >/dev/null 2>&1
-    [ $? -ne 0 ] && __err "unpackaged image but unable to update guest configuration file"
+    mv "${VM_DS_PATH}/${_name}/${_oldname}.conf" "${VM_DS_PATH}/${_name}/${_name}.conf" >/dev/null 2>&1
+    [ $? -eq 0 ] || util::err "unpackaged image but unable to update guest configuration file"
 
-    # update mac addresses and uuid
-    __config_load "${vm_dir}/${_name}/${_name}.conf"
-    __vm_config_set "${_name}" "uuid" "${_uuid}"
+    # update mac addresses and create a new uuid
+    _uuid=$(uuidgen)
 
-    # find all network interfaces and change mac
-    while [ 1 ]; do
-        __config_get "_entry" "network${_num}_type"
-        [ -z "${_entry}" ] && break
-
-        __vm_generate_static_mac
-        _num=$((_num + 1))
-    done
+    # remove unique settings from new image
+    vm::generalise "${_name}"
 
     # vm may be started when 'vm image create' is executed
-    rm -f "${vm_dir}/${_name}/run.lock" >/dev/null 2>&1
-    rm -f "${vm_dir}/${_name}/vm-bhyve.log*" >/dev/null 2>&1
+    rm -f "${VM_DS_PATH}/${_name}/run.lock" >/dev/null 2>&1
+    rm -f "${VM_DS_PATH}/${_name}/vm-bhyve.log*" >/dev/null 2>&1
 }
 
 # 'vm image list'
 # list available images
 #
-__zfs_image_list(){
+zfs::image_list(){
     local _file _uuid _ext
-    local _format="%-38s %-16s %-30s %s\n"
+    local _format="%s^%s^%s^%s\n"
 
-    printf "${_format}" "UUID" "NAME" "CREATED" "DESCRIPTION"
+    {
+        printf "${_format}" "UUID" "NAME" "CREATED" "DESCRIPTION"
 
-    [ ! -e "${vm_dir}/images" ] && exit
+        [ ! -e "${vm_dir}/images" ] && exit
 
-    ls -1 ${vm_dir}/images/ | \
-    while read _file; do
-        if [ "${_file##*.}" = "manifest" ]; then
-            _uuid=${_file%.*}
-            _desc=$(sysrc -inqf "${vm_dir}/images/${_uuid}.manifest" description)
-            _created=$(sysrc -inqf "${vm_dir}/images/${_uuid}.manifest" created)
-            _name=$(sysrc -inqf "${vm_dir}/images/${_uuid}.manifest" name)
+        ls -1 ${vm_dir}/images/ | \
+        while read _file; do
+            if [ "${_file##*.}" = "manifest" ]; then
+                _uuid=${_file%.*}
+                _desc=$(sysrc -inqf "${vm_dir}/images/${_uuid}.manifest" description)
+                _created=$(sysrc -inqf "${vm_dir}/images/${_uuid}.manifest" created)
+                _name=$(sysrc -inqf "${vm_dir}/images/${_uuid}.manifest" name)
 
-            printf "${_format}" "${_uuid}" "${_name}" "${_created}" "${_desc}"
-        fi
-    done
+                printf "${_format}" "${_uuid}" "${_name}" "${_created}" "${_desc}"
+            fi
+        done
+    } | column -ts^
 }
 
 # 'vm image destroy'
@@ -455,39 +503,17 @@ __zfs_image_list(){
 #
 # @param string _uuid the uuid of the image
 #
-__zfs_image_destroy(){
+zfs::image_destroy(){
     local _uuid="$1"
     local _file
 
-    [ -z "${_uuid}" ] && __usage
-    [ ! -e "${vm_dir}/images/${_uuid}.manifest" ] && __err "unable to locate image with uuid ${_uuid}"
+    [ -z "${_uuid}" ] && util::usage
+    [ ! -e "${vm_dir}/images/${_uuid}.manifest" ] && util::err "unable to locate image with uuid ${_uuid}"
 
     # get the image filename
     _file=$(sysrc -inqf "${vm_dir}/images/${_uuid}.manifest" filename)
-    [ -z "${_file}" ] && __err "unable to locate filename for the specified image"
+    [ -z "${_file}" ] && util::err "unable to locate filename for the specified image"
 
     unlink "${vm_dir}/images/${_uuid}.manifest"
     unlink "${vm_dir}/images/${_file}"
 }
-
-# cmd 'vm image ...'
-# parse the image command set
-# these all rely on ZFS snapshots, so kept with zfs functions
-#
-# @param string _cmd the command after 'vm image '
-#
-__zfs_parse_image_cmd(){
-    local _cmd="$1"
-    shift
-
-    # we only support these commands on zfs
-    [ ! "${VM_ZFS}" = "1" ] && __err "the image command set is only available with ZFS storage"
-
-    case "${_cmd}" in
-        list)      __zfs_image_list ;;
-        create)    __zfs_image_create "$@" ;;
-        provision) __zfs_image_provision "$@" ;;
-        destroy)   __zfs_image_destroy "$@" ;;
-        *)         __usage ;;
-    esac
-}

rc.d/vm

@@ -3,9 +3,9 @@
 # $FreeBSD$
 
 # PROVIDE: vm
-# REQUIRE: NETWORKING SERVERS
-# BEFORE: dnsmasq
-# KEYWORD: shutdown nojail
+# REQUIRE: NETWORKING SERVERS dmesg
+# BEFORE: dnsmasq ipfw pf
+# KEYWORD: shutdown nojailvnet
 
 . /etc/rc.subr
 
@@ -19,7 +19,7 @@ load_rc_config $name
 
 command="/usr/local/sbin/${name}"
 start_cmd="${name}_start"
-stop_cmd="${command} stopall"
+stop_cmd="${command} stopall -f"
 
 vm_start()
 {

sample-templates/alpine.conf

@@ -1,4 +1,3 @@
-guest="linux"
 loader="grub"
 cpu=1
 memory=512M
@@ -6,7 +5,7 @@ network0_type="virtio-net"
 network0_switch="public"
 disk0_type="virtio-blk"
 disk0_name="disk0.img"
-grub_install0="linux /boot/grsec initrd=/boot/initramfs-grsec alpine_dev=cdrom:iso9660 modules=loop,squashfs,sd-mod,usb-storage,sr-mod"
-grub_install1="initrd /boot/initramfs-grsec"
-grub_run0="linux /boot/vmlinuz-grsec root=/dev/vda3 modules=ext4"
-grub_run1="initrd /boot/initramfs-grsec"
+grub_install0="linux /boot/vmlinuz-vanilla initrd=/boot/initramfs-vanilla alpine_dev=cdrom:iso9660 modules=loop,squashfs,sd-mod,usb-storage,sr-mod"
+grub_install1="initrd /boot/initramfs-vanilla"
+grub_run0="linux /boot/vmlinuz-vanilla root=/dev/vda3 modules=ext4"
+grub_run1="initrd /boot/initramfs-vanilla"

sample-templates/arch.conf

@@ -0,0 +1,9 @@
+loader="grub"
+cpu=1
+memory=512M
+network0_type="virtio-net"
+network0_switch="public"
+disk0_type="virtio-blk"
+disk0_name="disk0.img"
+grub_install0="linux /arch/boot/x86_64/vmlinuz archisobasedir=arch archisolabel=ARCH_201611 ro"
+grub_install1="initrd /arch/boot/x86_64/archiso.img"

sample-templates/centos6.conf

@@ -1,4 +1,3 @@
-guest="linux"
 loader="grub"
 cpu=1
 memory=512M

sample-templates/centos7.conf

@@ -0,0 +1,9 @@
+loader="uefi"
+graphics="yes"
+xhci_mouse="yes"
+cpu=1
+memory=512M
+network0_type="virtio-net"
+network0_switch="public"
+disk0_type="virtio-blk"
+disk0_name="disk0.img"

sample-templates/config.sample

@@ -1,11 +1,8 @@
 # This is a sample configuration file containing all supported options
-# Please do not try and use this for a guest
+# Please do not try and use this file itself for a guest
 # For any option that contains a number in the name, such as "network0_type",
 # you can add additional devices of that type by creating a new set of
 # variables using the next number in sequence, e.g "network1_type"
-# The only exception to this is that only one disk device is supported
-# in templates. If additional disks are required, they will need to be
-# added once the guest has been created.
 #
 # Please make sure all option names are specified in lowercase and
 # at the beginning of the line. If there is any whitespace before
@@ -13,62 +10,77 @@
 # The '#' character signifies the start of a comment, even within
 # double-quotes, and so cannot be used inside any values.
 
-# guest
-# The type of operating system this virtual machine will use.
-# Different systems often require specific steps to load the
-# virtual machine, or specific bhyve options. The generic option
-# does no specific loading and can be used for standard uefi guests.
-#
-# This is no longer a required option and will default to 'generic'
-# if not specified. If set to freebsd, the bhyveload loader is 
-# automatically used, and if set to linux, grub-bhyve will be used.
-#
-# Default: generic
-# Valid Options: freebsd,linux,windows,generic
-#
-guest=""
-
 # loader
-# If the guest you are using requires either bhyveload or grub-bhyve,
-# specify the loader to use here. If the guest is freebsd or linux,
-# this is not required as the correct loader is used by default.
+# Specify the loader to use for the guest. This can either be
+# one of the original bhyve loaders (bhyveload/grub), or
+# you can specify to use uefi firmware to load the guest
 #
-# If using guests such as netbsd, you can either set the guest to
-# linux, or set to generic and specify the grub loader here
-#
-# Valid Options: bhyveload,grub
+# Valid Options: bhyveload,grub,uefi,uefi-csm
 #
 loader=""
 
+# bhyveload_loader
+# If using bhyveload, this option can be used to specify the path
+# to the loader inside the guest to use
+#
+# the default is /boot/userboot.so
+#
+bhyveload_loader=""
+
+# bhyveload_args
+# If using bhyveload, this option can be used to pass command line
+# arguments to the loader
+#
+bhyveload_args="-e machdep.hyperthreading_allowed=0"
+
 # loader_timeout
-# By default grub-bhyve will wait 3 seconds before booting the default
+# By default bhyveload & grub-bhyve will wait 3 seconds before booting the default
 # option. This setting allows you to either reduce the timeout to
 # make boot faster, or increase it so that it's easier to access
 # the grub console before it starts booting
 #
 loader_timeout="3"
 
-# uefi
-# Tells bhyve that it should load the UEFI firmware. To use UEFI
-# this can be any value other than [empty]/off/false/no/0. If
-# it contains the string 'csm', the UEFI BIOS compatability (CSM)
-# firmware will be used
+# uefi_vars
+# set to a true value to support persistent UEFI vars
+# this relies on a version of uefi-firmware that comes with the BHYVE_UEFI_VARS.fd template,
+# and support in bhyve
 #
-# Default: no
-#
-uefi=""
+uefi_vars="no"
 
 # cpu (required)
 # specify the number of cpu cores to give to the guest
 #
 cpu="1"
 
+# cpu_sockets
+# manually configure the number of sockets that bhyve should
+# expose to the guest.
+# note that sockets*cores*threads should equal the above cpu count
+#
+cpu_sockets="1"
+
+# cpu_cores
+# the number of cores to create per physical processor
+#
+cpu_cores="1"
+
+# cpu_threads
+# the number of cpu threads per core
+#
+cpu_threads="1"
+
 # memory (required)
 # specify the amount of ram to give to the guest. This can be
 # followed by M or G.
 #
 memory="512M"
 
+# wired_memory
+# All requested memory should be wired to the guest
+#
+wired_memory="no"
+
 # hostbridge
 # Allows you to specify the type of hostbridge to use for the
 # guest hardware. This can usually be left as default. The
@@ -82,6 +94,20 @@ memory="512M"
 #
 hostbridge=""
 
+# ignore_bad_msr
+# Instruct bhyve to ignore accesses to model specific registers
+# that are not implemented in the current CPU.
+# This appears to be required for AMD processors when using
+# some guest operating systems. Note that this is enabled
+# by default when running a UEFI guest
+#
+ignore_bad_msr="no"
+
+# bhyve_options
+# any additional bhyve command line options
+#
+bhyve_options="-p 1:1"
+
 # comports
 # This allows you to define the com ports which should be available.
 # By default only com1 is connected, and can be accessed using the
@@ -96,12 +122,24 @@ hostbridge=""
 comports=""
 
 # utctime
-# Set to any value other than [empty]/no/off/false/0 if you want the guest
-# clock to use UTC time.
+# bhyve normally sets the guests RTC to the host's localtime. The utctime
+# option causes bhyve to try and configure the guests RTC to UTC.
 #
-# Default: no
+# As of vm-bhyve 1.2, this setting defaults to yes, giving the guest a
+# UTC realtime clock. I consider this more consistent, and is actually
+# expected by some guests. The guest should show correct time as long as
+# its timezone is configured correctly. Note that the following command
+# is useful to verify the time of a guest's "hardware" RTC:
+# bhyvectl --vm={guestname} --get-rtc-time
 #
-utctime=""
+# To revert to the default bhyve behaviour, explicitly set this to off/no/false/0
+#
+# Additionally it is generally advised to run a time sync daemon, such as ntpd
+# in the guest, as each OS will have its own clock that will inevitably drift.
+#
+# Default: yes
+#
+utctime="no"
 
 # debug
 # Set to a value other than [empty]/no/off/false/0 to run vm-bhyve in debug mode.
@@ -115,17 +153,28 @@ debug=""
 
 # uuid
 # This is set automatically by vm-bhyve when creating a new guest. Normally
-# bhyve assigns a UUID at runtime based on host and guest name. This 
+# bhyve assigns a UUID at runtime based on host and guest name. This
 # option allows you to specify a fixed UUID that will always be used. Remove
 # this or leave blank to return to the normal bhyve behaviour.
 #
 uuid=""
 
+# ahci_device_limit
+# By default all ahci devices (ahci-hd/ahci-cd) are configured on independent
+# slots with their own controller. In FreeBSD 12 it's possible to put up
+# to 32 devices on each controller. This setting allows you to configure
+# the number of devices vm-bhyve will allocate on each controller.
+#
+# Valid Options: 2-32
+# Default: 1
+#
+ahci_device_limit="8"
+
 # disk0_type (required)
 # This specifies the emulation type for disk0. Please note that each disk requires
 # at least a type and name.
 #
-# Valid Options: virtio-blk,ahci-hd
+# Valid Options: virtio-blk,ahci-hd,ahci-cd,nvme
 #
 disk0_type="virtio-blk"
 
@@ -135,9 +184,11 @@ disk0_type="virtio-blk"
 # For the zvol options, the zvol must be directly under the guest dataset.
 # There is also a 'custom' option, in which case the disk name should be the full path
 # to the file or device you want to use.
+# For 'iscsi', the disk name must be set to a unique target and lun combination
+# when matched against iscsictl -L output.
 #
 # Default: file
-# Valid Options: file,zvol,sparse-zvol,custom
+# Valid Options: file,zvol,sparse-zvol,custom,iscsi
 #
 disk0_dev=""
 
@@ -151,6 +202,7 @@ disk0_dev=""
 # file              'disk0.img' -> '$vm_dir/$name/disk0.img'
 # zvol|sparse-zvol  'disk0'     -> '/dev/zvol/pool/dataset/path/guest/disk0'
 # custom            '/dev/da10' -> '/dev/da10'
+# iscsi             'tgt[/lun]' -> '/dev/daNN' (lun defaults to 0 if omitted)
 #
 disk0_name="disk0.img"
 
@@ -166,7 +218,9 @@ disk0_opts=""
 # disk0_size
 # When a new guest is created, vm will create a 20G disk image by
 # default. This option can be used to specify a different size
-# for this disk.
+# for this disk. Make sure to include a human readable suffix
+# compatible with 'zfs create' (G for gigabytes, T terabytes,
+# etc)
 #
 # The size of the first disk (disk0) can also be overridden
 # using the -s option to 'vm create'.
@@ -177,14 +231,14 @@ disk0_opts=""
 # a new guest, all 'diskX_size' options are stripped from
 # its configuration file.
 #
-disk0_size="50"
+disk0_size="50G"
 
 # network0_type
 # This specifies the emulation type to use for the first network interface.
 # Networking is not required, although this field is mandatory if you do want
 # to add an interface
 #
-# Valid Options: virtio-net
+# Valid Options: virtio-net,e1000
 #
 network0_type="virtio-net"
 
@@ -206,10 +260,15 @@ network0_switch="public"
 #
 network0_device=""
 
+# network0_name
+# if specified, the interface will be given this name
+#
+network0_name="web1"
+
 # network0_mac
 # This allows you to specify a fixed mac address for this interface inside the guest.
 # When a guest is run, vm-bhyve will automatically assign a mac address for each
-# interface if one is not specified. This mac address is then written to the 
+# interface if one is not specified. This mac address is then written to the
 # configuration file using this option. If we didn't do this guests might get
 # a different mac if the tap device changes (very possible in vm-bhyve as all
 # tap devices are dynamic by default). Guests like Windows treat an interface
@@ -217,12 +276,25 @@ network0_device=""
 #
 network0_mac=""
 
+# network0_span
+# Set to any value other than [empty]/off/false/no/0 to create the specified
+# port as a SPAN port rather than as an ordinary bridge member.
+#
+# NOTE: Does not work with VALE switches yet.
+#
+network0_span="no"
+
 # passthru0
 # Add a pass-through PCI device to the virtual machine. This allows the guest
 # to access a hardware device no differently than if it was running on bare
 # metal. The value of this option is the B/S/F of the appropriate device.
 # e.g "3/0/0"
 #
+# The slot to use in bhyve can be specified as below. This example will
+# force the host device 6/0/0 to use slot 2:0 in the guest
+#
+# passthru0="6/0/0=2:0"
+#
 # Please note that in order to stop the bhyve host from attaching to the device,
 # there are some steps required to reserve the device in /boot/loader.conf.
 #
@@ -234,12 +306,106 @@ network0_mac=""
 #
 passthru0=""
 
+# start_slot
+# The slot to start creating devices at inside the guest. Note that
+# we create disk devices first, and some UEFI guests require disks to
+# be in slots 3-6. The default is 4, with 3 being left available for
+# an installation ISO
+#
+start_slot="4"
+
+# install_slot
+# The slot to use for an installation ISO. By default this is 3,
+# which is the first available slot with the original UEFI firmware.
+# Using this makes sure the ISO is the first device, and leaves
+# 4-6 available for hd devices. Being able to change this may
+# be useful for non-UEFI guests, especially if a passthru device
+# requires this slot.
+#
+install_slot="3"
+
 # virt_random
 # Set to any value other than [empty]/off/false/no/0 to create
 # a virtio-rnd device for the guest
 #
 virt_random=""
 
+# graphics
+# Set to a value other than [empty]/off/false/no/0 to enable
+# the bhyve frame buffer device. This creates a graphics console
+# in the guest, which is accessible using vnc
+#
+# By default this is set at 800x600, and we find an available vnc
+# port starting at 5900. The port can be seen in vm list|info output.
+#
+graphics="yes"
+
+# graphics_port
+# Use this option to specify a fixed network port that the vnc service
+# should listen on. If specifying port numbers manually, please make
+# sure all guests have a unique port.
+#
+graphics_port="5999"
+
+# graphics_listen
+# By default, the vnc service will listen on 0.0.0.0, so you can connect by
+# using any IP address assigned to the bhyve host. Use this option if you
+# want to specify a specific IP address that the service should bind to
+#
+# Default: 0.0.0.0
+#
+graphics_listen="10.0.0.1"
+
+# graphics_res
+# This allows you to specify a resolution for the graphical console.
+# Pleas note only the below options are supported
+#
+# Default: 800x600
+# Valid Options: 1920x1200,1920x1080,1600x1200,1600x900,1280x1024,1280x720,1024x768,800x600,640x480
+#
+graphics_res="800x600"
+
+# graphics_wait
+# Set to yes in order to make guest boot wait for the VNC console
+# to be opened. This can help when installing operating systems
+# that require immediate keyboard input (such as a timed 'enter setup'
+# screen). The default setting of auto will add the wait option
+# if the guest is run in install mode. Note that in auto mode
+# the wait option will only be present on the first boot. If you
+# need the guest to wait on every boot during install, the yes
+# option should be used.
+#
+# Valid Options: no,yes,auto
+#
+graphics_wait="auto"
+
+# graphics_vga
+# valid options for this are on/off/io. io is the default
+# please see the bhyve man page for details on this option
+#
+graphics_vga="io"
+
+# xhci_mouse
+# When graphics are enabled, a PS2 mouse is created by default. This
+# doesn't track very well, and can be replaced with an XHCI mouse
+# by setting this option to yes. Please note only some guests support
+# this mouse
+#
+xhci_mouse="yes"
+
+# virt_console0
+# create up to 16 virtual console devices
+#
+# the value can be yes|on|1 to create a numbered port. FreeBSD < 12
+# only supports virtio consoles configured in this way
+#
+# For guests with named console support (FreeBSD 12+, Linux?), the
+# value can be the name of the port to create. The name "org.freenas.byhve-agent"
+# can be useful as it ties in with tools written to make use of the
+# FreeNAS bhyve-agent interface.
+#
+virt_console0="org.freenas.byhve-agent"
+
 # grub_install0
 # use this to specify grub commands that should be run inside the
 # guest when installing.
@@ -301,6 +467,27 @@ zfs_dataset_opts=""
 #
 zfs_zvol_opts=""
 
+# prestart
+# specify a script to run when the guest starts
+# if just a name rather than full path is provided, we look in the guest directory
+# the script must be executable and is run in the following way -
+#
+# {scriptname} <guest-name> [zfs-dataset?]
+#
+# we also change directory to <guest-path> before running the script
+# note that if taking guest snapshots, the -f option must be used as although
+# the guest is technically stopped when this script runs, vm-bhyve still has it
+# locked
+#
+prestart="myscript.pl"
+
+# priority
+# set a priority (nice value) for a guest
+# valid range is -20 (highest) to 20 (only run when system idle), with
+# 0 being the default system priority
+#
+priority="10"
+
 # limit_pcpu
 # use rctl to limit guest to the specified cpu percentage
 #

sample-templates/coreos.conf

@@ -0,0 +1,14 @@
+# Use GRUB when booting from an installation medium
+#loader="grub"
+
+# Use UEFI when booting from a disk
+loader="uefi"
+
+cpu=1
+memory=1024M
+network0_type="virtio-net"
+network0_switch="public"
+disk0_type="virtio-blk"
+disk0_name="disk0.img"
+grub_install0="linux /coreos/vmlinuz coreos.autologin"
+grub_install1="initrd /coreos/cpio.gz"

sample-templates/debian.conf

@@ -1,8 +1,9 @@
-guest="linux"
 loader="grub"
 cpu=1
 memory=512M
 network0_type="virtio-net"
 network0_switch="public"
-disk0_type="virtio-blk"
+disk0_type="ahci-hd"
 disk0_name="disk0.img"
+grub_run_partition="1"
+grub_run_dir="/boot/grub"

sample-templates/default.conf

@@ -1,4 +1,3 @@
-guest="freebsd"
 loader="bhyveload"
 cpu=1
 memory=256M

sample-templates/dragonfly.conf

@@ -0,0 +1,14 @@
+# DragonFly 4.6 and later can boot with UEFI.  The Dragonfly installer works
+# with UEFI beginning at 4.8 
+loader="uefi"
+# The live CD has a serial console, but the installer requires graphics
+graphics="yes"
+graphics_wait="no"
+cpu=1
+# 4GB of RAM is the minimum when using HAMMER.
+memory=4G
+network0_type="virtio-net"
+network0_switch="public"
+# The installer requires ahci.  It can't correctly partition a virtio-blk
+disk0_type="ahci-hd"
+disk0_name="disk0.img"

sample-templates/freebsd-zvol.conf

@@ -1,4 +1,3 @@
-guest="freebsd"
 loader="bhyveload"
 cpu=1
 memory=256M

sample-templates/freepbx.conf

@@ -0,0 +1,12 @@
+loader="grub"
+cpu=1
+memory=512M
+network0_type="virtio-net"
+network0_switch="public"
+disk0_type="virtio-blk"
+disk0_name="disk0.img"
+grub_install0="linux /isolinux/vmlinuz vga=normal ramdisk_size=32768 ks=cdrom:/kickstart-simple-asterisk13.cfg asknetwork LANG=en_US.UTF-8 KEYTABLE=us SYSFONT=latarcyrheb-sun16 console=ttyS0"
+grub_install1="initrd /isolinux/initrd.img"
+grub_run0="linux /vmlinuz-2.6.32-642.6.2.el6.x86_64 root=/dev/mapper/VolGroup-lv_root LANG=en_US.UTF-8 KEYTABLE=us SYSFONT=latarcyrheb-sun16 rd_NO_LUKS LANG=en_US.UTF-8 rd_NO_MD rd_LVM_LV=VolGroup/lv_swap SYSFONT=latarcyrheb-sun16 console=ttyS0 crashkernel=auto rd_LVM_LV=VolGroup/lv_root KEYBOARDTYPE=pc KEYTABLE=us rd_NO_DM"
+grub_run1="initrd /initramfs-2.6.32-642.6.2.el6.x86_64.img"
+

sample-templates/gentoo.conf

@@ -0,0 +1,13 @@
+loader="grub"
+cpu=1
+memory=512MB
+network0_type="virtio-net"
+network0_switch="public"
+disk0_type="virtio-blk"
+disk0_name="disk0.img"
+grub_install0="linux /boot/gentoo root=/dev/ram0 init=/linuxrc dokeymap looptype=squashfs loop=/image.squashfs cdroot"
+grub_install1="initrd /boot/gentoo.igz"
+# Make sure to modify the "root" variable according to your partitioning scheme.
+grub_run0="set root=(hd0,gpt2)"
+grub_run1="set timeout=1"
+grub_run2="configfile /grub/grub.cfg"

sample-templates/linux-zvol.conf

@@ -0,0 +1,8 @@
+loader="grub"
+cpu=1
+memory=512M
+network0_type="virtio-net"
+network0_switch="public"
+disk0_name="disk0"
+disk0_dev="sparse-zvol"
+disk0_type="virtio-blk"

sample-templates/netbsd.conf

@@ -1,4 +1,3 @@
-guest="generic"
 loader="grub"
 cpu=1
 memory=256M
@@ -7,4 +6,4 @@ network0_switch="public"
 disk0_type="virtio-blk"
 disk0_name="disk0.img"
 grub_install0="knetbsd -h -r cd0a /netbsd"
-grub_run0="knetbsd -h -r ld0a /netbsd"
+grub_run0="knetbsd -h -r dk0 /netbsd"

sample-templates/openbsd.conf

@@ -1,11 +1,17 @@
-guest="generic"
-loader="grub"
+loader="uefi"
 cpu=1
-memory=256M
+memory=512M
 network0_type="virtio-net"
 network0_switch="public"
 disk0_type="virtio-blk"
 disk0_name="disk0.img"
-grub_install0="kopenbsd -h com0 /5.9/amd64/bsd.rd"
-grub_run0="kopenbsd -h com0 -r sd0a /bsd"
-
+bhyve_options="-w"
+graphics="yes"
+xhci_mouse="yes"
+graphics_res="1600x900"
+graphics_wait="no"
+# Uncomment two lines below and download installXX.img, place in vm folder.
+# Replace XX with OpenBSD version.
+# You can delete the two lines below when installation is done
+#disk1_type="virtio-blk"
+#disk1_name="installXX.img"

sample-templates/resflash.conf

@@ -0,0 +1,10 @@
+loader="grub"
+cpu=1
+memory=256M
+network0_type="virtio-net"
+network0_switch="public"
+disk0_type="virtio-blk"
+disk0_name="disk0.img"
+grub_run_partition="openbsd4"
+grub_run0="kopenbsd -h com0 -r sd0d /bsd"
+bhyve_options="-w"

sample-templates/ubuntu.conf

@@ -1,7 +1,6 @@
-guest="linux"
 loader="grub"
 cpu=1
-memory=512M
+memory=1024M
 network0_type="virtio-net"
 network0_switch="public"
 disk0_type="virtio-blk"

sample-templates/windows.conf

@@ -1,8 +1,21 @@
-guest="windows"
-uefi="yes"
-cpu=1
+loader="uefi"
+graphics="yes"
+xhci_mouse="yes"
+cpu=2
 memory=2G
-network0_type="virtio-net"
+
+# put up to 8 disks on a single ahci controller.
+# without this, adding a disk pushes the following network devices onto higher slot numbers,
+# which causes windows to see them as a new interface
+ahci_device_limit="8"
+
+# ideally this should be changed to virtio-net and drivers installed in the guest
+# e1000 works out-of-the-box
+network0_type="e1000"
 network0_switch="public"
+
 disk0_type="ahci-hd"
 disk0_name="disk0.img"
+
+# windows expects the host to expose localtime by default, not UTC
+utctime="no"

vm

@@ -1,6 +1,6 @@
 #!/bin/sh
 #-------------------------------------------------------------------------+
-# Copyright (C) 2015 Matt Churchyard (churchers@gmail.com)
+# Copyright (C) 2016 Matt Churchyard (churchers@gmail.com)
 # All rights reserved
 #
 # Redistribution and use in source and binary forms, with or without
@@ -24,59 +24,26 @@
 # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-VERSION=1.0-beta
-VERSION_INT=100006
-VERSION_BSD=$(uname -K)
-PATH=${PATH}:/bin:/usr/bin:/usr/local/bin:/sbin:/usr/sbin:/usr/local/sbin
-
-. /etc/rc.subr
-load_rc_config "vm"
-
 # get libs
-if [ -e "./lib/vm-core" ]; then
-    LIB="./lib"
-elif [ -e "/usr/local/lib/vm-bhyve" ]; then
+if [ -e "/usr/local/lib/vm-bhyve" ]; then
     LIB="/usr/local/lib/vm-bhyve"
 else
-    echo "unable to locate vm-bhyve libriaries"
+    echo "unable to locate vm-bhyve libraries"
     exit 1
 fi
 
 # load libs
 . "${LIB}/vm-cmd"
-. "${LIB}/vm-common"
 . "${LIB}/vm-config"
 . "${LIB}/vm-core"
+. "${LIB}/vm-datastore"
 . "${LIB}/vm-guest"
 . "${LIB}/vm-info"
+. "${LIB}/vm-migration"
 . "${LIB}/vm-rctl"
 . "${LIB}/vm-run"
 . "${LIB}/vm-switch"
-. "${LIB}/vm-sysrc"
+. "${LIB}/vm-util"
 . "${LIB}/vm-zfs"
 
-# check environment
-[ `id -u` -ne 0 ] && __err "virtual machines can only be managed by root"
-[ ${VERSION_BSD} -lt 1000000 ] && __err "please upgrade to FreeBSD 10 or newer for bhyve support"
-
-# we should be enabled in rc.conf
-# or call it using forcestart
-if [ -z "$rc_force" ] && ! checkyesno vm_enable; then
-    __err "\$vm_enable is not enabled in /etc/rc.conf!"
-fi
-
-# check for any global arguments
-__parse_cmd_args $@
-
-# init for zfs
-__zfs_init
-
-# create directories as needed
-[ ! -d "${vm_dir}" ] && __err "\$vm_dir has not been configured or is not a valid directory"
-[ ! -d "${vm_dir}/.config" ] && mkdir "${vm_dir}/.config"
-[ ! -e "${vm_dir}/.config/null.iso" ] && touch "${vm_dir}/.config/null.iso"
-[ ! -d "${vm_dir}/.templates" ] && mkdir "${vm_dir}/.templates"
-[ ! -d "${vm_dir}/.iso" ] && mkdir "${vm_dir}/.iso"
-
-# run the requested command
-__parse_cmd ${VM_COMMAND}
+. "${LIB}/vm-base"