Merge branch 'master' into vm-parameters

README.md

@@ -249,6 +249,27 @@ a full shutdown and restart of the guest
 See the man page for a full description of all available commands.
 
     # man vm
+
+## Using cloud images
+
+You can use cloud images to create virtual machines. The `vm img` command will download the image to datastore and 
+uncompress it if needed (.xz, .tar.gz, and .gz files are supported). The image should be in RAW or QCOW2 format.
+
+To launch FreeBSD using official cloud image:
+
+    # vm img ftp://ftp.icm.edu.pl/pub/FreeBSD/releases/VM-IMAGES/11.2-RELEASE/amd64/Latest/FreeBSD-11.2-RELEASE-amd64.raw.xz
+    # vm create -t freebsd-zvol -i FreeBSD-11.2-RELEASE-amd64.raw freebsd-cloud
+    # vm start freebsd-cloud
+
+To list downloaded images:
+
+    # vm img
+    DATASTORE           FILENAME
+    default             CentOS-7-x86_64-GenericCloud-20180930_02.raw
+    default             debian-9-openstack-amd64.qcow2
+    default             Fedora-AtomicHost-28-1.1.x86_64.raw
+    default             FreeBSD-11.2-RELEASE-amd64.raw
+    default             xenial-server-cloudimg-amd64-uefi1.img
     
 ## Adding custom disks
 

lib/vm-base

@@ -51,6 +51,7 @@ zfs::init
 [ ! -e "${vm_dir}/.config/null.iso" ] && touch "${vm_dir}/.config/null.iso"
 [ ! -d "${vm_dir}/.templates" ] && mkdir "${vm_dir}/.templates"
 [ ! -d "${vm_dir}/.iso" ] && mkdir "${vm_dir}/.iso"
+[ ! -d "${vm_dir}/.img" ] && mkdir "${vm_dir}/.img"
 
 # load core configuration
 config::core::load

lib/vm-cmd

@@ -25,7 +25,7 @@
 # POSSIBILITY OF SUCH DAMAGE.
 
 CMD_VALID_LIST="init,switch,datastore,image,get,set,list,create,destroy,rename,install,start,stop,restart"
-CMD_VALID_LIST="${CMD_VALID_LIST},add,reset,poweroff,startall,stopall,console,iso,configure,passthru,_run"
+CMD_VALID_LIST="${CMD_VALID_LIST},add,reset,poweroff,startall,stopall,console,iso,img,configure,passthru,_run"
 CMD_VALID_LIST="${CMD_VALID_LIST},info,clone,snapshot,rollback,send,recv,version,usage"
 
 # cmd: vm ...
@@ -82,6 +82,7 @@ cmd::parse(){
         stopall)   core::stopall ;;
         console)   core::console "$@" ;;
         iso)       core::iso "$@" ;;
+        img)       core::img "$@" ;;
         configure) core::configure "$@" ;;
         passthru)  core::passthru ;;
         _run)      vm::run "$@" ;;
@@ -134,7 +135,7 @@ cmd::parse_datastore(){
     local _cmd
 
     # try to find a matching command
-    cmd::find "_cmd" "$1" "list,add,remove,iso" || util::usage
+    cmd::find "_cmd" "$1" "list,add,remove,iso,img" || util::usage
     shift
 
     case "${_cmd}" in
@@ -142,6 +143,7 @@ cmd::parse_datastore(){
         add)    datastore::add "$@" ;;
         remove) datastore::remove "$@" ;;
         iso)    datastore::iso "$@" ;;
+        img)    datastore::img "$@" ;;
         *)      util::err "unknown command '${_user_cmd}'. please run 'vm usage' or view the manpage for help" ;;
     esac
 }

lib/vm-core

@@ -97,15 +97,16 @@ core::list(){
 #
 core::create(){
     local _name _opt _size _vmdir _disk _disk_dev _num=0
-    local _zfs_opts _disk_size _template="default" _ds="default" _ds_path _cpu _memory
+    local _zfs_opts _disk_size _template="default" _ds="default" _ds_path _img _cpu _memory
 
-    while getopts d:t:s: _opt ; do
+    while getopts d:t:s:i: _opt ; do
         case $_opt in
             t) _template=${OPTARG} ;;
             s) _size=${OPTARG} ;;
             d) _ds=${OPTARG} ;;
             c) _cpu=${OPTARG} ;;
             m) _memory=${OPTARG} ;;
+            i) _img=${OPTARG} ;;
             *) util::usage ;;
         esac
     done
@@ -159,9 +160,11 @@ core::create(){
         case "${_disk_dev}" in 
             zvol)
                 zfs::make_zvol "${VM_DS_ZFS_DATASET}/${_name}/${_disk}" "${_disk_size}" "0" "${_zfs_opts}"
+                [ $_num -eq 0 ] && [ ! -z "$_img" ] && core::write_img "/dev/zvol/${VM_DS_ZFS_DATASET}/${_name}/${_disk}" "${_img}"
                 ;;			
             sparse-zvol)
                 zfs::make_zvol "${VM_DS_ZFS_DATASET}/${_name}/${_disk}" "${_disk_size}" "1" "${_zfs_opts}"
+                [ $_num -eq 0 ] && [ ! -z "$_img" ] && core::write_img "/dev/zvol/${VM_DS_ZFS_DATASET}/${_name}/${_disk}" "${_img}"
                 ;;
             *)
                 truncate -s "${_disk_size}" "${VM_DS_PATH}/${_name}/${_disk}"
@@ -169,6 +172,7 @@ core::create(){
 
                 # make sure only owner can read the disk image
                 chmod 600 "${VM_DS_PATH}/${_name}/${_disk}"
+                [ $_num -eq 0 ] && [ ! -z "$_img" ] && core::write_img "${VM_DS_PATH}/${_name}/${_disk}" "${_img}"
                 ;;
         esac
 
@@ -181,6 +185,27 @@ core::create(){
         config::get "_disk_dev" "disk${_num}_dev"
         config::get "_disk_size" "disk${_num}_size" "20G"
     done
+
+    exit 0
+}
+
+# write cloud image to disk image
+#
+# @private
+# @param string _disk_dev device to write image to
+# @param string _img the img file in $vm_dir/.img to use
+#
+core::write_img(){
+    local _disk_dev _img _imgpath
+
+    cmd::parse_args "$@"
+    shift $?
+    _disk_dev="${1}"
+    _img="$2"
+
+    # just run start with an iso
+    datastore::img_find "_imgpath" "${_img}" || util::err "unable to locate img file - '${_img}'"
+    qemu-img dd -O raw if="${_imgpath}" of="${_disk_dev}" bs=1M
 }
 
 # 'vm add'
@@ -669,6 +694,8 @@ core::destroy(){
 
     zfs::destroy_dataset "${VM_DS_ZFS_DATASET:?}/${_name:?}"
     [ -e "${VM_DS_PATH}/${_name}" ] && rm -R "${VM_DS_PATH:?}/${_name:?}"
+
+    exit 0
 }
 
 # 'vm rename'
@@ -796,6 +823,59 @@ core::iso(){
     fi
 }
 
+# uncompress cloud image
+#
+# @private
+# @param string _filepath path to file to decompress
+#
+core::decompress(){
+    local _filepath
+
+    cmd::parse_args "$@"
+    shift $?
+    _filepath="${1}"
+
+    if echo "${_filepath}" | grep "\.xz$" > /dev/null; then
+        xz -d "${_filepath}"
+    elif echo "${_filepath}" | grep "\.tar\.gz$" > /dev/null; then
+        tar Szxf "${_filepath}" -C "$(dirname "${_filepath}")"
+        rm -f "${_filepath}"
+    elif echo "${_filepath}" | grep "\.gz$" > /dev/null; then
+        gunzip "${_filepath}"
+    fi
+}
+
+# 'vm img'
+# list cloud images or get a new one
+#
+# @param string _url if specified, the url will be fetch'ed into $vm_dir/.img
+#
+core::img(){
+    local _url _ds="default" _filename
+    if ! which qemu-img > /dev/null; then
+        util::err "Error: qemu-img is required to work with cloud images! Run 'pkg install qemu-utils'."
+    fi
+
+    while getopts d:u _opt ; do
+        case $_opt in
+            d) _ds=${OPTARG} ;;
+            *) util::usage ;;
+        esac
+    done
+
+    shift $((OPTIND - 1))
+    _url=$1
+
+    if [ -n "${_url}" ]; then
+        datastore::get_img "${_ds}" || util::err "unable to locate path for datastore '${_ds}'"
+        _filename=$(basename "${_url}")
+        fetch -o "${VM_DS_PATH}" "${_url}"
+        core::decompress "${VM_DS_PATH}/${_filename}"
+    else
+        datastore::img_list
+    fi
+}
+
 # 'vm passthru'
 # show a list of available passthrough devices
 # and their device number

lib/vm-datastore

@@ -63,6 +63,10 @@ datastore::list(){
             _type="iso"
             _path="${_spec#*:}"
             _dataset="-"
+        elif [ "${_spec%%:*}" = "img" ]; then
+            _type="img"
+            _path="${_spec#*:}"
+            _dataset="-"
         else
             _type="directory"
             _path="${_spec}"
@@ -157,7 +161,7 @@ datastore::__resolve_path(){
             setvar "${_var}" "${_mount}"
             return 0
         fi
-    elif [ "${_spec%%:*}" = "iso" ]; then
+    elif [ "${_spec%%:*}" = "iso" ] || [ "${_spec%%:*}" = "img" ]; then
         setvar "${_var}" "${_spec#*:}"
         return 0
     else
@@ -208,7 +212,9 @@ datastore::get_guest(){
             [ "${_ds}" = "default" ] && continue
 
             config::core::get "_spec" "path_${_ds}"
-            [ "${_spec%%:*}" = "iso" ] && continue
+            if [ "${_spec%%:*}" = "iso" ] || [ "${_spec%%:*}" = "img" ]; then
+                continue
+            fi
 
             datastore::__resolve_path "_path" "${_spec}"
 
@@ -254,8 +260,10 @@ datastore::get(){
     config::core::get "_spec" "path_${_ds}"
     [ -z "${_spec}" ] && return 1
 
-    # don't find iso stores
-    [ "${_spec%%:*}" = "iso" ] && return 1
+    # skip iso and img stores
+    if [ "${_spec%%:*}" = "iso" ] || [ "${_spec%%:*}" = "img" ]; then
+        return 1
+    fi 
 
     datastore::__resolve_path "_path" "${_spec}" || return 1
     [ "${_spec%%:*}" = "zfs" ] && _zfs="1" && _dataset="${_spec#*:}"
@@ -291,6 +299,30 @@ datastore::get_iso(){
     VM_DS_PATH="${_path}"
 }
 
+# get the path for an img datastore
+#
+# @param string _ds datastore name
+#
+datastore::get_img(){
+    local _ds="$1"
+
+    # default?
+    # we use the .img subdir in that case
+    if [ "${_ds}" = "default" ]; then
+        VM_DS_PATH="${vm_dir}/.img"
+        return 0
+    fi
+
+    config::core::get "_spec" "path_${_ds}"
+    [ -z "${_spec}" ] && return 1
+
+    # should be an img ds
+    [ "${_spec%%:*}" = "img" ] || return 1
+
+    datastore::__resolve_path "_path" "${_spec}" || return 1
+    VM_DS_PATH="${_path}"
+}
+
 # add a datastore for iso files
 #
 # @param string _name the name of the datastore
@@ -317,6 +349,32 @@ datastore::iso(){
     [ $? -eq 0 ] || util::err "error saving settings to configuration file"
 }
 
+# add a datastore for img files
+#
+# @param string _name the name of the datastore
+# @param string _path filesystem path
+#
+datastore::img(){
+    local _name="$1"
+    local _path="$2"
+
+    [ -z "${_name}" -o -z "${_path}" ] && util::usage
+    util::check_name "${_name}" || util::err "invalid datastore name - '${_name}'"
+
+    # check name not in use
+    for _curr in ${VM_DATASTORE_LIST}; do
+        [ "${_curr}" = "${_name}" ] && util::err "datstore '${_name}' already exists!"
+    done
+
+    # make sure directory exists
+    [ ! -d "${_path}" ] && util::err "specified directory does not appear to be valid"
+
+    # save
+    config::core::set "datastore_list" "${_name}" "1"
+    config::core::set "path_${_name}" "img:${_path}"
+    [ $? -eq 0 ] || util::err "error saving settings to configuration file"
+}
+
 # find an iso file by looking in the default location
 # and any "iso" datastores
 #
@@ -360,6 +418,49 @@ datastore::iso_find(){
     return 1
 }
 
+# find an img file by looking in the default location
+# and any "img" datastores
+#
+# @param string _var variable name to put full img path into
+# @param string _file img filename to look for
+# @return int success if found
+#
+datastore::img_find(){
+    local _var="$1"
+    local _file="$2"
+    local _ds _spec
+
+    # given a full path?
+    if [ -z "${_file%%/*}" ] && [ -r "${_file}" ]; then
+        setvar "${_var}" "${_file}"
+        return 0
+    fi
+
+    # file exists in current dir?
+    if [ -r "$(pwd)/${_file}" ]; then
+        setvar "${_var}" "$(pwd)/${_file}"
+        return 0
+    fi
+
+    # look in default store
+    if [ -r "${vm_dir}/.img/${_file}" ]; then
+        setvar "${_var}" "${vm_dir}/.img/${_file}"
+        return 0
+    fi
+
+    for _ds in ${VM_DATASTORE_LIST}; do
+            config::core::get "_spec" "path_${_ds}"
+            [ "${_spec%%:*}" != "img" ] && continue
+
+            if [ -r "${_spec#*:}/${_file}" ]; then
+                setvar "${_var}" "${_spec#*:}/${_file}"
+                return 0
+            fi
+    done
+
+    return 1
+}
+
 # list iso files
 #
 datastore::iso_list(){
@@ -378,3 +479,22 @@ datastore::iso_list(){
         [ -d "${_spec#*:}" ] && ls -1 ${_spec#*:} | awk '{printf "'${_format}'","'${_ds}'",$1}'
     done
 }
+
+# list img files
+#
+datastore::img_list(){
+    local _ds _spec _format="%-20s%s\n"
+
+    printf "${_format}" "DATASTORE" "FILENAME"
+
+    # look for default img location
+    [ -d "${vm_dir}/.img" ] && ls -1 "${vm_dir}/.img" | awk '{printf "'${_format}'","default",$1}'
+
+    # look for img datastores
+    for _ds in ${VM_DATASTORE_LIST}; do
+        config::core::get "_spec" "path_${_ds}"
+        [ "${_spec%%:*}" != "img" ] && continue
+
+        [ -d "${_spec#*:}" ] && ls -1 ${_spec#*:} | awk '{printf "'${_format}'","'${_ds}'",$1}'
+    done
+}

lib/vm-util

@@ -176,6 +176,7 @@ Usage: vm ...
     snapshot [-f] <name[@snapshot]>
     rollback [-r] <name@snapshot>
     iso [url]
+    img [url]
     image list
     image create [-d description] [-u] <name>
     image destroy <uuid>

sample-templates/linux-zvol.conf

@@ -0,0 +1,8 @@
+loader="grub"
+cpu=1
+memory=512M
+network0_type="virtio-net"
+network0_switch="public"
+disk0_name="disk0"
+disk0_dev="sparse-zvol"
+disk0_type="virtio-blk"