LUKS/cryptsetup with SSH unlock, LVM, NBD, BTRFS, XFS - all for state…

cmd/control/bootstrap.go

@@ -22,9 +22,28 @@ func bootstrapAction(c *cli.Context) error {
 	cfg := config.LoadConfig()
 
 	if cfg.Rancher.State.MdadmScan {
-		if err := mdadmScan(); err != nil {
-			log.Errorf("Failed to run mdadm scan: %v", err)
+		cmdRun("mdadm", "--assemble", "--scan")
+	}
+
+	if cfg.Rancher.State.NetConf || cfg.Rancher.State.Nbd.Host != "" || cfg.Rancher.State.Decrypt {
+                if err := cmdStart("netconf"); err != nil {
+                        log.Errorf("Failed to start netconf: %v", err)
+                }
+	}
+
+	if cfg.Rancher.State.Nbd.Host != "" {
+		nbd := cfg.Rancher.State.Nbd
+		args := []string { nbd.Host }
+		if nbd.Port != 0 {
+			args = append(args, string(nbd.Port))
 		}
+		if nbd.Name != "" {
+			args = append(args, "-name", nbd.Name)
+		}
+		if nbd.BlockSize != 0 {
+			args = append(args, "-block-size", string(nbd.BlockSize))
+		}
+                cmdRun("nbd-client", args...)
 	}
 
 	stateScript := cfg.Rancher.State.Script
@@ -36,6 +55,15 @@ func bootstrapAction(c *cli.Context) error {
 
 	util.RunCommandSequence(cfg.Bootcmd)
 
+	if cfg.Rancher.State.Decrypt {
+		cmdRun("ros", "console-init")
+	}
+
+	if cfg.Rancher.State.LvmScan {
+		cmdRun("pvscan", "--activate", "ay")
+		cmdRun("vgchange", "--activate", "ay")
+	}
+
 	if cfg.Rancher.State.Dev != "" && cfg.Rancher.State.Wait {
 		waitForRoot(cfg)
 	}
@@ -54,11 +82,22 @@ func bootstrapAction(c *cli.Context) error {
 	return nil
 }
 
-func mdadmScan() error {
-	cmd := exec.Command("mdadm", "--assemble", "--scan")
+func cmdRun(cmdName string, args ...string) bool {
+	cmd := exec.Command(cmdName, args...)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
-	return cmd.Run()
+	if err := cmd.Run(); err != nil {
+		log.Errorf("Failed to run %v %v: %v", cmdName, args, err)
+		return false
+	}
+	return true
+}
+
+func cmdStart(cmdName string, args ...string) error {
+        cmd := exec.Command(cmdName, args...)
+        cmd.Stdout = os.Stdout
+        cmd.Stderr = os.Stderr
+        return cmd.Start()
 }
 
 func runStateScript(script string) error {

cmd/control/install.go

@@ -39,6 +39,10 @@ var installCommand = cli.Command{
 			Name:  "device, d",
 			Usage: "storage device",
 		},
+		cli.StringFlag{
+                        Name:  "file-system-type, s",
+                        Usage: "ext4 or btrfs",
+                },
 		cli.BoolFlag{
 			Name:  "force, f",
 			Usage: "[ DANGEROUS! Data loss can happen ] partition/format without prompting",
@@ -63,6 +67,12 @@ func installAction(c *cli.Context) error {
 		log.Fatal("Can not proceed without -d <dev> specified")
 	}
 
+	fileSystemType := c.String("file-system-type")
+	if fileSystemType == "" {
+		log.Info("No file system specified...defaulting to ext4")
+		fileSystemType = "ext4"
+	}
+
 	image := c.String("image")
 	cfg := config.LoadConfig()
 	if image == "" {
@@ -90,14 +100,14 @@ func installAction(c *cli.Context) error {
 	force := c.Bool("force")
 	reboot := !c.Bool("no-reboot")
 
-	if err := runInstall(image, installType, cloudConfig, device, append, force, reboot); err != nil {
+	if err := runInstall(image, installType, cloudConfig, device, fileSystemType, append, force, reboot); err != nil {
 		log.WithFields(log.Fields{"err": err}).Fatal("Failed to run install")
 	}
 
 	return nil
 }
 
-func runInstall(image, installType, cloudConfig, device, append string, force, reboot bool) error {
+func runInstall(image, installType, cloudConfig, device, fileSystemType, append string, force, reboot bool) error {
 	fmt.Printf("Installing from %s\n", image)
 
 	if !force {
@@ -115,7 +125,7 @@ func runInstall(image, installType, cloudConfig, device, append string, force, r
 		}
 	}
 	cmd := exec.Command("system-docker", "run", "--net=host", "--privileged", "--volumes-from=user-volumes",
-		"--volumes-from=command-volumes", image, "-d", device, "-t", installType, "-c", cloudConfig, "-a", append)
+		"--volumes-from=command-volumes", image, "-d", device, "-s", fileSystemType, "-t", installType, "-c", cloudConfig, "-a", append)
 	cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
 	if err := cmd.Run(); err != nil {
 		return err

cmd/control/user_docker.go

@@ -20,6 +20,7 @@ import (
 	rosDocker "github.com/rancher/os/docker"
 	"github.com/rancher/os/log"
 	"github.com/rancher/os/util"
+	"github.com/docker/docker/daemon/graphdriver"
 )
 
 const (
@@ -48,6 +49,24 @@ func userDockerAction(c *cli.Context) error {
 
 	cfg := config.LoadConfig()
 
+	if cfg.Rancher.Docker.StorageDriver == "" {
+		if cfg.Rancher.Docker.Graph == "" {
+			cfg.Rancher.Docker.Graph =  "/var/lib/docker"
+		}
+
+		fsMagic, err := graphdriver.GetFSMagic(cfg.Rancher.Docker.Graph)
+		if err == nil {
+			switch fsMagic {
+			case graphdriver.FsMagicBtrfs:
+				cfg.Rancher.Docker.StorageDriver = "btrfs"
+			case graphdriver.FsMagicZfs:
+				cfg.Rancher.Docker.StorageDriver = "zfs"
+			default:
+				cfg.Rancher.Docker.StorageDriver = "overlay"
+			}
+		}
+	}
+
 	return startDocker(cfg)
 }
 

config/schema.go

@@ -148,6 +148,19 @@ var schema = `{
       }
     },
 
+    "nbd_config": {
+      "id": "#/definitions/nbd_config",
+      "type": "object",
+      "additionalProperties": false,
+
+      "properties": {
+        "host": {"type": "string"},
+        "port": {"type": "integer"},
+        "name": {"type": "string"},
+        "block_size": {"type": "integer"}
+      }
+    },
+
     "state_config": {
       "id": "#/definitions/state_config",
       "type": "object",
@@ -161,6 +174,10 @@ var schema = `{
         "required": {"type": "boolean"},
         "autoformat": {"$ref": "#/definitions/list_of_strings"},
         "mdadm_scan": {"type": "boolean"},
+        "lvm_scan": {"type": "boolean"},
+        "net_conf": {"type": "boolean"},
+        "nbd_conf": {"$ref": "#/definitions/nbd_config"},
+        "decrypt": {"type": "boolean"},
         "script": {"type": "string"},
         "oem_fstype": {"type": "string"},
         "oem_dev": {"type": "string"}

config/types.go

@@ -207,6 +207,13 @@ type SSHConfig struct {
 	Keys map[string]string `yaml:"keys,omitempty"`
 }
 
+type NbdConfig struct {
+	Host       string `yaml:"host,omitempty"`
+	Port       uint16 `yaml:"port,omitempty"`
+	Name       string `yaml:"name,omitempty"`
+	BlockSize  uint16 `yaml:"block_size,omitempty"` 
+}
+
 type StateConfig struct {
 	Directory  string   `yaml:"directory,omitempty"`
 	FsType     string   `yaml:"fstype,omitempty"`
@@ -215,6 +222,10 @@ type StateConfig struct {
 	Required   bool     `yaml:"required,omitempty"`
 	Autoformat []string `yaml:"autoformat,omitempty"`
 	MdadmScan  bool     `yaml:"mdadm_scan,omitempty"`
+	LvmScan    bool     `yaml:"lvm_scan,omitempty"`
+	NetConf    bool     `yaml:"net_conf,omitempty"`
+	Decrypt    bool     `yaml:"decrypt,omitempty"`
+	Nbd        NbdConfig`yaml:"nbd,omitempty"`
 	Script     string   `yaml:"script,omitempty"`
 	OemFsType  string   `yaml:"oem_fstype,omitempty"`
 	OemDev     string   `yaml:"oem_dev,omitempty"`

images/02-bootstrap/Dockerfile

@@ -1,3 +1,3 @@
-FROM rancher/os-base
-COPY auto-format.sh /usr/sbin/
+FROM rancher/os-console
+COPY auto-format.sh unlock /usr/sbin/
 COPY od-1m0 /

images/02-bootstrap/auto-format.sh

@@ -20,9 +20,11 @@ for dev in ${DEVS[@]}; do
 
 
         if [ -e "/userdata.tar" ]; then
-            mkfs.ext4 -L B2D_STATE ${dev}
+            fstype=`ros config get rancher.state.fstype`
+            ( [ $fstype == auto ] || [ $fstype == "" ] || ! which mkfs.$fstype ) && fstype=ext4
+            mkfs.$fstype -L B2D_STATE ${dev}
             mkdir -p /mnt/new-root
-            mount -t ext4 ${dev} /mnt/new-root
+            mount -t $fstype ${dev} /mnt/new-root
             pushd /mnt/new-root
             mkdir -p ./var/lib/rancher/conf/cloud-config.d
             echo $(tar -xvf /userdata.tar)
@@ -54,7 +56,7 @@ EOF
             popd
             umount /mnt/new-root
         else
-            mkfs.ext4 -L RANCHER_STATE ${dev}
+            mkfs.$fstype -L RANCHER_STATE ${dev}
         fi
 
         # do not check another device

images/02-bootstrap/unlock

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -e
+[ "$UID" -eq 0 ] || exec sudo bash "$0" "$@"
+
+read -p "Passphrase please:" pass
+for blk in `blkid -o list | grep LUKS | cut -f 1 -d ' '`; do
+  echo $pass | cryptsetup open --allow-discards $blk luks-`basename $blk`
+done
+
+[[ $1 == stay ]] || kill `ps -o pid,comm | grep respawn | cut -f 3 -d ' '`

init/bootstrap.go

@@ -11,7 +11,10 @@ import (
 )
 
 func bootstrapServices(cfg *config.CloudConfig) (*config.CloudConfig, error) {
-	if (len(cfg.Rancher.State.Autoformat) == 0 || util.ResolveDevice(cfg.Rancher.State.Dev) != "") && len(cfg.Bootcmd) == 0 {
+	if len(cfg.Bootcmd) == 0 &&
+	   len(cfg.Rancher.State.Autoformat) == 0 &&
+	   cfg.Rancher.State.Nbd.Host == "" &&
+	   util.ResolveDevice(cfg.Rancher.State.Dev) != "" {
 		return cfg, nil
 	}
 	log.Info("Running Bootstrap")

os-config.tpl.yml

@@ -22,14 +22,19 @@ rancher:
         io.rancher.os.detach: "false"
         io.rancher.os.scope: system
       log_driver: json-file
-      net: none
+      net: host
+      uts: host
+      pid: host
+      ipc: host
       privileged: true
       volumes:
       - /dev:/host/dev
       - /lib/modules:/lib/modules
       - /lib/firmware:/lib/firmware
       - /usr/bin/ros:/usr/bin/ros:ro
       - /usr/share/ros:/usr/share/ros:ro
+      - /usr/bin/ros:/usr/bin/respawn:ro
+      - /usr/bin/ros:/usr/sbin/netconf:ro
   cloud_init_services:
     cloud-init:
       image: {{.OS_REPO}}/os-base:{{.VERSION}}{{.SUFFIX}}
@@ -55,7 +60,6 @@ rancher:
       - /var/lib/rancher/conf:/var/lib/rancher/conf
   bootstrap_docker:
     bridge: none
-    storage_driver: overlay
     restart: false
     graph: /var/lib/system-docker
     group: root
@@ -305,7 +309,6 @@ rancher:
       - /var/lib/system-docker:/var/lib/system-docker:shared
   system_docker:
     exec: true
-    storage_driver: overlay
     restart: false
     graph: /var/lib/system-docker
     group: root
@@ -326,7 +329,6 @@ rancher:
     {{else -}}
     engine: docker-1.11.2
     {{end -}}
-    storage_driver: overlay
     group: docker
     host: ["unix:///var/run/docker.sock"]
     log_opts:

scripts/installer/lay-down-os

@@ -12,6 +12,7 @@ do
         f) FILES="$OPTARG" ;;
         c) CLOUD_CONFIG="$OPTARG" ;;
         d) DEVICE="$OPTARG" ;;
+        s) FILE_SYSTEM_TYPE="$OPTARG" ;;
         o) OEM="$OPTARG" ;;
         p) PARTITION="$OPTARG" ;;
         r) ROLLBACK_VERSION="$OPTARG" ;;
@@ -31,6 +32,7 @@ BOOT=boot/
 # TODO: Change this to a number so that users can specify.
 # Will need to make it so that our builds and packer APIs remain consistent.
 PARTITION=${PARTITION:=${DEVICE}1}
+FILE_SYSTEM_TYPE={$FILE_SYSTEM_TYPE:-ext4}
 KERNEL_ARGS="rancher.state.dev=LABEL=RANCHER_STATE rancher.state.wait console=${CONSOLE}"
 
 device_defined()
@@ -44,7 +46,10 @@ device_defined()
 format_device()
 {
     device_defined ${DEVICE}
-    mkfs.ext4 -F -i 4096 -L RANCHER_STATE ${PARTITION}
+    case ${FILE_SYSTEM_TYPE} in
+       btrfs) mkfs.btrfs -f         -L RANCHER_STATE ${PARTITION} ;;
+       xfs)   mkfs.xfs   -f -i 4096 -L RANCHER_STATE ${PARTITION} ;;
+       *)     mkfs.ext4  -F -i 4096 -L RANCHER_STATE ${PARTITION} ;;
 }
 
 get_dev()

scripts/schema.json

@@ -146,6 +146,19 @@
       }
     },
 
+    "nbd_config": {
+      "id": "#/definitions/nbd_config",
+      "type": "object",
+      "additionalProperties": false,
+
+      "properties": {
+        "host": {"type": "string"},
+        "port": {"type": "integer"},
+        "name": {"type": "string"},
+        "block_size": {"type": "integer"}
+      }
+    },
+
     "state_config": {
       "id": "#/definitions/state_config",
       "type": "object",
@@ -159,6 +172,10 @@
         "required": {"type": "boolean"},
         "autoformat": {"$ref": "#/definitions/list_of_strings"},
         "mdadm_scan": {"type": "boolean"},
+        "lvm_scan": {"type": "boolean"},
+        "net_conf": {"type": "boolean"},
+        "nbd_conf": {"$ref": "#/definitions/nbd_config"},
+        "decrypt": {"type": "boolean"},
         "script": {"type": "string"},
         "oem_fstype": {"type": "string"},
         "oem_dev": {"type": "string"}

util/cutil.go

@@ -20,6 +20,14 @@ func ResolveDevice(spec string) string {
 	defer C.free(unsafe.Pointer(cSpec))
 	cString := C.blkid_evaluate_spec(cSpec, nil)
 	defer C.free(unsafe.Pointer(cString))
+	cProbe := C.blkid_new_probe_from_filename(cString)
+	if cProbe == nil {
+		return ""
+	}
+	defer C.blkid_free_probe(cProbe)
+	if C.blkid_probe_get_devno(cProbe) == 0 || C.blkid_probe_get_sectors(cProbe) <= 0 {
+		return ""
+	}
 	return C.GoString(cString)
 }