From 69295546526f911ccc94fc13f73cd66e7373fa40 Mon Sep 17 00:00:00 2001 From: Serhiy Berezin Date: Wed, 30 Nov 2016 23:32:48 +0000 Subject: [PATCH] LUKS/cryptsetup with SSH unlock, LVM, NBD, BTRFS, XFS - all for state partiton fully supported --- cmd/control/bootstrap.go | 49 +++++++++++++++++++++++++++--- cmd/control/install.go | 16 ++++++++-- cmd/control/user_docker.go | 19 ++++++++++++ config/schema.go | 17 +++++++++++ config/types.go | 11 +++++++ images/02-bootstrap/Dockerfile | 4 +-- images/02-bootstrap/auto-format.sh | 8 +++-- images/02-bootstrap/unlock | 10 ++++++ init/bootstrap.go | 5 ++- os-config.tpl.yml | 10 +++--- scripts/installer/lay-down-os | 7 ++++- scripts/schema.json | 17 +++++++++++ util/cutil.go | 8 +++++ 13 files changed, 162 insertions(+), 19 deletions(-) create mode 100755 images/02-bootstrap/unlock diff --git a/cmd/control/bootstrap.go b/cmd/control/bootstrap.go index 7d3225672..427fc7463 100644 --- a/cmd/control/bootstrap.go +++ b/cmd/control/bootstrap.go @@ -22,9 +22,28 @@ func bootstrapAction(c *cli.Context) error { cfg := config.LoadConfig() if cfg.Rancher.State.MdadmScan { - if err := mdadmScan(); err != nil { - log.Errorf("Failed to run mdadm scan: %v", err) + cmdRun("mdadm", "--assemble", "--scan") + } + + if cfg.Rancher.State.NetConf || cfg.Rancher.State.Nbd.Host != "" || cfg.Rancher.State.Decrypt { + if err := cmdStart("netconf"); err != nil { + log.Errorf("Failed to start netconf: %v", err) + } + } + + if cfg.Rancher.State.Nbd.Host != "" { + nbd := cfg.Rancher.State.Nbd + args := []string { nbd.Host } + if nbd.Port != 0 { + args = append(args, string(nbd.Port)) } + if nbd.Name != "" { + args = append(args, "-name", nbd.Name) + } + if nbd.BlockSize != 0 { + args = append(args, "-block-size", string(nbd.BlockSize)) + } + cmdRun("nbd-client", args...) } stateScript := cfg.Rancher.State.Script @@ -36,6 +55,15 @@ func bootstrapAction(c *cli.Context) error { util.RunCommandSequence(cfg.Bootcmd) + if cfg.Rancher.State.Decrypt { + cmdRun("ros", "console-init") + } + + if cfg.Rancher.State.LvmScan { + cmdRun("pvscan", "--activate", "ay") + cmdRun("vgchange", "--activate", "ay") + } + if cfg.Rancher.State.Dev != "" && cfg.Rancher.State.Wait { waitForRoot(cfg) } @@ -54,11 +82,22 @@ func bootstrapAction(c *cli.Context) error { return nil } -func mdadmScan() error { - cmd := exec.Command("mdadm", "--assemble", "--scan") +func cmdRun(cmdName string, args ...string) bool { + cmd := exec.Command(cmdName, args...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr - return cmd.Run() + if err := cmd.Run(); err != nil { + log.Errorf("Failed to run %v %v: %v", cmdName, args, err) + return false + } + return true +} + +func cmdStart(cmdName string, args ...string) error { + cmd := exec.Command(cmdName, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Start() } func runStateScript(script string) error { diff --git a/cmd/control/install.go b/cmd/control/install.go index 669c7be3a..c2a1d0a0d 100644 --- a/cmd/control/install.go +++ b/cmd/control/install.go @@ -39,6 +39,10 @@ var installCommand = cli.Command{ Name: "device, d", Usage: "storage device", }, + cli.StringFlag{ + Name: "file-system-type, s", + Usage: "ext4 or btrfs", + }, cli.BoolFlag{ Name: "force, f", Usage: "[ DANGEROUS! Data loss can happen ] partition/format without prompting", @@ -63,6 +67,12 @@ func installAction(c *cli.Context) error { log.Fatal("Can not proceed without -d specified") } + fileSystemType := c.String("file-system-type") + if fileSystemType == "" { + log.Info("No file system specified...defaulting to ext4") + fileSystemType = "ext4" + } + image := c.String("image") cfg := config.LoadConfig() if image == "" { @@ -90,14 +100,14 @@ func installAction(c *cli.Context) error { force := c.Bool("force") reboot := !c.Bool("no-reboot") - if err := runInstall(image, installType, cloudConfig, device, append, force, reboot); err != nil { + if err := runInstall(image, installType, cloudConfig, device, fileSystemType, append, force, reboot); err != nil { log.WithFields(log.Fields{"err": err}).Fatal("Failed to run install") } return nil } -func runInstall(image, installType, cloudConfig, device, append string, force, reboot bool) error { +func runInstall(image, installType, cloudConfig, device, fileSystemType, append string, force, reboot bool) error { fmt.Printf("Installing from %s\n", image) if !force { @@ -115,7 +125,7 @@ func runInstall(image, installType, cloudConfig, device, append string, force, r } } cmd := exec.Command("system-docker", "run", "--net=host", "--privileged", "--volumes-from=user-volumes", - "--volumes-from=command-volumes", image, "-d", device, "-t", installType, "-c", cloudConfig, "-a", append) + "--volumes-from=command-volumes", image, "-d", device, "-s", fileSystemType, "-t", installType, "-c", cloudConfig, "-a", append) cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr if err := cmd.Run(); err != nil { return err diff --git a/cmd/control/user_docker.go b/cmd/control/user_docker.go index da3e616cb..0a707d620 100644 --- a/cmd/control/user_docker.go +++ b/cmd/control/user_docker.go @@ -20,6 +20,7 @@ import ( rosDocker "github.com/rancher/os/docker" "github.com/rancher/os/log" "github.com/rancher/os/util" + "github.com/docker/docker/daemon/graphdriver" ) const ( @@ -48,6 +49,24 @@ func userDockerAction(c *cli.Context) error { cfg := config.LoadConfig() + if cfg.Rancher.Docker.StorageDriver == "" { + if cfg.Rancher.Docker.Graph == "" { + cfg.Rancher.Docker.Graph = "/var/lib/docker" + } + + fsMagic, err := graphdriver.GetFSMagic(cfg.Rancher.Docker.Graph) + if err == nil { + switch fsMagic { + case graphdriver.FsMagicBtrfs: + cfg.Rancher.Docker.StorageDriver = "btrfs" + case graphdriver.FsMagicZfs: + cfg.Rancher.Docker.StorageDriver = "zfs" + default: + cfg.Rancher.Docker.StorageDriver = "overlay" + } + } + } + return startDocker(cfg) } diff --git a/config/schema.go b/config/schema.go index 0876f1e72..29285555c 100644 --- a/config/schema.go +++ b/config/schema.go @@ -148,6 +148,19 @@ var schema = `{ } }, + "nbd_config": { + "id": "#/definitions/nbd_config", + "type": "object", + "additionalProperties": false, + + "properties": { + "host": {"type": "string"}, + "port": {"type": "integer"}, + "name": {"type": "string"}, + "block_size": {"type": "integer"} + } + }, + "state_config": { "id": "#/definitions/state_config", "type": "object", @@ -161,6 +174,10 @@ var schema = `{ "required": {"type": "boolean"}, "autoformat": {"$ref": "#/definitions/list_of_strings"}, "mdadm_scan": {"type": "boolean"}, + "lvm_scan": {"type": "boolean"}, + "net_conf": {"type": "boolean"}, + "nbd_conf": {"$ref": "#/definitions/nbd_config"}, + "decrypt": {"type": "boolean"}, "script": {"type": "string"}, "oem_fstype": {"type": "string"}, "oem_dev": {"type": "string"} diff --git a/config/types.go b/config/types.go index 588578ae8..9cf62a226 100644 --- a/config/types.go +++ b/config/types.go @@ -207,6 +207,13 @@ type SSHConfig struct { Keys map[string]string `yaml:"keys,omitempty"` } +type NbdConfig struct { + Host string `yaml:"host,omitempty"` + Port uint16 `yaml:"port,omitempty"` + Name string `yaml:"name,omitempty"` + BlockSize uint16 `yaml:"block_size,omitempty"` +} + type StateConfig struct { Directory string `yaml:"directory,omitempty"` FsType string `yaml:"fstype,omitempty"` @@ -215,6 +222,10 @@ type StateConfig struct { Required bool `yaml:"required,omitempty"` Autoformat []string `yaml:"autoformat,omitempty"` MdadmScan bool `yaml:"mdadm_scan,omitempty"` + LvmScan bool `yaml:"lvm_scan,omitempty"` + NetConf bool `yaml:"net_conf,omitempty"` + Decrypt bool `yaml:"decrypt,omitempty"` + Nbd NbdConfig`yaml:"nbd,omitempty"` Script string `yaml:"script,omitempty"` OemFsType string `yaml:"oem_fstype,omitempty"` OemDev string `yaml:"oem_dev,omitempty"` diff --git a/images/02-bootstrap/Dockerfile b/images/02-bootstrap/Dockerfile index 554d121e4..a6ca1be37 100644 --- a/images/02-bootstrap/Dockerfile +++ b/images/02-bootstrap/Dockerfile @@ -1,3 +1,3 @@ -FROM rancher/os-base -COPY auto-format.sh /usr/sbin/ +FROM rancher/os-console +COPY auto-format.sh unlock /usr/sbin/ COPY od-1m0 / diff --git a/images/02-bootstrap/auto-format.sh b/images/02-bootstrap/auto-format.sh index 48e7994c0..a053f8d33 100755 --- a/images/02-bootstrap/auto-format.sh +++ b/images/02-bootstrap/auto-format.sh @@ -20,9 +20,11 @@ for dev in ${DEVS[@]}; do if [ -e "/userdata.tar" ]; then - mkfs.ext4 -L B2D_STATE ${dev} + fstype=`ros config get rancher.state.fstype` + ( [ $fstype == auto ] || [ $fstype == "" ] || ! which mkfs.$fstype ) && fstype=ext4 + mkfs.$fstype -L B2D_STATE ${dev} mkdir -p /mnt/new-root - mount -t ext4 ${dev} /mnt/new-root + mount -t $fstype ${dev} /mnt/new-root pushd /mnt/new-root mkdir -p ./var/lib/rancher/conf/cloud-config.d echo $(tar -xvf /userdata.tar) @@ -54,7 +56,7 @@ EOF popd umount /mnt/new-root else - mkfs.ext4 -L RANCHER_STATE ${dev} + mkfs.$fstype -L RANCHER_STATE ${dev} fi # do not check another device diff --git a/images/02-bootstrap/unlock b/images/02-bootstrap/unlock new file mode 100755 index 000000000..db2ec4e94 --- /dev/null +++ b/images/02-bootstrap/unlock @@ -0,0 +1,10 @@ +#!/bin/bash +set -e +[ "$UID" -eq 0 ] || exec sudo bash "$0" "$@" + +read -p "Passphrase please:" pass +for blk in `blkid -o list | grep LUKS | cut -f 1 -d ' '`; do + echo $pass | cryptsetup open --allow-discards $blk luks-`basename $blk` +done + +[[ $1 == stay ]] || kill `ps -o pid,comm | grep respawn | cut -f 3 -d ' '` diff --git a/init/bootstrap.go b/init/bootstrap.go index d5142e797..1e05101d4 100644 --- a/init/bootstrap.go +++ b/init/bootstrap.go @@ -11,7 +11,10 @@ import ( ) func bootstrapServices(cfg *config.CloudConfig) (*config.CloudConfig, error) { - if (len(cfg.Rancher.State.Autoformat) == 0 || util.ResolveDevice(cfg.Rancher.State.Dev) != "") && len(cfg.Bootcmd) == 0 { + if len(cfg.Bootcmd) == 0 && + len(cfg.Rancher.State.Autoformat) == 0 && + cfg.Rancher.State.Nbd.Host == "" && + util.ResolveDevice(cfg.Rancher.State.Dev) != "" { return cfg, nil } log.Info("Running Bootstrap") diff --git a/os-config.tpl.yml b/os-config.tpl.yml index 7ad2606e1..70eef8c9a 100644 --- a/os-config.tpl.yml +++ b/os-config.tpl.yml @@ -22,7 +22,10 @@ rancher: io.rancher.os.detach: "false" io.rancher.os.scope: system log_driver: json-file - net: none + net: host + uts: host + pid: host + ipc: host privileged: true volumes: - /dev:/host/dev @@ -30,6 +33,8 @@ rancher: - /lib/firmware:/lib/firmware - /usr/bin/ros:/usr/bin/ros:ro - /usr/share/ros:/usr/share/ros:ro + - /usr/bin/ros:/usr/bin/respawn:ro + - /usr/bin/ros:/usr/sbin/netconf:ro cloud_init_services: cloud-init: image: {{.OS_REPO}}/os-base:{{.VERSION}}{{.SUFFIX}} @@ -55,7 +60,6 @@ rancher: - /var/lib/rancher/conf:/var/lib/rancher/conf bootstrap_docker: bridge: none - storage_driver: overlay restart: false graph: /var/lib/system-docker group: root @@ -305,7 +309,6 @@ rancher: - /var/lib/system-docker:/var/lib/system-docker:shared system_docker: exec: true - storage_driver: overlay restart: false graph: /var/lib/system-docker group: root @@ -326,7 +329,6 @@ rancher: {{else -}} engine: docker-1.11.2 {{end -}} - storage_driver: overlay group: docker host: ["unix:///var/run/docker.sock"] log_opts: diff --git a/scripts/installer/lay-down-os b/scripts/installer/lay-down-os index e64c62d74..581b68a0e 100755 --- a/scripts/installer/lay-down-os +++ b/scripts/installer/lay-down-os @@ -12,6 +12,7 @@ do f) FILES="$OPTARG" ;; c) CLOUD_CONFIG="$OPTARG" ;; d) DEVICE="$OPTARG" ;; + s) FILE_SYSTEM_TYPE="$OPTARG" ;; o) OEM="$OPTARG" ;; p) PARTITION="$OPTARG" ;; r) ROLLBACK_VERSION="$OPTARG" ;; @@ -31,6 +32,7 @@ BOOT=boot/ # TODO: Change this to a number so that users can specify. # Will need to make it so that our builds and packer APIs remain consistent. PARTITION=${PARTITION:=${DEVICE}1} +FILE_SYSTEM_TYPE={$FILE_SYSTEM_TYPE:-ext4} KERNEL_ARGS="rancher.state.dev=LABEL=RANCHER_STATE rancher.state.wait console=${CONSOLE}" device_defined() @@ -44,7 +46,10 @@ device_defined() format_device() { device_defined ${DEVICE} - mkfs.ext4 -F -i 4096 -L RANCHER_STATE ${PARTITION} + case ${FILE_SYSTEM_TYPE} in + btrfs) mkfs.btrfs -f -L RANCHER_STATE ${PARTITION} ;; + xfs) mkfs.xfs -f -i 4096 -L RANCHER_STATE ${PARTITION} ;; + *) mkfs.ext4 -F -i 4096 -L RANCHER_STATE ${PARTITION} ;; } get_dev() diff --git a/scripts/schema.json b/scripts/schema.json index f16d88bbf..41366814b 100644 --- a/scripts/schema.json +++ b/scripts/schema.json @@ -146,6 +146,19 @@ } }, + "nbd_config": { + "id": "#/definitions/nbd_config", + "type": "object", + "additionalProperties": false, + + "properties": { + "host": {"type": "string"}, + "port": {"type": "integer"}, + "name": {"type": "string"}, + "block_size": {"type": "integer"} + } + }, + "state_config": { "id": "#/definitions/state_config", "type": "object", @@ -159,6 +172,10 @@ "required": {"type": "boolean"}, "autoformat": {"$ref": "#/definitions/list_of_strings"}, "mdadm_scan": {"type": "boolean"}, + "lvm_scan": {"type": "boolean"}, + "net_conf": {"type": "boolean"}, + "nbd_conf": {"$ref": "#/definitions/nbd_config"}, + "decrypt": {"type": "boolean"}, "script": {"type": "string"}, "oem_fstype": {"type": "string"}, "oem_dev": {"type": "string"} diff --git a/util/cutil.go b/util/cutil.go index 07d58b6f9..b2af8a6da 100644 --- a/util/cutil.go +++ b/util/cutil.go @@ -20,6 +20,14 @@ func ResolveDevice(spec string) string { defer C.free(unsafe.Pointer(cSpec)) cString := C.blkid_evaluate_spec(cSpec, nil) defer C.free(unsafe.Pointer(cString)) + cProbe := C.blkid_new_probe_from_filename(cString) + if cProbe == nil { + return "" + } + defer C.blkid_free_probe(cProbe) + if C.blkid_probe_get_devno(cProbe) == 0 || C.blkid_probe_get_sectors(cProbe) <= 0 { + return "" + } return C.GoString(cString) }