package daemon

import (
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"strings"
	"sync"
	"syscall"
	"time"

	imgref "github.com/containers/image/docker/reference"
	ignv2 "github.com/coreos/ignition/config/v2_2"
	ignv2_2types "github.com/coreos/ignition/config/v2_2/types"
	"github.com/golang/glog"
	drain "github.com/openshift/kubernetes-drain"
	"github.com/openshift/machine-config-operator/lib/resourceread"
	mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
	"github.com/openshift/machine-config-operator/pkg/daemon/constants"
	mcfgclientset "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned"
	mcfginformersv1 "github.com/openshift/machine-config-operator/pkg/generated/informers/externalversions/machineconfiguration.openshift.io/v1"
	mcfglistersv1 "github.com/openshift/machine-config-operator/pkg/generated/listers/machineconfiguration.openshift.io/v1"
	"github.com/pkg/errors"
	"github.com/vincent-petithory/dataurl"
	"golang.org/x/time/rate"
	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/util/diff"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	coreinformersv1 "k8s.io/client-go/informers/core/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/scheme"
	clientsetcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
	corelisterv1 "k8s.io/client-go/listers/core/v1"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/record"
	"k8s.io/client-go/util/workqueue"
)

// Daemon is the dispatch point for the functions of the agent on the
// machine. it keeps track of connections and the current state of the update
// process.
type Daemon struct {
	// name is the node name.
	name string
	// OperatingSystem the operating system the MCD is running on
	OperatingSystem string

	// NodeUpdaterClient an instance of the client which interfaces with host content deployments
	NodeUpdaterClient NodeUpdaterClient

	// bootID is a unique value per boot (generated by the kernel)
	bootID string

	// bootedOSImageURL is the currently booted URL of the operating system
	bootedOSImageURL string

	// kubeClient allows interaction with Kubernetes, including the node we are running on.
	kubeClient kubernetes.Interface

	mcClient mcfgclientset.Interface

	// recorder sends events to the apiserver
	recorder record.EventRecorder

	// rootMount is the location for the MCD to chroot in
	rootMount string

	// nodeLister is used to watch for updates via the informer
	nodeLister       corelisterv1.NodeLister
	nodeListerSynced cache.InformerSynced

	mcLister       mcfglistersv1.MachineConfigLister
	mcListerSynced cache.InformerSynced

	// onceFrom defines where the source config is to run the daemon once and exit
	onceFrom string

	// skipReboot skips the reboot after a sync, only valid with onceFrom != ""
	skipReboot bool

	kubeletHealthzEnabled  bool
	kubeletHealthzEndpoint string

	updateActive     bool
	updateActiveLock sync.Mutex

	nodeWriter NodeWriter

	// channel used by callbacks to signal Run() of an error
	exitCh chan<- error

	// channel used to ensure all spawned goroutines exit when we exit.
	stopCh <-chan struct{}

	// node is the current instance of the node being processed through handleNodeUpdate
	// or the very first instance grabbed when the daemon starts
	node *corev1.Node

	// remove the funcs below once proper e2e testing is done for updating ssh keys
	atomicSSHKeysWriter func(ignv2_2types.PasswdUser, string) error

	queue       workqueue.RateLimitingInterface
	enqueueNode func(*corev1.Node)
	syncHandler func(node string) error

	booting bool

	currentConfigPath string

	loggerSupportsJournal bool
}

const (
	// pathSystemd is the path systemd modifiable units, services, etc.. reside
	pathSystemd = "/etc/systemd/system"
	// wantsPathSystemd is the path where enabled units should be linked
	wantsPathSystemd = "/etc/systemd/system/multi-user.target.wants/"
	// pathDevNull is the systems path to and endless blackhole
	pathDevNull = "/dev/null"
	// currentConfigPath is where we store the current config on disk to validate
	// against annotations changes
	currentConfigPath = "/etc/machine-config-daemon/currentconfig"
	// pendingStateMessageID is the id we store the pending state in journal. We use it to
	// also retrieve the pending config after a reboot
	pendingStateMessageID = "machine-config-daemon-pending-state"

	kubeletHealthzEndpoint         = "http://localhost:10248/healthz"
	kubeletHealthzPollingInterval  = time.Duration(30 * time.Second)
	kubeletHealthzTimeout          = time.Duration(30 * time.Second)
	kubeletHealthzFailureThreshold = 3

	// updateDelay is the baseline speed at which we react to changes.  We don't
	// need to react in milliseconds as any change would involve rebooting the node.
	// Having this be relatively high limits the number of times we retry before
	// the MCC/MCO will time out.  We don't want to spam our logs with the same
	// error.
	updateDelay = 5 * time.Second
	// maxUpdateBackoff is the maximum time to react to a change as we back off
	// in the face of errors.
	maxUpdateBackoff = 60 * time.Second
)

type onceFromOrigin int

const (
	onceFromUnknownConfig onceFromOrigin = iota
	onceFromLocalConfig
	onceFromRemoteConfig
)

var (
	defaultRebootTimeout = 24 * time.Hour
)

func rebootCommand(rationale string) *exec.Cmd {
	return exec.Command("systemd-run", "--unit", "machine-config-daemon-reboot",
		"--description", fmt.Sprintf("machine-config-daemon: %s", rationale), "reboot")
}

// New sets up the systemd and kubernetes connections needed to update the
// machine.
func New(
	rootMount string,
	nodeName string,
	operatingSystem string,
	nodeUpdaterClient NodeUpdaterClient,
	bootID,
	onceFrom string,
	skipReboot bool,
	mcClient mcfgclientset.Interface,
	kubeClient kubernetes.Interface,
	kubeletHealthzEnabled bool,
	kubeletHealthzEndpoint string,
	nodeWriter NodeWriter,
	exitCh chan<- error,
	stopCh <-chan struct{},
) (*Daemon, error) {
	var (
		osImageURL string
		err        error
	)
	// Only pull the osImageURL from OSTree when we are on RHCOS
	if operatingSystem == machineConfigDaemonOSRHCOS {
		var osVersion string
		osImageURL, osVersion, err = nodeUpdaterClient.GetBootedOSImageURL(rootMount)
		if err != nil {
			return nil, fmt.Errorf("error reading osImageURL from rpm-ostree: %v", err)
		}
		glog.Infof("Booted osImageURL: %s (%s)", osImageURL, osVersion)
	}

	dn := &Daemon{
		name:                   nodeName,
		OperatingSystem:        operatingSystem,
		NodeUpdaterClient:      nodeUpdaterClient,
		rootMount:              rootMount,
		bootedOSImageURL:       osImageURL,
		bootID:                 bootID,
		onceFrom:               onceFrom,
		skipReboot:             skipReboot,
		kubeletHealthzEnabled:  kubeletHealthzEnabled,
		kubeletHealthzEndpoint: kubeletHealthzEndpoint,
		nodeWriter:             nodeWriter,
		exitCh:                 exitCh,
		stopCh:                 stopCh,
		kubeClient:             kubeClient,
		mcClient:               mcClient,
	}
	dn.currentConfigPath = currentConfigPath
	dn.atomicSSHKeysWriter = dn.atomicallyWriteSSHKey
	dn.loggerSupportsJournal = dn.isLoggingToJournalSupported()
	return dn, nil
}

// NewClusterDrivenDaemon sets up the systemd and kubernetes connections needed to update the
// machine.
func NewClusterDrivenDaemon(
	rootMount,
	nodeName,
	operatingSystem string,
	nodeUpdaterClient NodeUpdaterClient,
	mcInformer mcfginformersv1.MachineConfigInformer,
	kubeClient kubernetes.Interface,
	bootID,
	onceFrom string,
	skipReboot bool,
	nodeInformer coreinformersv1.NodeInformer,
	kubeletHealthzEnabled bool,
	kubeletHealthzEndpoint string,
	nodeWriter NodeWriter,
	exitCh chan<- error,
	stopCh <-chan struct{},
) (*Daemon, error) {
	dn, err := New(
		rootMount,
		nodeName,
		operatingSystem,
		nodeUpdaterClient,
		bootID,
		onceFrom,
		skipReboot,
		nil,
		kubeClient,
		kubeletHealthzEnabled,
		kubeletHealthzEndpoint,
		nodeWriter,
		exitCh,
		stopCh,
	)
	if err != nil {
		return nil, err
	}

	// Other controllers start out with the default controller limiter which retries
	// in milliseconds; since any change here will involve rebooting the node
	// we don't need to react in milliseconds.  See also updateDelay above.
	dn.queue = workqueue.NewNamedRateLimitingQueue(workqueue.NewMaxOfRateLimiter(
		&workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(updateDelay), 1)},
		workqueue.NewItemExponentialFailureRateLimiter(1*time.Second, maxUpdateBackoff)), "machineconfigdaemon")

	eventBroadcaster := record.NewBroadcaster()
	eventBroadcaster.StartLogging(glog.V(2).Infof)
	eventBroadcaster.StartRecordingToSink(&clientsetcorev1.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")})
	dn.recorder = eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: "machineconfigdaemon", Host: nodeName})

	go dn.runLoginMonitor(dn.stopCh, dn.exitCh)

	nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
		UpdateFunc: dn.handleNodeUpdate,
	})
	dn.nodeLister = nodeInformer.Lister()
	dn.nodeListerSynced = nodeInformer.Informer().HasSynced
	dn.mcLister = mcInformer.Lister()
	dn.mcListerSynced = mcInformer.Informer().HasSynced

	dn.enqueueNode = dn.enqueueDefault
	dn.syncHandler = dn.syncNode
	dn.booting = true

	return dn, nil
}

// worker runs a worker thread that just dequeues items, processes them, and marks them done.
// It enforces that the syncHandler is never invoked concurrently with the same key.
func (dn *Daemon) worker() {
	for dn.processNextWorkItem() {
	}
}

func (dn *Daemon) processNextWorkItem() bool {
	key, quit := dn.queue.Get()
	if quit {
		return false
	}
	defer dn.queue.Done(key)

	err := dn.syncHandler(key.(string))
	dn.handleErr(err, key)

	return true
}

// bootstrapNode takes care of the very first sync of the MCD on a node.
// It loads the node annotation from the bootstrap (if we're really bootstrapping)
// and then proceed to checking the state of the node, which includes
// finalizing an update and/or reconciling the current and desired machine configs.
func (dn *Daemon) bootstrapNode() error {
	node, err := dn.nodeLister.Get(dn.name)
	if err != nil {
		return err
	}
	node, err = dn.loadNodeAnnotations(node)
	if err != nil {
		return err
	}
	dn.node = node
	if err := dn.CheckStateOnBoot(); err != nil {
		return err
	}
	// finished syncing node for the first time
	dn.booting = false
	return nil
}

func (dn *Daemon) handleErr(err error, key interface{}) {
	if err == nil {
		dn.queue.Forget(key)
		return
	}

	dn.updateErrorState(err)
	// This is at V(2) since the updateErrorState() call above ends up logging too
	glog.V(2).Infof("Error syncing node %v (retries %d): %v", key, dn.queue.NumRequeues(key), err)
	dn.queue.AddRateLimited(key)
}

func (dn *Daemon) updateErrorState(err error) {
	switch errors.Cause(err) {
	case errUnreconcilable:
		dn.nodeWriter.SetUnreconcilable(err, dn.kubeClient.CoreV1().Nodes(), dn.nodeLister, dn.name)
	default:
		dn.nodeWriter.SetDegraded(err, dn.kubeClient.CoreV1().Nodes(), dn.nodeLister, dn.name)
	}
}

func (dn *Daemon) syncNode(key string) error {
	startTime := time.Now()
	glog.V(4).Infof("Started syncing node %q (%v)", key, startTime)
	defer func() {
		glog.V(4).Infof("Finished syncing node %q (%v)", key, time.Since(startTime))
	}()

	// Handle initial setup
	if dn.booting {
		if err := dn.bootstrapNode(); err != nil {
			return errors.Wrapf(err, "during bootstrap")
		}
		return nil
	}

	_, name, err := cache.SplitMetaNamespaceKey(key)
	if err != nil {
		return err
	}
	node, err := dn.nodeLister.Get(name)
	if apierrors.IsNotFound(err) {
		glog.V(2).Infof("node %v has been deleted", key)
		return nil
	}
	if err != nil {
		return err
	}

	// Deep-copy otherwise we are mutating our cache.
	node = node.DeepCopy()

	// Check for Deleted Node
	if node.DeletionTimestamp != nil {
		return nil
	}

	// First check if the node that was updated is this daemon's node
	if node.Name == dn.name {
		// stash the current node being processed
		dn.node = node
		// Pass to the shared update prep method
		current, desired, err := dn.prepUpdateFromCluster()
		if err != nil {
			glog.Infof("Unable to prep update: %s", err)
			return err
		}
		if current != nil || desired != nil {
			if err := dn.triggerUpdateWithMachineConfig(current, desired); err != nil {
				return err
			}
		}
		glog.V(2).Infof("Node %s is already synced", node.Name)
	}
	return nil
}

// enqueueDefault calls a default enqueue function
func (dn *Daemon) enqueueDefault(node *corev1.Node) {
	key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(node)
	if err != nil {
		utilruntime.HandleError(fmt.Errorf("couldn't get key for object %#v: %v", node, err))
		return
	}
	dn.queue.AddRateLimited(key)
}

const (
	// IDs are taken from https://cgit.freedesktop.org/systemd/systemd/plain/src/systemd/sd-messages.h
	sdMessageSessionStart = "8d45620c1a4348dbb17410da57c60c66"
)

// detectEarlySSHAccessesFromBoot taints the node if we find a login before the daemon started up.
func (dn *Daemon) detectEarlySSHAccessesFromBoot() error {
	journalOutput, err := exec.Command("journalctl", "-b", "-o", "cat", "MESSAGE_ID="+sdMessageSessionStart).CombinedOutput()
	if err != nil {
		return err
	}
	if len(journalOutput) > 0 {
		glog.Info("Detected a login session before the daemon took over on first boot")
		glog.Infof("Applying annotation: %v", machineConfigDaemonSSHAccessAnnotationKey)
		if err := dn.applySSHAccessedAnnotation(); err != nil {
			return err
		}
	}
	return nil
}

func (dn *Daemon) runOnceFrom() error {
	configi, contentFrom, err := dn.senseAndLoadOnceFrom()
	if err != nil {
		glog.Warningf("Unable to decipher onceFrom config type: %s", err)
		return err
	}
	switch configi.(type) {
	case ignv2_2types.Config:
		glog.V(2).Info("Daemon running directly from Ignition")
		return dn.runOnceFromIgnition(configi.(ignv2_2types.Config))
	case mcfgv1.MachineConfig:
		glog.V(2).Info("Daemon running directly from MachineConfig")
		return dn.runOnceFromMachineConfig(configi.(mcfgv1.MachineConfig), contentFrom)
	}
	return errors.New("unsupported onceFrom type provided")
}

// InstallSignalHandler installs the handler for the signals the daemon should act on
func (dn *Daemon) InstallSignalHandler(signaled chan struct{}) {
	termChan := make(chan os.Signal, 2048)
	signal.Notify(termChan, syscall.SIGTERM)

	// Catch SIGTERM - if we're actively updating, we should avoid
	// having the process be killed.
	// https://github.com/openshift/machine-config-operator/issues/407
	go func() {
		for sig := range termChan {
			switch sig {
			case syscall.SIGTERM:
				dn.updateActiveLock.Lock()
				updateActive := dn.updateActive
				dn.updateActiveLock.Unlock()
				if updateActive {
					glog.Info("Got SIGTERM, but actively updating")
				} else {
					close(signaled)
				}
			}
		}
	}()
}

// Run finishes informer setup and then blocks, and the informer will be
// responsible for triggering callbacks to handle updates. Successful
// updates shouldn't return, and should just reboot the node.
func (dn *Daemon) Run(stopCh, signaled <-chan struct{}, exitCh <-chan error) error {
	dn.logSystem("Starting to manage node: %s", dn.name)
	dn.LogSystemData()

	if dn.kubeletHealthzEnabled {
		glog.Info("Enabling Kubelet Healthz Monitor")
		go dn.runKubeletHealthzMonitor(stopCh, dn.exitCh)
	}

	// Catch quickly if we've been asked to run once.
	if dn.onceFrom != "" {
		return dn.runOnceFrom()
	}

	defer utilruntime.HandleCrash()
	defer dn.queue.ShutDown()

	if !cache.WaitForCacheSync(stopCh, dn.nodeListerSynced, dn.mcListerSynced) {
		return errors.New("failed to sync initial listers cache")
	}

	go wait.Until(dn.worker, time.Second, stopCh)

	for {
		select {
		case <-stopCh:
			return nil
		case <-signaled:
			return nil
		case err := <-exitCh:
			// This channel gets errors from auxiliary goroutines like loginmonitor and kubehealth
			glog.Warningf("Got an error from auxiliary tools: %v", err)
		}
	}
}

// BindPodMounts ensures that the daemon can still see e.g. /run/secrets/kubernetes.io
// service account tokens after chrooting.  This function must be called before chroot.
func (dn *Daemon) BindPodMounts() error {
	targetSecrets := filepath.Join(dn.rootMount, "/run/secrets")
	if err := os.MkdirAll(targetSecrets, 0755); err != nil {
		return err
	}
	// This will only affect our mount namespace, not the host
	mnt := exec.Command("mount", "--rbind", "/run/secrets", targetSecrets)
	return mnt.Run()
}

func (dn *Daemon) runLoginMonitor(stopCh <-chan struct{}, exitCh chan<- error) {
	cmd := exec.Command("journalctl", "-b", "-f", "-o", "cat", "MESSAGE_ID="+sdMessageSessionStart)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		exitCh <- err
		return
	}
	if err := cmd.Start(); err != nil {
		exitCh <- err
		return
	}
	worker := make(chan struct{})
	go func() {
		for {
			select {
			case <-worker:
				return
			default:
				buf := make([]byte, 1024)
				l, err := stdout.Read(buf)
				if err != nil {
					if err == io.EOF {
						return
					}
					exitCh <- err
					return
				}
				if l > 0 {
					line := strings.Split(string(buf), "\n")[0]
					glog.Infof("Detected a new login session: %s", line)
					glog.Infof("Login access is discouraged! Applying annotation: %v", machineConfigDaemonSSHAccessAnnotationKey)
					if err := dn.applySSHAccessedAnnotation(); err != nil {
						exitCh <- err
					}
				}
			}
		}
	}()
	select {
	case <-stopCh:
		close(worker)
		cmd.Process.Kill()
	}
}

func (dn *Daemon) applySSHAccessedAnnotation() error {
	if err := dn.nodeWriter.SetSSHAccessed(dn.kubeClient.CoreV1().Nodes(), dn.nodeLister, dn.name); err != nil {
		return fmt.Errorf("error: cannot apply annotation for SSH access due to: %v", err)
	}
	return nil
}

func (dn *Daemon) runKubeletHealthzMonitor(stopCh <-chan struct{}, exitCh chan<- error) {
	failureCount := 0
	for {
		select {
		case <-stopCh:
			return
		case <-time.After(kubeletHealthzPollingInterval):
			if err := dn.getHealth(); err != nil {
				glog.Warningf("Failed kubelet health check: %v", err)
				failureCount++
				if failureCount >= kubeletHealthzFailureThreshold {
					exitCh <- fmt.Errorf("kubelet health failure threshold reached")
				}
			} else {
				failureCount = 0 // reset failure count on success
			}
		}
	}
}

func (dn *Daemon) getHealth() error {
	glog.V(2).Info("Kubelet health running")
	ctx, cancel := context.WithTimeout(context.Background(), kubeletHealthzTimeout)
	defer cancel()

	req, err := http.NewRequest("GET", dn.kubeletHealthzEndpoint, nil)
	if err != nil {
		return err
	}
	req = req.WithContext(ctx)

	client := http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	respData, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	if string(respData) != "ok" {
		glog.Warningf("Kubelet Healthz Endpoint returned: %s", string(respData))
		return nil
	}

	glog.V(2).Info("Kubelet health ok")

	return nil
}

// stateAndConfigs is the "state" node annotation plus parsed machine configs
// referenced by the currentConfig and desiredConfig annotations.  If we have
// a "pending" config (we're coming up after a reboot attempting to apply a config),
// we'll load that as well - otherwise it will be nil.
//
// If any of the object names are the same, they will be pointer-equal.
type stateAndConfigs struct {
	bootstrapping bool
	state         string
	currentConfig *mcfgv1.MachineConfig
	pendingConfig *mcfgv1.MachineConfig
	desiredConfig *mcfgv1.MachineConfig
}

func (dn *Daemon) getStateAndConfigs(pendingConfigName string) (*stateAndConfigs, error) {
	_, err := os.Lstat(constants.InitialNodeAnnotationsFilePath)
	var bootstrapping bool
	if err != nil {
		if !os.IsNotExist(err) {
			return nil, err
		}
		// The node annotation file (laid down by the MCS)
		// doesn't exist, we must not be bootstrapping
	} else {
		bootstrapping = true
		glog.Info("In bootstrap mode")
	}

	currentConfigName, err := getNodeAnnotation(dn.node, constants.CurrentMachineConfigAnnotationKey)
	if err != nil {
		return nil, err
	}
	desiredConfigName, err := getNodeAnnotation(dn.node, constants.DesiredMachineConfigAnnotationKey)
	if err != nil {
		return nil, err
	}
	currentConfig, err := dn.mcLister.Get(currentConfigName)
	if err != nil {
		return nil, err
	}
	state, err := getNodeAnnotationExt(dn.node, constants.MachineConfigDaemonStateAnnotationKey, true)
	if err != nil {
		return nil, err
	}
	// Temporary hack: the MCS used to not write the state=done annotation
	// key.  If it's unset, let's write it now.
	if state == "" {
		state = constants.MachineConfigDaemonStateDone
	}

	var desiredConfig *mcfgv1.MachineConfig
	if currentConfigName == desiredConfigName {
		desiredConfig = currentConfig
		glog.Infof("Current+desired config: %s", currentConfigName)
	} else {
		desiredConfig, err = dn.mcLister.Get(desiredConfigName)
		if err != nil {
			return nil, err
		}

		glog.Infof("Current config: %s", currentConfigName)
		glog.Infof("Desired config: %s", desiredConfigName)
	}

	var pendingConfig *mcfgv1.MachineConfig
	// We usually expect that if current != desired, pending == desired; however,
	// it can happen that desiredConfig changed while we were rebooting.
	if pendingConfigName == desiredConfigName {
		dn.logSystem("using pending config same as desired")
		pendingConfig = desiredConfig
	} else if pendingConfigName != "" {
		pendingConfig, err = dn.mcLister.Get(pendingConfigName)
		if err != nil {
			return nil, err
		}

		glog.Infof("Pending config: %s", pendingConfigName)
	}

	return &stateAndConfigs{
		bootstrapping: bootstrapping,
		currentConfig: currentConfig,
		pendingConfig: pendingConfig,
		desiredConfig: desiredConfig,
		state:         state,
	}, nil
}

// LogSystemData gathers data from the OS and adds it to our stdout; should only
// be called once on MCD startup to log things which generally shouldn't change
// dynamically after a reboot.
func (dn *Daemon) LogSystemData() {
	// Print status if available
	if dn.OperatingSystem == machineConfigDaemonOSRHCOS {
		status, err := dn.NodeUpdaterClient.GetStatus()
		if err != nil {
			glog.Fatalf("unable to get rpm-ostree status: %s", err)
		}
		glog.Info(status)
	}

	boots, err := RunGetOut("journalctl", "--list-boots")
	if err != nil {
		glog.Errorf("Listing boots: %v", err)
	}
	glog.Infof("journalctl --list-boots:\n" + string(boots))
}

const (
	pendingConfigPath = "/etc/machine-config-daemon/state.json"
)

type pendingConfigState struct {
	PendingConfig string `json:"pendingConfig,omitempty"`
	BootID        string `json:"bootID,omitempty"`
}

// XXX: drop this
func (dn *Daemon) writePendingConfig(desiredConfig *mcfgv1.MachineConfig) error {
	t := &pendingConfigState{
		PendingConfig: desiredConfig.GetName(),
		BootID:        dn.bootID,
	}
	b, err := json.Marshal(t)
	if err != nil {
		return err
	}
	return writeFileAtomicallyWithDefaults(pendingConfigPath, b)
}

// XXX: drop this
// we need this compatibility layer for now
func (dn *Daemon) getPendingConfig() (string, error) {
	s, err := ioutil.ReadFile("/etc/machine-config-daemon/state.json")
	if err != nil {
		if !os.IsNotExist(err) {
			return "", errors.Wrapf(err, "loading transient state")
		}
		dn.logSystem("error loading pending config %v", err)
		return "", nil
	}
	var p pendingConfigState
	if err := json.Unmarshal([]byte(s), &p); err != nil {
		return "", errors.Wrapf(err, "parsing transient state")
	}

	if p.BootID == dn.bootID {
		return "", fmt.Errorf("pending config %s bootID %s matches current! Failed to reboot?", p.PendingConfig, dn.bootID)
	}
	return p.PendingConfig, nil
}

func (dn *Daemon) getCurrentConfigOnDisk() (*mcfgv1.MachineConfig, error) {
	mcJSON, err := os.Open(dn.currentConfigPath)
	if err != nil {
		return nil, err
	}
	defer mcJSON.Close()
	currentOnDisk := &mcfgv1.MachineConfig{}
	if err := json.NewDecoder(bufio.NewReader(mcJSON)).Decode(currentOnDisk); err != nil {
		return nil, err
	}
	return currentOnDisk, nil
}

func (dn *Daemon) storeCurrentConfigOnDisk(current *mcfgv1.MachineConfig) error {
	mcJSON, err := json.Marshal(current)
	if err != nil {
		return err
	}
	return writeFileAtomicallyWithDefaults(dn.currentConfigPath, mcJSON)
}

// CheckStateOnBoot is a core entrypoint for our state machine.
// It determines whether we're in our desired state, or if we're
// transitioning between states, and whether or not we need to update
// to a new state. It also checks if someone jumped on a node before
// the daemon took over.
//
// Some more background in this PR: https://github.com/openshift/machine-config-operator/pull/245
func (dn *Daemon) CheckStateOnBoot() error {
	pendingConfigName, err := dn.getPendingState()
	if err != nil {
		return err
	}
	// XXX: drop this
	// we need this compatibility layer for now
	if pendingConfigName == "" {
		pendingConfigName, err = dn.getPendingConfig()
		if err != nil {
			return err
		}
	}
	state, err := dn.getStateAndConfigs(pendingConfigName)
	if err != nil {
		return err
	}
	if err := dn.detectEarlySSHAccessesFromBoot(); err != nil {
		return fmt.Errorf("error detecting previous SSH accesses: %v", err)
	}

	if state.bootstrapping {
		targetOSImageURL := state.currentConfig.Spec.OSImageURL
		osMatch, err := dn.checkOS(targetOSImageURL)
		if err != nil {
			return err
		}
		if !osMatch {
			glog.Infof("Bootstrap pivot required to: %s", targetOSImageURL)
			// This only returns on error
			return dn.updateOSAndReboot(state.currentConfig)
		}
		glog.Info("No bootstrap pivot required; unlinking bootstrap node annotations")

		// Rename the bootstrap node annotations; the
		// currentConfig's osImageURL should now be *truth*.
		// In other words if it drifts somehow, we go degraded.
		if err := os.Rename(constants.InitialNodeAnnotationsFilePath, constants.InitialNodeAnnotationsBakPath); err != nil {
			return errors.Wrap(err, "renaming initial node annotation file")
		}
	}

	var currentOnDisk *mcfgv1.MachineConfig
	if !state.bootstrapping {
		var err error
		currentOnDisk, err = dn.getCurrentConfigOnDisk()
		// we allow ENOENT for previous MCO versions that don't write this...
		if err != nil && !os.IsNotExist(err) {
			return err
		}
	}

	if currentOnDisk != nil && state.currentConfig.GetName() != currentOnDisk.GetName() {
		// The on disk state (if available) is always considered truth.
		// We want to handle the case where etcd state was restored from a backup.
		glog.Infof("Using current config on disk %s", currentOnDisk.GetName())
		dn.logSystem("Using current config on disk %s", currentOnDisk.GetName())
		state.currentConfig = currentOnDisk
	}

	// Validate the on-disk state against what we *expect*.
	//
	// In the case where we're booting a node for the first time, or the MCD
	// is restarted, that will be the current config.
	//
	// In the case where we have
	// a pending config, this is where we validate that it actually applied.
	// We currently just do this on startup, but in the future it could e.g. be
	// a once-a-day or week cron job.
	var expectedConfig *mcfgv1.MachineConfig
	if state.pendingConfig != nil {
		dn.logSystem("Validating against pending config %s", state.pendingConfig.GetName())
		glog.Infof("Validating against pending config %s", state.pendingConfig.GetName())
		expectedConfig = state.pendingConfig
	} else {
		dn.logSystem("Validating against current config %s", state.currentConfig.GetName())
		glog.Infof("Validating against current config %s", state.currentConfig.GetName())
		expectedConfig = state.currentConfig
	}
	if !dn.validateOnDiskState(expectedConfig) {
		return fmt.Errorf("unexpected on-disk state validating against %s", expectedConfig.GetName())
	}
	glog.Info("Validated on-disk state")

	// We've validated our state.  In the case where we had a pendingConfig,
	// make that now currentConfig.  We update the node annotation, delete the
	// state file, etc.
	//
	// However, it may be the case that desiredConfig changed while we
	// were coming up, so we next look at that before uncordoning the node (so
	// we don't uncordon and then immediately re-cordon)
	if state.pendingConfig != nil {
		if dn.recorder != nil {
			dn.recorder.Eventf(getNodeRef(dn.node), corev1.EventTypeNormal, "NodeDone", fmt.Sprintf("Setting node %s, currentConfig %s to Done", dn.node.Name, state.pendingConfig.GetName()))
		}
		if err := dn.nodeWriter.SetDone(dn.kubeClient.CoreV1().Nodes(), dn.nodeLister, dn.name, state.pendingConfig.GetName()); err != nil {
			return errors.Wrap(err, "error setting node's state to Done")
		}
		if out, err := dn.storePendingState(state.pendingConfig, 0); err != nil {
			return errors.Wrapf(err, "failed to reset pending config: %s", string(out))
		}

		state.currentConfig = state.pendingConfig
	}

	if state.bootstrapping {
		if err := dn.storeCurrentConfigOnDisk(state.currentConfig); err != nil {
			return err
		}
	}

	inDesiredConfig := state.currentConfig.GetName() == state.desiredConfig.GetName()
	if inDesiredConfig {
		if state.pendingConfig != nil {
			// Great, we've successfully rebooted for the desired config,
			// let's mark it done!
			glog.Infof("Completing pending config %s", state.pendingConfig.GetName())
			if err := dn.completeUpdate(dn.node, state.pendingConfig.GetName()); err != nil {
				return err
			}
		}
		// if we're degraded here, it means we got an error likely on startup and we retried
		// if it's the case, clear it out
		if state.state == constants.MachineConfigDaemonStateDegraded {
			if err := dn.nodeWriter.SetDone(dn.kubeClient.CoreV1().Nodes(), dn.nodeLister, dn.name, state.currentConfig.GetName()); err != nil {
				return errors.Wrap(err, "error setting node's state to Done")
			}
		}

		glog.Infof("In desired config %s", state.currentConfig.GetName())

		// All good!
		return nil
	}
	if dn.recorder != nil {
		dn.recorder.Eventf(getNodeRef(dn.node), corev1.EventTypeNormal, "BootResync", fmt.Sprintf("Booting node %s, currentConfig %s, desiredConfig %s", dn.node.Name, state.currentConfig.GetName(), state.desiredConfig.GetName()))
	}
	// currentConfig != desiredConfig, and we're not booting up into the desiredConfig.
	// Kick off an update.
	return dn.triggerUpdateWithMachineConfig(state.currentConfig, state.desiredConfig)
}

// runOnceFromMachineConfig utilizes a parsed machineConfig and executes in onceFrom
// mode. If the content was remote, it executes cluster calls, otherwise it assumes
// no cluster is present yet.
func (dn *Daemon) runOnceFromMachineConfig(machineConfig mcfgv1.MachineConfig, contentFrom onceFromOrigin) error {
	if contentFrom == onceFromRemoteConfig {
		if dn.kubeClient == nil || dn.mcClient == nil {
			panic("running in onceFrom mode with a remote MachineConfig without a cluster")
		}
		// NOTE: This case expects a cluster to exists already.
		current, desired, err := dn.prepUpdateFromCluster()
		if err != nil {
			dn.nodeWriter.SetDegraded(err, dn.kubeClient.CoreV1().Nodes(), dn.nodeLister, dn.name)
			return err
		}
		if current == nil || desired == nil {
			return nil
		}
		// At this point we have verified we need to update
		if err := dn.triggerUpdateWithMachineConfig(current, &machineConfig); err != nil {
			dn.nodeWriter.SetDegraded(err, dn.kubeClient.CoreV1().Nodes(), dn.nodeLister, dn.name)
			return err
		}
		return nil
	}
	if contentFrom == onceFromLocalConfig {
		// NOTE: This case expects that the cluster is NOT CREATED YET.
		oldConfig := mcfgv1.MachineConfig{}
		// Execute update without hitting the cluster
		return dn.update(&oldConfig, &machineConfig)
	}
	// Otherwise return an error as the input format is unsupported
	return fmt.Errorf("%s is not a path nor url; can not run once", dn.onceFrom)
}

// runOnceFromIgnition executes MCD's subset of Ignition functionality in onceFrom mode
func (dn *Daemon) runOnceFromIgnition(ignConfig ignv2_2types.Config) error {
	// Execute update without hitting the cluster
	if err := dn.writeFiles(ignConfig.Storage.Files); err != nil {
		return err
	}
	if err := dn.writeUnits(ignConfig.Systemd.Units); err != nil {
		return err
	}
	return dn.reboot("runOnceFromIgnition complete", defaultRebootTimeout, rebootCommand("runOnceFromIgnition complete"))
}

func (dn *Daemon) handleNodeUpdate(old, cur interface{}) {
	oldNode := old.(*corev1.Node)
	curNode := cur.(*corev1.Node)

	glog.V(4).Infof("Updating Node %s", oldNode.Name)
	dn.enqueueNode(curNode)
}

// prepUpdateFromCluster handles the shared update prepping functionality for
// flows that expect the cluster to already be available. Returns true if an
// update is required, false otherwise.
func (dn *Daemon) prepUpdateFromCluster() (*mcfgv1.MachineConfig, *mcfgv1.MachineConfig, error) {
	desiredConfigName, err := getNodeAnnotationExt(dn.node, constants.DesiredMachineConfigAnnotationKey, true)
	if err != nil {
		return nil, nil, err
	}
	desiredConfig, err := dn.mcLister.Get(desiredConfigName)
	if err != nil {
		return nil, nil, err
	}
	// currentConfig is always expected to be there as loadNodeAnnotations
	// is one of the very first calls when the daemon starts.
	currentConfigName, err := getNodeAnnotation(dn.node, constants.CurrentMachineConfigAnnotationKey)
	if err != nil {
		return nil, nil, err
	}
	currentConfig, err := dn.mcLister.Get(currentConfigName)
	if err != nil {
		return nil, nil, err
	}
	state, err := getNodeAnnotation(dn.node, constants.MachineConfigDaemonStateAnnotationKey)
	if err != nil {
		return nil, nil, err
	}

	currentOnDisk, err := dn.getCurrentConfigOnDisk()
	if err != nil && !os.IsNotExist(err) {
		return nil, nil, err
	}

	if currentOnDisk != nil && currentOnDisk.GetName() != currentConfig.GetName() {
		return currentOnDisk, desiredConfig, nil
	}

	// Detect if there is an update
	if desiredConfigName == currentConfigName && state == constants.MachineConfigDaemonStateDone {
		// No actual update to the config
		glog.V(2).Info("No updating is required")
		return nil, nil, nil
	}
	return currentConfig, desiredConfig, nil
}

// completeUpdate marks the node as schedulable again, then deletes the
// "transient state" file, which signifies that all of those prior steps have
// been completed.
func (dn *Daemon) completeUpdate(node *corev1.Node, desiredConfigName string) error {
	if err := drain.Uncordon(dn.kubeClient.CoreV1().Nodes(), node, nil); err != nil {
		return err
	}

	dn.logSystem("completed update for config %s", desiredConfigName)

	return nil
}

// triggerUpdateWithMachineConfig starts the update. It queries the cluster for
// the current and desired config if they weren't passed.
func (dn *Daemon) triggerUpdateWithMachineConfig(currentConfig *mcfgv1.MachineConfig, desiredConfig *mcfgv1.MachineConfig) error {
	if currentConfig == nil {
		ccAnnotation, err := getNodeAnnotation(dn.node, constants.CurrentMachineConfigAnnotationKey)
		if err != nil {
			return err
		}
		currentConfig, err = dn.mcLister.Get(ccAnnotation)
		if err != nil {
			return err
		}
	}

	if desiredConfig == nil {
		dcAnnotation, err := getNodeAnnotation(dn.node, constants.DesiredMachineConfigAnnotationKey)
		if err != nil {
			return err
		}
		desiredConfig, err = dn.mcLister.Get(dcAnnotation)
		if err != nil {
			return err
		}
	}

	// run the update process. this function doesn't currently return.
	return dn.update(currentConfig, desiredConfig)
}

// validateOnDiskState compares the on-disk state against what a configuration
// specifies.  If for example an admin ssh'd into a node, or another operator
// is stomping on our files, we want to highlight that and mark the system
// degraded.
func (dn *Daemon) validateOnDiskState(currentConfig *mcfgv1.MachineConfig) bool {
	// Be sure we're booted into the OS we expect
	osMatch, err := dn.checkOS(currentConfig.Spec.OSImageURL)
	if err != nil {
		glog.Errorf("%s", err)
		return false
	}
	if !osMatch {
		glog.Errorf("expected target osImageURL %s", currentConfig.Spec.OSImageURL)
		return false
	}
	// And the rest of the disk state
	if !checkFiles(currentConfig.Spec.Config.Storage.Files) {
		return false
	}
	if !checkUnits(currentConfig.Spec.Config.Systemd.Units) {
		return false
	}
	return true
}

// getRefDigest parses a Docker/OCI image reference and returns
// its digest, or an error if the string fails to parse as
// a "canonical" image reference with a digest.
func getRefDigest(ref string) (string, error) {
	refParsed, err := imgref.ParseNamed(ref)
	if err != nil {
		return "", errors.Wrapf(err, "parsing reference: %q", ref)
	}
	canon, ok := refParsed.(imgref.Canonical)
	if !ok {
		return "", fmt.Errorf("not canonical form: %q", ref)
	}

	return canon.Digest().String(), nil
}

// compareOSImageURL is the backend for checkOS.
func compareOSImageURL(current, desired string) (bool, error) {
	// Since https://github.com/openshift/machine-config-operator/pull/426 landed
	// we don't use the "unspecified" osImageURL anymore, but let's keep supporting
	// it for now.
	// The ://dummy syntax is legacy
	if desired == "" || desired == "://dummy" {
		glog.Info(`No target osImageURL provided`)
		return true, nil
	}

	if current == desired {
		return true, nil
	}

	bootedDigest, err := getRefDigest(current)
	if err != nil {
		return false, errors.Wrap(err, "parsing booted osImageURL")
	}
	desiredDigest, err := getRefDigest(desired)
	if err != nil {
		return false, errors.Wrap(err, "parsing desired osImageURL")
	}

	if bootedDigest == desiredDigest {
		glog.Infof("Current and target osImageURL have matching digest %q", bootedDigest)
		return true, nil
	}

	return false, nil
}

// checkOS determines whether the booted system matches the target
// osImageURL and if not whether we need to take action.  This function
// returns `true` if no action is required, which is the case if we're
// not running RHCOS, or if the target osImageURL is "" (unspecified),
// or if the digests match.
// Otherwise if `false` is returned, then we need to perform an update.
func (dn *Daemon) checkOS(osImageURL string) (bool, error) {
	// Nothing to do if we're not on RHCOS
	if dn.OperatingSystem != machineConfigDaemonOSRHCOS {
		glog.Infof(`Not booted into Red Hat CoreOS, ignoring target OSImageURL %s`, osImageURL)
		return true, nil
	}

	return compareOSImageURL(dn.bootedOSImageURL, osImageURL)
}

// checkUnits validates the contents of all the units in the
// target config and retursn true if they match.
func checkUnits(units []ignv2_2types.Unit) bool {
	for _, u := range units {
		for j := range u.Dropins {
			path := filepath.Join(pathSystemd, u.Name+".d", u.Dropins[j].Name)
			if status := checkFileContentsAndMode(path, []byte(u.Dropins[j].Contents), defaultFilePermissions); !status {
				return false
			}
		}

		if u.Contents == "" {
			continue
		}

		path := filepath.Join(pathSystemd, u.Name)
		if u.Mask {
			link, err := filepath.EvalSymlinks(path)
			if err != nil {
				glog.Errorf("state validation: error while evaluation symlink for path: %q, err: %v", path, err)
				return false
			}
			if strings.Compare(pathDevNull, link) != 0 {
				glog.Errorf("state validation: invalid unit masked setting. path: %q; expected: %v; received: %v", path, pathDevNull, link)
				return false
			}
		}
		if status := checkFileContentsAndMode(path, []byte(u.Contents), defaultFilePermissions); !status {
			return false
		}

	}
	return true
}

// checkFiles validates the contents of  all the files in the
// target config.
func checkFiles(files []ignv2_2types.File) bool {
	checkedFiles := make(map[string]bool)
	for i := len(files) - 1; i >= 0; i-- {
		f := files[i]
		// skip over checked validated files
		if _, ok := checkedFiles[f.Path]; ok {
			continue
		}
		mode := defaultFilePermissions
		if f.Mode != nil {
			mode = os.FileMode(*f.Mode)
		}
		contents, err := dataurl.DecodeString(f.Contents.Source)
		if err != nil {
			glog.Errorf("couldn't parse file: %v", err)
			return false
		}
		if status := checkFileContentsAndMode(f.Path, contents.Data, mode); !status {
			return false
		}
		checkedFiles[f.Path] = true
	}
	return true
}

// checkFileContentsAndMode reads the file from the filepath and compares its
// contents and mode with the expectedContent and mode parameters. It logs an
// error in case of an error or mismatch and returns the status of the
// evaluation.
func checkFileContentsAndMode(filePath string, expectedContent []byte, mode os.FileMode) bool {
	fi, err := os.Lstat(filePath)
	if err != nil {
		glog.Errorf("could not stat file: %q, error: %v", filePath, err)
		return false
	}
	if fi.Mode() != mode {
		glog.Errorf("mode mismatch for file: %q; expected: %v; received: %v", filePath, mode, fi.Mode())
		return false
	}
	contents, err := ioutil.ReadFile(filePath)
	if err != nil {
		glog.Errorf("could not read file: %q, error: %v", filePath, err)
		return false
	}
	if !bytes.Equal(contents, expectedContent) {
		glog.Errorf("content mismatch for file %s: %s", filePath, diff.StringDiff(string(contents), string(expectedContent)))
		return false
	}
	return true
}

// Close closes all the connections the node agent has open for it's lifetime
func (dn *Daemon) Close() {
}

// ValidPath attempts to see if the path provided is indeed an acceptable
// filesystem path. This function does not check if the path exists.
func ValidPath(path string) bool {
	for _, validStart := range []string{".", "..", "/"} {
		if strings.HasPrefix(path, validStart) {
			return true
		}
	}
	return false
}

// senseAndLoadOnceFrom gets a hold of the content for supported onceFrom configurations,
// parses to verify the type, and returns back the genericInterface, the type description,
// if it was local or remote, and error.
func (dn *Daemon) senseAndLoadOnceFrom() (interface{}, onceFromOrigin, error) {
	var (
		content     []byte
		contentFrom onceFromOrigin
	)
	// Read the content from a remote endpoint if requested
	if strings.HasPrefix(dn.onceFrom, "http://") || strings.HasPrefix(dn.onceFrom, "https://") {
		contentFrom = onceFromRemoteConfig
		resp, err := http.Get(dn.onceFrom)
		if err != nil {
			return nil, contentFrom, err
		}
		defer resp.Body.Close()
		// Read the body content from the request
		content, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, contentFrom, err
		}

	} else {
		// Otherwise read it from a local file
		contentFrom = onceFromLocalConfig
		absoluteOnceFrom, err := filepath.Abs(filepath.Clean(dn.onceFrom))
		if err != nil {
			return nil, contentFrom, err
		}
		content, err = ioutil.ReadFile(absoluteOnceFrom)
		if err != nil {
			return nil, contentFrom, err
		}
	}

	// Try each supported parser
	ignConfig, _, err := ignv2.Parse(content)
	if err == nil && ignConfig.Ignition.Version != "" {
		glog.V(2).Info("onceFrom file is of type Ignition")
		return ignConfig, contentFrom, nil
	}

	glog.V(2).Infof("%s is not an Ignition config: %v. Trying MachineConfig.", dn.onceFrom, err)

	// Try to parse as a machine config
	mc, err := resourceread.ReadMachineConfigV1(content)
	if err == nil && mc != nil {
		glog.V(2).Info("onceFrom file is of type MachineConfig")
		return *mc, contentFrom, nil
	}

	return nil, onceFromUnknownConfig, fmt.Errorf("unable to decipher onceFrom config type: %v", err)
}
