<commit_before>package main

const lcStmtActiveUnblockCondition = `
SELECT 	scicd.blocked_instance_config_id,
		scicd.blocking_instance_config_id,
		scicd.unblocking_state,
		p.status,
		p.next_status,
		p.check_instance_id
FROM    soma.check_instance_configuration_dependencies scicd
JOIN    soma.check_instance_configurations scic
ON      scicd.blocking_instance_config_id = scic.check_instance_config_id
AND     scicd.unblocking_state = scic.status
JOIN    soma.check_instance_configurations p
ON      scicd.blocked_instance_config_id = p.check_instance_config_id
JOIN    soma.check_instances sci
ON      p.check_instance_id = sci.check_instance_id
AND     scicd.blocking_instance_config_id = sci.current_instance_config_id;`

const lcStmtUpdateInstance = `
UPDATE	soma.check_instances
SET     update_available = $1::boolean,
        current_instance_config_id = $2::uuid
WHERE   check_instance_id = $3::uuid;`

const lcStmtUpdateConfig = `
UPDATE  soma.check_instance_configurations
SET     status = $1::varchar,
        next_status = $2::varchar,
		awaiting_deletion = $3::boolean
WHERE   check_instance_config_id = $4::uuid;`

const lcStmtDeleteDependency = `
DELETE FROM soma.check_instance_configuration_dependencies
WHERE       blocked_instance_config_id = $1::uuid
AND         blocking_instance_config_id = $2::uuid
AND         unblocking_state = $3::varchar;`

const lcStmtReadyDeployments = `
SELECT scic.check_instance_id,
       scic.monitoring_id,
	   sms.monitoring_callback_uri
FROM   soma.check_instance_configurations scic
JOIN   soma.monitoring_systems sms
ON     scic.monitoring_id = sms.monitoring_id
JOIN   soma.check_instances sci
ON     scic.check_instance_id = sci.check_instance_id
AND    scic.check_instance_config_id = sci.current_instance_config_id
WHERE  (  scic.status = 'awaiting_rollout'
       OR scic.status = 'awaiting_deprovision' )
AND    sms.monitoring_callback_uri IS NOT NULL
AND    sci.update_available;`

const lcStmtClearUpdateFlag = `
UPDATE soma.check_instances
SET    update_available = 'false'::boolean
WHERE  check_instance_id = $1::uuid;`

const lcStmtBlockedConfigsForDeletedInstance = `
SELECT scicd.blocked_instance_config_id,
       scicd.blocking_instance_config_id,
       scicd.unblocking_state
FROM   soma.check_instances sci
JOIN   soma.check_instance_configurations scic
  ON   sci.check_instance_id = scic.check_instance_id
JOIN   soma.check_instance_configuration_dependencies scicd
  ON   scic.check_instance_config_id = scicd.blocked_instance_config_id
WHERE  sci.deleted
  AND  scic.status = 'blocked';`

const lcStmtConfigAwaitingDeletion = `
UPDATE soma.check_instance_configurations
SET    status = 'awaiting_deletion"::varchar,
       next_status = 'none'::varchar,
       awaiting_deletion = 'yes'::boolean
WHERE  check_instance_config_id = $1::uuid;`

const lcStmtDeleteGhosts = `
UPDATE soma.check_instance_configurations scic
SET    status = 'awaiting_deletion'::varchar,
       next_status = 'none'::varchar,
       awaiting_deletion = 'yes'::boolean
FROM   soma.check_instances sci
WHERE  scic.check_instance_id = sci.check_instance_id
AND    scic.status = 'awaiting_rollout'
AND    sci.deleted
AND    sci.update_available;`

const lcStmtDeleteFailedRollouts = `
UPDATE soma.check_instance_configurations scic
SET    status = 'awaiting_deletion'::varchar,
       next_status = 'none'::varchar,
       awaiting_deletion = 'yes'::boolean
FROM   soma.check_instances sci
WHERE  scic.check_instance_id = sci.check_instance_id
AND    sci.deleted
AND    scic.status = 'rollout_failed';`

const lcStmtDeleteDeprovisioned = `
UPDATE soma.check_instance_configurations scic
SET    status = 'awaiting_deletion'::varchar,
       next_status = 'none'::varchar,
       awaiting_deletion = 'yes'::boolean
FROM   soma.check_instances sci
WHERE  scic.check_instance_id = sci.check_instance_id
AND    sci.deleted
AND    scic.status = 'deprovisioned'
AND    scic.next_status = 'none';`

const lcStmtDeprovisionDeletedActive = `
SELECT scic.check_instance_config_id,
       sci.check_instance_id
FROM   soma.check_instance_configurations scic
JOIN   soma.check_instances sci
  ON   scic.check_instance_id = sci.check_instance_id
WHERE  sci.deleted
  AND  scic.status = 'active'
  AND  scic.next_status = 'none';
`

const lcStmtDeprovisionConfiguration = `
UPDATE soma.check_instance_configurations
SET    status = 'awaiting_deprovision'::varchar,
       next_status = 'deprovision_in_progress'::varchar
WHERE  check_instance_config_id = $1::uuid;
`

const lcStmtDeadLockResolver = `
SELECT ci.check_instance_id,
       ci.current_instance_config_id
FROM   check_instances ci
JOIN   check_instance_configurations cic
  ON   ci.check_instance_id = cic.check_instance_id
 AND   ci.current_instance_config_id = cic.check_instance_config_id
JOIN   check_instance_configuration_dependencies cicd
  ON   ci.current_instance_config_id = cicd.blocking_instance_config_id
WHERE  cic.status = 'active';`

// vim: ts=4 sw=4 sts=4 noet fenc=utf-8 ffs=unix
<commit_msg>FIX: typo related syntax error in lifecycle SQL<commit_after>package main

const lcStmtActiveUnblockCondition = `
SELECT 	scicd.blocked_instance_config_id,
		scicd.blocking_instance_config_id,
		scicd.unblocking_state,
		p.status,
		p.next_status,
		p.check_instance_id
FROM    soma.check_instance_configuration_dependencies scicd
JOIN    soma.check_instance_configurations scic
ON      scicd.blocking_instance_config_id = scic.check_instance_config_id
AND     scicd.unblocking_state = scic.status
JOIN    soma.check_instance_configurations p
ON      scicd.blocked_instance_config_id = p.check_instance_config_id
JOIN    soma.check_instances sci
ON      p.check_instance_id = sci.check_instance_id
AND     scicd.blocking_instance_config_id = sci.current_instance_config_id;`

const lcStmtUpdateInstance = `
UPDATE	soma.check_instances
SET     update_available = $1::boolean,
        current_instance_config_id = $2::uuid
WHERE   check_instance_id = $3::uuid;`

const lcStmtUpdateConfig = `
UPDATE  soma.check_instance_configurations
SET     status = $1::varchar,
        next_status = $2::varchar,
		awaiting_deletion = $3::boolean
WHERE   check_instance_config_id = $4::uuid;`

const lcStmtDeleteDependency = `
DELETE FROM soma.check_instance_configuration_dependencies
WHERE       blocked_instance_config_id = $1::uuid
AND         blocking_instance_config_id = $2::uuid
AND         unblocking_state = $3::varchar;`

const lcStmtReadyDeployments = `
SELECT scic.check_instance_id,
       scic.monitoring_id,
	   sms.monitoring_callback_uri
FROM   soma.check_instance_configurations scic
JOIN   soma.monitoring_systems sms
ON     scic.monitoring_id = sms.monitoring_id
JOIN   soma.check_instances sci
ON     scic.check_instance_id = sci.check_instance_id
AND    scic.check_instance_config_id = sci.current_instance_config_id
WHERE  (  scic.status = 'awaiting_rollout'
       OR scic.status = 'awaiting_deprovision' )
AND    sms.monitoring_callback_uri IS NOT NULL
AND    sci.update_available;`

const lcStmtClearUpdateFlag = `
UPDATE soma.check_instances
SET    update_available = 'false'::boolean
WHERE  check_instance_id = $1::uuid;`

const lcStmtBlockedConfigsForDeletedInstance = `
SELECT scicd.blocked_instance_config_id,
       scicd.blocking_instance_config_id,
       scicd.unblocking_state
FROM   soma.check_instances sci
JOIN   soma.check_instance_configurations scic
  ON   sci.check_instance_id = scic.check_instance_id
JOIN   soma.check_instance_configuration_dependencies scicd
  ON   scic.check_instance_config_id = scicd.blocked_instance_config_id
WHERE  sci.deleted
  AND  scic.status = 'blocked';`

const lcStmtConfigAwaitingDeletion = `
UPDATE soma.check_instance_configurations
SET    status = 'awaiting_deletion'::varchar,
       next_status = 'none'::varchar,
       awaiting_deletion = 'yes'::boolean
WHERE  check_instance_config_id = $1::uuid;`

const lcStmtDeleteGhosts = `
UPDATE soma.check_instance_configurations scic
SET    status = 'awaiting_deletion'::varchar,
       next_status = 'none'::varchar,
       awaiting_deletion = 'yes'::boolean
FROM   soma.check_instances sci
WHERE  scic.check_instance_id = sci.check_instance_id
AND    scic.status = 'awaiting_rollout'
AND    sci.deleted
AND    sci.update_available;`

const lcStmtDeleteFailedRollouts = `
UPDATE soma.check_instance_configurations scic
SET    status = 'awaiting_deletion'::varchar,
       next_status = 'none'::varchar,
       awaiting_deletion = 'yes'::boolean
FROM   soma.check_instances sci
WHERE  scic.check_instance_id = sci.check_instance_id
AND    sci.deleted
AND    scic.status = 'rollout_failed';`

const lcStmtDeleteDeprovisioned = `
UPDATE soma.check_instance_configurations scic
SET    status = 'awaiting_deletion'::varchar,
       next_status = 'none'::varchar,
       awaiting_deletion = 'yes'::boolean
FROM   soma.check_instances sci
WHERE  scic.check_instance_id = sci.check_instance_id
AND    sci.deleted
AND    scic.status = 'deprovisioned'
AND    scic.next_status = 'none';`

const lcStmtDeprovisionDeletedActive = `
SELECT scic.check_instance_config_id,
       sci.check_instance_id
FROM   soma.check_instance_configurations scic
JOIN   soma.check_instances sci
  ON   scic.check_instance_id = sci.check_instance_id
WHERE  sci.deleted
  AND  scic.status = 'active'
  AND  scic.next_status = 'none';
`

const lcStmtDeprovisionConfiguration = `
UPDATE soma.check_instance_configurations
SET    status = 'awaiting_deprovision'::varchar,
       next_status = 'deprovision_in_progress'::varchar
WHERE  check_instance_config_id = $1::uuid;
`

const lcStmtDeadLockResolver = `
SELECT ci.check_instance_id,
       ci.current_instance_config_id
FROM   check_instances ci
JOIN   check_instance_configurations cic
  ON   ci.check_instance_id = cic.check_instance_id
 AND   ci.current_instance_config_id = cic.check_instance_config_id
JOIN   check_instance_configuration_dependencies cicd
  ON   ci.current_instance_config_id = cicd.blocking_instance_config_id
WHERE  cic.status = 'active';`

// vim: ts=4 sw=4 sts=4 noet fenc=utf-8 ffs=unix
<|endoftext|>


<commit_before>// Copyright 2012-2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build linux

package main

import (
	"flag"
	"fmt"
	"golang.org/x/sys/unix"
	"io"
	"log"
	"os"
	"path/filepath"
	"syscall"
)

var (
	help    = flag.Bool("h", false, "Help")
	version = flag.Bool("V", false, "Version")
)

func usage() string {
	return "switch_root [-h] [-V]\nswitch_root newroot init"
}

// getDev returns the device (as returned by the FSTAT syscall) for the given file descriptor.
func getDev(fd int) (dev uint64, err error) {
	var stat syscall.Stat_t

	if err := syscall.Fstat(fd, &stat); err != nil {
		return 0, err
	}

	return stat.Dev, nil
}

// recursiveDelete deletes a directory identified by `fd` and everything in it.
//
// This function allows deleting directories no longer referenceable by
// any file name. This function does not descend into mounts.
func recursiveDelete(fd int) error {
	parentDev, err := getDev(fd)
	if err != nil {
		return err
	}

  // The file descriptor is already open, but allocating a os.File
	// here makes reading the files in the dir so much nicer.
	dir := os.NewFile(uintptr(fd), "__ignored__")
	defer dir.Close()
	names, err := dir.Readdirnames(-1)
	if err != nil {
		return err
	}

	for _, name := range names {
		// Loop here, but handle loop in separate function to make defer work as expected.
		if err := recusiveDeleteInner(fd, parentDev, name); err != nil {
			return err
		}
	}
	return nil
}

// recusiveDeleteInner is called from recursiveDelete and either deletes
// or recurses into the given file or directory
//
// There should be no need to call this function directly.
func recusiveDeleteInner(parentFd int, parentDev uint64, childName string) error {
	// O_DIRECTORY and O_NOFOLLOW make this open fail for all files and all symlinks (even when pointing to a dir).
	// We need to filter out symlinks because getDev later follows them.
	childFd, err := syscall.Openat(parentFd, childName, syscall.O_DIRECTORY | syscall.O_NOFOLLOW, syscall.O_RDWR)
	if err != nil {
		// childName points to either a file or a symlink, delete in any case.
		if err := unix.Unlinkat(parentFd, childName, 0); err != nil {
			return err
		}
	} else {
		// Open succeeded, which means childName points to a real directory.
		defer unix.Close(childFd)

		// Don't descent into other file systems.
		if childFdDev, err := getDev(childFd); err != nil {
			return err
		} else if childFdDev != parentDev {
			// This means continue in recursiveDelete.
			return nil
		}

		if err:= recursiveDelete(childFd); err != nil {
			return err
		}
		// Back from recursion, the directory is now empty, delete.
		if err := unix.Unlinkat(parentFd, childName, unix.AT_REMOVEDIR); err != nil {
			return err
		}
	}
	return nil
}

// execCommand execs into the given command.
//
// In order to preserve whatever PID this program is running with,
// the implementation does an actual EXEC syscall without forking.
func execCommand(path string) error {
	return syscall.Exec(path, []string{path}, []string{})
}

// isEmpty returns true if the directory with the given path is empty.
func isEmpty(name string) (bool, error) {
	f, err := os.Open(name)
	if err != nil {
		return false, err
	}
	defer f.Close()

	_, err = f.Readdirnames(1)
	if err == io.EOF {
		return true, nil
	}
	return false, err
}

// moveMount moves mount
//
// This function is just a wrapper around the MOUNT syscall with the
// MOVE flag supplied.
func moveMount(oldPath string, newPath string) error {
	return syscall.Mount(oldPath, newPath, "", syscall.MS_MOVE, "")
}

// specialFS moves the 'special' mounts to the given target path
//
// 'special' in this context refers to the following non-blockdevice backed
// mounts that are almost always used: /dev, /proc, /sys, and /run.
// This function will create the target directories, if necessary.
// If the target directories already exists, they must be empty.
// This function skips missing mounts.
func specialFS(newRoot string) error {
	var mounts = []string{"/dev", "/proc", "/sys", "/run"}

	for _, mount := range mounts {
		path := filepath.Join(newRoot, mount)
		// Skip all mounting if the directory does not exists.
		if _, err := os.Stat(mount); os.IsNotExist(err) {
			fmt.Println("switch_root: Skipping", mount)
			continue
		} else if err != nil {
			return err
		}
		// Make sure the target dir exists and is empty.
		if _, err := os.Stat(path); os.IsNotExist(err) {
			if err := syscall.Mkdir(path, 0); err != nil {
				return err
			}
		} else if err != nil {
			return err
		}
		if empty, err := isEmpty(path); err != nil {
			return err
		} else if !empty {
			return fmt.Errorf("%v must be empty", path)
		}
		if err := moveMount(mount, path); err != nil {
			return err
		}
	}
	return nil
}

// switchroot moves special mounts (dev, proc, sys, run) to the new directory,
// then does a chroot, moves the root mount to the new directory and finally
// DELETES EVERYTHING in the old root and execs the given init.
func switchRoot(newRoot string, init string) error {
	log.Printf("switch_root: moving mounts")
	if err := specialFS(newRoot); err != nil {
		return fmt.Errorf("switch_root: moving mounts failed %v", err)
	}

	log.Printf("switch_root: Changing directory")
	if err := syscall.Chdir(newRoot); err != nil {
		return fmt.Errorf("switch_root: failed change directory to new_root %v", err)
	}

	// Open "/" now, we need the file descriptor later.
	oldRoot, err := os.Open("/")
	if err != nil {
		return err
	}
	defer oldRoot.Close()

	log.Printf("switch_root: Moving /")
	if err := moveMount(newRoot, "/"); err != nil {
		return err
	}

	log.Printf("switch_root: Changing root!")
	if err := syscall.Chroot("."); err != nil {
		return fmt.Errorf("switch_root: fatal chroot error %v", err)
	}

	log.Printf("switch_root: Deleting old /")
	if err := recursiveDelete(int(oldRoot.Fd())); err != nil {
		panic(err)
	}

	log.Printf("switch_root: executing init")
	if err := execCommand(init); err != nil {
		return fmt.Errorf("switch_root: exec failed %v", err)
	}

	return nil
}

func main() {
	flag.Parse()

	if len(flag.Args()) == 0 {
		fmt.Println(usage())
		os.Exit(0)
	}

	if *help {
		fmt.Println(usage())
		os.Exit(0)
	}

	if *version {
		fmt.Println("Version XX")
		os.Exit(0)
	}

	newRoot := flag.Args()[0]
	init := flag.Args()[1]

	if err := switchRoot(newRoot, init); err != nil {
		log.Fatalf("switch_root failed %v\n", err)
	}
}
<commit_msg>cmds/switch_root: Use unix pkg for most syscalls<commit_after>// Copyright 2012-2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build linux

package main

import (
	"flag"
	"fmt"
	"golang.org/x/sys/unix"
	"io"
	"log"
	"os"
	"path/filepath"
	"syscall"
)

var (
	help    = flag.Bool("h", false, "Help")
	version = flag.Bool("V", false, "Version")
)

func usage() string {
	return "switch_root [-h] [-V]\nswitch_root newroot init"
}

// getDev returns the device (as returned by the FSTAT syscall) for the given file descriptor.
func getDev(fd int) (dev uint64, err error) {
	var stat unix.Stat_t

	if err := unix.Fstat(fd, &stat); err != nil {
		return 0, err
	}

	return stat.Dev, nil
}

// recursiveDelete deletes a directory identified by `fd` and everything in it.
//
// This function allows deleting directories no longer referenceable by
// any file name. This function does not descend into mounts.
func recursiveDelete(fd int) error {
	parentDev, err := getDev(fd)
	if err != nil {
		return err
	}

  // The file descriptor is already open, but allocating a os.File
	// here makes reading the files in the dir so much nicer.
	dir := os.NewFile(uintptr(fd), "__ignored__")
	defer dir.Close()
	names, err := dir.Readdirnames(-1)
	if err != nil {
		return err
	}

	for _, name := range names {
		// Loop here, but handle loop in separate function to make defer work as expected.
		if err := recusiveDeleteInner(fd, parentDev, name); err != nil {
			return err
		}
	}
	return nil
}

// recusiveDeleteInner is called from recursiveDelete and either deletes
// or recurses into the given file or directory
//
// There should be no need to call this function directly.
func recusiveDeleteInner(parentFd int, parentDev uint64, childName string) error {
	// O_DIRECTORY and O_NOFOLLOW make this open fail for all files and all symlinks (even when pointing to a dir).
	// We need to filter out symlinks because getDev later follows them.
	childFd, err := unix.Openat(parentFd, childName, unix.O_DIRECTORY | unix.O_NOFOLLOW, unix.O_RDWR)
	if err != nil {
		// childName points to either a file or a symlink, delete in any case.
		if err := unix.Unlinkat(parentFd, childName, 0); err != nil {
			return err
		}
	} else {
		// Open succeeded, which means childName points to a real directory.
		defer unix.Close(childFd)

		// Don't descent into other file systems.
		if childFdDev, err := getDev(childFd); err != nil {
			return err
		} else if childFdDev != parentDev {
			// This means continue in recursiveDelete.
			return nil
		}

		if err:= recursiveDelete(childFd); err != nil {
			return err
		}
		// Back from recursion, the directory is now empty, delete.
		if err := unix.Unlinkat(parentFd, childName, unix.AT_REMOVEDIR); err != nil {
			return err
		}
	}
	return nil
}

// execCommand execs into the given command.
//
// In order to preserve whatever PID this program is running with,
// the implementation does an actual EXEC syscall without forking.
func execCommand(path string) error {
	return syscall.Exec(path, []string{path}, []string{})
}

// isEmpty returns true if the directory with the given path is empty.
func isEmpty(name string) (bool, error) {
	f, err := os.Open(name)
	if err != nil {
		return false, err
	}
	defer f.Close()

	_, err = f.Readdirnames(1)
	if err == io.EOF {
		return true, nil
	}
	return false, err
}

// moveMount moves mount
//
// This function is just a wrapper around the MOUNT syscall with the
// MOVE flag supplied.
func moveMount(oldPath string, newPath string) error {
	return unix.Mount(oldPath, newPath, "", unix.MS_MOVE, "")
}

// specialFS moves the 'special' mounts to the given target path
//
// 'special' in this context refers to the following non-blockdevice backed
// mounts that are almost always used: /dev, /proc, /sys, and /run.
// This function will create the target directories, if necessary.
// If the target directories already exists, they must be empty.
// This function skips missing mounts.
func specialFS(newRoot string) error {
	var mounts = []string{"/dev", "/proc", "/sys", "/run"}

	for _, mount := range mounts {
		path := filepath.Join(newRoot, mount)
		// Skip all mounting if the directory does not exists.
		if _, err := os.Stat(mount); os.IsNotExist(err) {
			fmt.Println("switch_root: Skipping", mount)
			continue
		} else if err != nil {
			return err
		}
		// Make sure the target dir exists and is empty.
		if _, err := os.Stat(path); os.IsNotExist(err) {
			if err := unix.Mkdir(path, 0); err != nil {
				return err
			}
		} else if err != nil {
			return err
		}
		if empty, err := isEmpty(path); err != nil {
			return err
		} else if !empty {
			return fmt.Errorf("%v must be empty", path)
		}
		if err := moveMount(mount, path); err != nil {
			return err
		}
	}
	return nil
}

// switchroot moves special mounts (dev, proc, sys, run) to the new directory,
// then does a chroot, moves the root mount to the new directory and finally
// DELETES EVERYTHING in the old root and execs the given init.
func switchRoot(newRoot string, init string) error {
	log.Printf("switch_root: moving mounts")
	if err := specialFS(newRoot); err != nil {
		return fmt.Errorf("switch_root: moving mounts failed %v", err)
	}

	log.Printf("switch_root: Changing directory")
	if err := unix.Chdir(newRoot); err != nil {
		return fmt.Errorf("switch_root: failed change directory to new_root %v", err)
	}

	// Open "/" now, we need the file descriptor later.
	oldRoot, err := os.Open("/")
	if err != nil {
		return err
	}
	defer oldRoot.Close()

	log.Printf("switch_root: Moving /")
	if err := moveMount(newRoot, "/"); err != nil {
		return err
	}

	log.Printf("switch_root: Changing root!")
	if err := unix.Chroot("."); err != nil {
		return fmt.Errorf("switch_root: fatal chroot error %v", err)
	}

	log.Printf("switch_root: Deleting old /")
	if err := recursiveDelete(int(oldRoot.Fd())); err != nil {
		panic(err)
	}

	log.Printf("switch_root: executing init")
	if err := execCommand(init); err != nil {
		return fmt.Errorf("switch_root: exec failed %v", err)
	}

	return nil
}

func main() {
	flag.Parse()

	if len(flag.Args()) == 0 {
		fmt.Println(usage())
		os.Exit(0)
	}

	if *help {
		fmt.Println(usage())
		os.Exit(0)
	}

	if *version {
		fmt.Println("Version XX")
		os.Exit(0)
	}

	newRoot := flag.Args()[0]
	init := flag.Args()[1]

	if err := switchRoot(newRoot, init); err != nil {
		log.Fatalf("switch_root failed %v\n", err)
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2018 Ashley Jeffs
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package processor

import (
	"bytes"
	"fmt"
	"net/url"
	"regexp"
	"strconv"
	"time"

	"github.com/Jeffail/benthos/lib/log"
	"github.com/Jeffail/benthos/lib/metrics"
	"github.com/Jeffail/benthos/lib/types"
	"github.com/Jeffail/benthos/lib/util/text"
	"github.com/microcosm-cc/bluemonday"
	"github.com/opentracing/opentracing-go"
)

//------------------------------------------------------------------------------

func init() {
	Constructors[TypeText] = TypeSpec{
		constructor: NewText,
		description: `
Performs text based mutations on payloads.

This processor will interpolate functions within the ` + "`value`" + ` field,
you can find a list of functions [here](../config_interpolation.md#functions).

Value interpolations are resolved once per message batch, in order to resolve it
for each message of the batch place it within a
` + "[`for_each`](#for_each)" + ` processor:

` + "``` yaml" + `
for_each:
- text:
    operator: set
    value: ${!json_field:document.content}
` + "```" + `

### Operators

#### ` + "`append`" + `

Appends text to the end of the payload.

#### ` + "`escape_url_query`" + `

Escapes text so that it is safe to place within the query section of a URL.

#### ` + "`unescape_url_query`" + `

Unescapes text that has been url escaped.

#### ` + "`find_regexp`" + `

Extract the matching section of the argument regular expression in a message.

#### ` + "`prepend`" + `

Prepends text to the beginning of the payload.

#### ` + "`quote`" + `

Returns a doubled-quoted string, using escape sequences (\t, \n, \xFF, \u0100)
for control characters and other non-printable characters.

#### ` + "`replace`" + `

Replaces all occurrences of the argument in a message with a value.

#### ` + "`replace_regexp`" + `

Replaces all occurrences of the argument regular expression in a message with a
value. Inside the value $ signs are interpreted as submatch expansions, e.g. $1
represents the text of the first submatch.

#### ` + "`set`" + `

Replace the contents of a message entirely with a value.

#### ` + "`strip_html`" + `

Removes all HTML tags from a message.

#### ` + "`to_lower`" + `

Converts all text into lower case.

#### ` + "`to_upper`" + `

Converts all text into upper case.

#### ` + "`trim`" + `

Removes all leading and trailing occurrences of characters within the arg field.

#### ` + "`trim_space`" + `

Removes all leading and trailing whitespace from the payload.

#### ` + "`unquote`" + `

Unquotes a single, double, or back-quoted string literal`,
	}
}

//------------------------------------------------------------------------------

// TextConfig contains configuration fields for the Text processor.
type TextConfig struct {
	Parts    []int  `json:"parts" yaml:"parts"`
	Operator string `json:"operator" yaml:"operator"`
	Arg      string `json:"arg" yaml:"arg"`
	Value    string `json:"value" yaml:"value"`
}

// NewTextConfig returns a TextConfig with default values.
func NewTextConfig() TextConfig {
	return TextConfig{
		Parts:    []int{},
		Operator: "trim_space",
		Arg:      "",
		Value:    "",
	}
}

//------------------------------------------------------------------------------

type textOperator func(body []byte, value []byte) ([]byte, error)

func newTextAppendOperator() textOperator {
	return func(body []byte, value []byte) ([]byte, error) {
		if len(value) == 0 {
			return body, nil
		}
		return append(body[:], value...), nil
	}
}

func newTextEscapeURLQueryOperator() textOperator {
	return func(body []byte, value []byte) ([]byte, error) {
		return []byte(url.QueryEscape(string(body))), nil
	}
}

func newTextUnescapeURLQueryOperator() textOperator {
	return func(body []byte, value []byte) ([]byte, error) {
		s, err := url.QueryUnescape(string(body))
		if err != nil {
			return nil, err
		}
		return []byte(s), nil
	}
}

func newTextPrependOperator() textOperator {
	return func(body []byte, value []byte) ([]byte, error) {
		if len(value) == 0 {
			return body, nil
		}
		return append(value[:], body...), nil
	}
}

func newTextQuoteOperator() textOperator {
	return func(body []byte, value []byte) ([]byte, error) {
		return []byte(strconv.Quote(string(body))), nil
	}
}

func newTextTrimSpaceOperator() textOperator {
	return func(body []byte, value []byte) ([]byte, error) {
		return bytes.TrimSpace(body), nil
	}
}

func newTextToUpperOperator() textOperator {
	return func(body []byte, value []byte) ([]byte, error) {
		return bytes.ToUpper(body), nil
	}
}

func newTextToLowerOperator() textOperator {
	return func(body []byte, value []byte) ([]byte, error) {
		return bytes.ToLower(body), nil
	}
}

func newTextTrimOperator(arg string) textOperator {
	return func(body []byte, value []byte) ([]byte, error) {
		return bytes.Trim(body, arg), nil
	}
}

func newTextSetOperator() textOperator {
	return func(body []byte, value []byte) ([]byte, error) {
		return value, nil
	}
}

func newTextReplaceOperator(arg string) textOperator {
	replaceArg := []byte(arg)
	return func(body []byte, value []byte) ([]byte, error) {
		return bytes.Replace(body, replaceArg, value, -1), nil
	}
}

func newTextReplaceRegexpOperator(arg string) (textOperator, error) {
	rp, err := regexp.Compile(arg)
	if err != nil {
		return nil, err
	}
	return func(body []byte, value []byte) ([]byte, error) {
		return rp.ReplaceAll(body, value), nil
	}, nil
}

func newTextFindRegexpOperator(arg string) (textOperator, error) {
	rp, err := regexp.Compile(arg)
	if err != nil {
		return nil, err
	}
	return func(body []byte, value []byte) ([]byte, error) {
		return rp.Find(body), nil
	}, nil
}

func newTextStripHTMLOperator(arg string) textOperator {
	p := bluemonday.NewPolicy()
	return func(body []byte, value []byte) ([]byte, error) {
		return p.SanitizeBytes(body), nil
	}
}

func newTextUnquoteOperator() textOperator {
	return func(body []byte, value []byte) ([]byte, error) {
		res, err := strconv.Unquote(string(body))
		if err != nil {
			return nil, err
		}
		return []byte(res), err
	}
}

func getTextOperator(opStr string, arg string) (textOperator, error) {
	switch opStr {
	case "append":
		return newTextAppendOperator(), nil
	case "escape_url_query":
		return newTextEscapeURLQueryOperator(), nil
	case "unescape_url_query":
		return newTextUnescapeURLQueryOperator(), nil
	case "find_regexp":
		return newTextFindRegexpOperator(arg)
	case "prepend":
		return newTextPrependOperator(), nil
	case "quote":
		return newTextQuoteOperator(), nil
	case "replace":
		return newTextReplaceOperator(arg), nil
	case "replace_regexp":
		return newTextReplaceRegexpOperator(arg)
	case "set":
		return newTextSetOperator(), nil
	case "strip_html":
		return newTextStripHTMLOperator(arg), nil
	case "to_lower":
		return newTextToLowerOperator(), nil
	case "to_upper":
		return newTextToUpperOperator(), nil
	case "trim":
		return newTextTrimOperator(arg), nil
	case "trim_space":
		return newTextTrimSpaceOperator(), nil
	case "unquote":
		return newTextUnquoteOperator(), nil
	}
	return nil, fmt.Errorf("operator not recognised: %v", opStr)
}

//------------------------------------------------------------------------------

// Text is a processor that performs a text based operation on a payload.
type Text struct {
	parts       []int
	interpolate bool
	valueBytes  []byte
	operator    textOperator

	conf  Config
	log   log.Modular
	stats metrics.Type

	mCount     metrics.StatCounter
	mErr       metrics.StatCounter
	mSent      metrics.StatCounter
	mBatchSent metrics.StatCounter
}

// NewText returns a Text processor.
func NewText(
	conf Config, mgr types.Manager, log log.Modular, stats metrics.Type,
) (Type, error) {
	t := &Text{
		parts: conf.Text.Parts,
		conf:  conf,
		log:   log,
		stats: stats,

		valueBytes: []byte(conf.Text.Value),

		mCount:     stats.GetCounter("count"),
		mErr:       stats.GetCounter("error"),
		mSent:      stats.GetCounter("sent"),
		mBatchSent: stats.GetCounter("batch.sent"),
	}

	t.interpolate = text.ContainsFunctionVariables(t.valueBytes)

	var err error
	if t.operator, err = getTextOperator(conf.Text.Operator, conf.Text.Arg); err != nil {
		return nil, err
	}
	return t, nil
}

//------------------------------------------------------------------------------

// ProcessMessage applies the processor to a message, either creating >0
// resulting messages or a response to be sent back to the message source.
func (t *Text) ProcessMessage(msg types.Message) ([]types.Message, types.Response) {
	t.mCount.Incr(1)
	newMsg := msg.Copy()

	valueBytes := t.valueBytes
	if t.interpolate {
		valueBytes = text.ReplaceFunctionVariables(msg, valueBytes)
	}

	proc := func(index int, span opentracing.Span, part types.Part) error {
		data := part.Get()
		var err error
		if data, err = t.operator(data, valueBytes); err != nil {
			t.mErr.Incr(1)
			t.log.Debugf("Failed to apply operator: %v\n", err)
			return err
		}
		part.Set(data)
		return nil
	}

	IteratePartsWithSpan(TypeText, t.parts, newMsg, proc)

	msgs := [1]types.Message{newMsg}

	t.mBatchSent.Incr(1)
	t.mSent.Incr(int64(newMsg.Len()))
	return msgs[:], nil
}

// CloseAsync shuts down the processor and stops processing requests.
func (t *Text) CloseAsync() {
}

// WaitForClose blocks until the processor has closed down.
func (t *Text) WaitForClose(timeout time.Duration) error {
	return nil
}

//------------------------------------------------------------------------------
<commit_msg>Prevent prepend to share same array between parts<commit_after>// Copyright (c) 2018 Ashley Jeffs
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package processor

import (
	"bytes"
	"fmt"
	"net/url"
	"regexp"
	"strconv"
	"time"

	"github.com/Jeffail/benthos/lib/log"
	"github.com/Jeffail/benthos/lib/metrics"
	"github.com/Jeffail/benthos/lib/types"
	"github.com/Jeffail/benthos/lib/util/text"
	"github.com/microcosm-cc/bluemonday"
	"github.com/opentracing/opentracing-go"
)

//------------------------------------------------------------------------------

func init() {
	Constructors[TypeText] = TypeSpec{
		constructor: NewText,
		description: `
Performs text based mutations on payloads.

This processor will interpolate functions within the ` + "`value`" + ` field,
you can find a list of functions [here](../config_interpolation.md#functions).

Value interpolations are resolved once per message batch, in order to resolve it
for each message of the batch place it within a
` + "[`for_each`](#for_each)" + ` processor:

` + "``` yaml" + `
for_each:
- text:
    operator: set
    value: ${!json_field:document.content}
` + "```" + `

### Operators

#### ` + "`append`" + `

Appends text to the end of the payload.

#### ` + "`escape_url_query`" + `

Escapes text so that it is safe to place within the query section of a URL.

#### ` + "`unescape_url_query`" + `

Unescapes text that has been url escaped.

#### ` + "`find_regexp`" + `

Extract the matching section of the argument regular expression in a message.

#### ` + "`prepend`" + `

Prepends text to the beginning of the payload.

#### ` + "`quote`" + `

Returns a doubled-quoted string, using escape sequences (\t, \n, \xFF, \u0100)
for control characters and other non-printable characters.

#### ` + "`replace`" + `

Replaces all occurrences of the argument in a message with a value.

#### ` + "`replace_regexp`" + `

Replaces all occurrences of the argument regular expression in a message with a
value. Inside the value $ signs are interpreted as submatch expansions, e.g. $1
represents the text of the first submatch.

#### ` + "`set`" + `

Replace the contents of a message entirely with a value.

#### ` + "`strip_html`" + `

Removes all HTML tags from a message.

#### ` + "`to_lower`" + `

Converts all text into lower case.

#### ` + "`to_upper`" + `

Converts all text into upper case.

#### ` + "`trim`" + `

Removes all leading and trailing occurrences of characters within the arg field.

#### ` + "`trim_space`" + `

Removes all leading and trailing whitespace from the payload.

#### ` + "`unquote`" + `

Unquotes a single, double, or back-quoted string literal`,
	}
}

//------------------------------------------------------------------------------

// TextConfig contains configuration fields for the Text processor.
type TextConfig struct {
	Parts    []int  `json:"parts" yaml:"parts"`
	Operator string `json:"operator" yaml:"operator"`
	Arg      string `json:"arg" yaml:"arg"`
	Value    string `json:"value" yaml:"value"`
}

// NewTextConfig returns a TextConfig with default values.
func NewTextConfig() TextConfig {
	return TextConfig{
		Parts:    []int{},
		Operator: "trim_space",
		Arg:      "",
		Value:    "",
	}
}

//------------------------------------------------------------------------------

type textOperator func(body []byte, value []byte) ([]byte, error)

func newTextAppendOperator() textOperator {
	return func(body []byte, value []byte) ([]byte, error) {
		if len(value) == 0 {
			return body, nil
		}
		return append(body[:], value...), nil
	}
}

func newTextEscapeURLQueryOperator() textOperator {
	return func(body []byte, value []byte) ([]byte, error) {
		return []byte(url.QueryEscape(string(body))), nil
	}
}

func newTextUnescapeURLQueryOperator() textOperator {
	return func(body []byte, value []byte) ([]byte, error) {
		s, err := url.QueryUnescape(string(body))
		if err != nil {
			return nil, err
		}
		return []byte(s), nil
	}
}

func newTextPrependOperator() textOperator {
	return func(body []byte, value []byte) ([]byte, error) {
		if len(value) == 0 {
			return body, nil
		}
		return append(value[:len(value):len(value)], body...), nil
	}
}

func newTextQuoteOperator() textOperator {
	return func(body []byte, value []byte) ([]byte, error) {
		return []byte(strconv.Quote(string(body))), nil
	}
}

func newTextTrimSpaceOperator() textOperator {
	return func(body []byte, value []byte) ([]byte, error) {
		return bytes.TrimSpace(body), nil
	}
}

func newTextToUpperOperator() textOperator {
	return func(body []byte, value []byte) ([]byte, error) {
		return bytes.ToUpper(body), nil
	}
}

func newTextToLowerOperator() textOperator {
	return func(body []byte, value []byte) ([]byte, error) {
		return bytes.ToLower(body), nil
	}
}

func newTextTrimOperator(arg string) textOperator {
	return func(body []byte, value []byte) ([]byte, error) {
		return bytes.Trim(body, arg), nil
	}
}

func newTextSetOperator() textOperator {
	return func(body []byte, value []byte) ([]byte, error) {
		return value, nil
	}
}

func newTextReplaceOperator(arg string) textOperator {
	replaceArg := []byte(arg)
	return func(body []byte, value []byte) ([]byte, error) {
		return bytes.Replace(body, replaceArg, value, -1), nil
	}
}

func newTextReplaceRegexpOperator(arg string) (textOperator, error) {
	rp, err := regexp.Compile(arg)
	if err != nil {
		return nil, err
	}
	return func(body []byte, value []byte) ([]byte, error) {
		return rp.ReplaceAll(body, value), nil
	}, nil
}

func newTextFindRegexpOperator(arg string) (textOperator, error) {
	rp, err := regexp.Compile(arg)
	if err != nil {
		return nil, err
	}
	return func(body []byte, value []byte) ([]byte, error) {
		return rp.Find(body), nil
	}, nil
}

func newTextStripHTMLOperator(arg string) textOperator {
	p := bluemonday.NewPolicy()
	return func(body []byte, value []byte) ([]byte, error) {
		return p.SanitizeBytes(body), nil
	}
}

func newTextUnquoteOperator() textOperator {
	return func(body []byte, value []byte) ([]byte, error) {
		res, err := strconv.Unquote(string(body))
		if err != nil {
			return nil, err
		}
		return []byte(res), err
	}
}

func getTextOperator(opStr string, arg string) (textOperator, error) {
	switch opStr {
	case "append":
		return newTextAppendOperator(), nil
	case "escape_url_query":
		return newTextEscapeURLQueryOperator(), nil
	case "unescape_url_query":
		return newTextUnescapeURLQueryOperator(), nil
	case "find_regexp":
		return newTextFindRegexpOperator(arg)
	case "prepend":
		return newTextPrependOperator(), nil
	case "quote":
		return newTextQuoteOperator(), nil
	case "replace":
		return newTextReplaceOperator(arg), nil
	case "replace_regexp":
		return newTextReplaceRegexpOperator(arg)
	case "set":
		return newTextSetOperator(), nil
	case "strip_html":
		return newTextStripHTMLOperator(arg), nil
	case "to_lower":
		return newTextToLowerOperator(), nil
	case "to_upper":
		return newTextToUpperOperator(), nil
	case "trim":
		return newTextTrimOperator(arg), nil
	case "trim_space":
		return newTextTrimSpaceOperator(), nil
	case "unquote":
		return newTextUnquoteOperator(), nil
	}
	return nil, fmt.Errorf("operator not recognised: %v", opStr)
}

//------------------------------------------------------------------------------

// Text is a processor that performs a text based operation on a payload.
type Text struct {
	parts       []int
	interpolate bool
	valueBytes  []byte
	operator    textOperator

	conf  Config
	log   log.Modular
	stats metrics.Type

	mCount     metrics.StatCounter
	mErr       metrics.StatCounter
	mSent      metrics.StatCounter
	mBatchSent metrics.StatCounter
}

// NewText returns a Text processor.
func NewText(
	conf Config, mgr types.Manager, log log.Modular, stats metrics.Type,
) (Type, error) {
	t := &Text{
		parts: conf.Text.Parts,
		conf:  conf,
		log:   log,
		stats: stats,

		valueBytes: []byte(conf.Text.Value),

		mCount:     stats.GetCounter("count"),
		mErr:       stats.GetCounter("error"),
		mSent:      stats.GetCounter("sent"),
		mBatchSent: stats.GetCounter("batch.sent"),
	}

	t.interpolate = text.ContainsFunctionVariables(t.valueBytes)

	var err error
	if t.operator, err = getTextOperator(conf.Text.Operator, conf.Text.Arg); err != nil {
		return nil, err
	}
	return t, nil
}

//------------------------------------------------------------------------------

// ProcessMessage applies the processor to a message, either creating >0
// resulting messages or a response to be sent back to the message source.
func (t *Text) ProcessMessage(msg types.Message) ([]types.Message, types.Response) {
	t.mCount.Incr(1)
	newMsg := msg.Copy()

	valueBytes := t.valueBytes
	if t.interpolate {
		valueBytes = text.ReplaceFunctionVariables(msg, valueBytes)
	}

	proc := func(index int, span opentracing.Span, part types.Part) error {
		data := part.Get()
		var err error
		if data, err = t.operator(data, valueBytes); err != nil {
			t.mErr.Incr(1)
			t.log.Debugf("Failed to apply operator: %v\n", err)
			return err
		}
		part.Set(data)
		return nil
	}

	IteratePartsWithSpan(TypeText, t.parts, newMsg, proc)

	msgs := [1]types.Message{newMsg}

	t.mBatchSent.Incr(1)
	t.mSent.Incr(int64(newMsg.Len()))
	return msgs[:], nil
}

// CloseAsync shuts down the processor and stops processing requests.
func (t *Text) CloseAsync() {
}

// WaitForClose blocks until the processor has closed down.
func (t *Text) WaitForClose(timeout time.Duration) error {
	return nil
}

//------------------------------------------------------------------------------
<|endoftext|>


<commit_before><commit_msg>Fixed mem average<commit_after><|endoftext|>


<commit_before>package main

import (
	"log"
	"path"

	ostree "github.com/sjoerdsimons/ostree-go/pkg/otbuiltin"
)

type OstreeDeployAction struct {
	*BaseAction
	Repository       string
	RemoteRepository string
	Branch           string
	Os               string
}

func (ot *OstreeDeployAction) Run(context *YaibContext) {
	repoPath := "file://" + path.Join(context.artifactdir, ot.Repository)

	sysroot := ostree.NewSysroot(context.imageMntDir)
	err := sysroot.InitializeFS()
	if err != nil {
		log.Fatal(err)
	}

	err = sysroot.InitOsname(ot.Os, nil)
	if err != nil {
		log.Fatalf("%s", err)
	}

	dstRepo, err := sysroot.Repo(nil)
	if err != nil {
		log.Fatalf("%s", err)
	}

	err = dstRepo.RemoteAdd("origin", ot.RemoteRepository, nil, nil)
	if err != nil {
		log.Fatalf("%s", err)
	}

	var options ostree.PullOptions
	options.OverrideRemoteName = "origin"
	options.Refs = []string{ot.Branch}

	err = dstRepo.PullWithOptions(repoPath, options, nil, nil)
	if err != nil {
		log.Fatalf("%s", err)
	}

	/* Required by ostree to make sure a bunch of information was pulled in  */
	sysroot.Load(nil)

	revision, err := dstRepo.ResolveRev(ot.Branch, false)
	if err != nil {
		log.Fatalf("%s", err)
	}

	origin := sysroot.OriginNewFromRefspec(ot.Branch)
	deployment, err := sysroot.DeployTree(ot.Os, revision, origin, nil, nil, nil)
	if err != nil {
		log.Fatalf("%s", err)
	}

	err = sysroot.SimpleWriteDeployment(ot.Os, deployment, nil, 0, nil)
	if err != nil {
		log.Fatalf("%s", err)
	}
}
<commit_msg>Import image fstab and kernel cmdline into the ostree deployment<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"strings"

	ostree "github.com/sjoerdsimons/ostree-go/pkg/otbuiltin"
)

type OstreeDeployAction struct {
	*BaseAction
	Repository       string
	RemoteRepository string
	Branch           string
	Os               string
}

func (ot *OstreeDeployAction) Run(context *YaibContext) {
	repoPath := "file://" + path.Join(context.artifactdir, ot.Repository)

	sysroot := ostree.NewSysroot(context.imageMntDir)
	err := sysroot.InitializeFS()
	if err != nil {
		log.Fatal(err)
	}

	err = sysroot.InitOsname(ot.Os, nil)
	if err != nil {
		log.Fatalf("%s", err)
	}

	dstRepo, err := sysroot.Repo(nil)
	if err != nil {
		log.Fatalf("%s", err)
	}

	err = dstRepo.RemoteAdd("origin", ot.RemoteRepository, nil, nil)
	if err != nil {
		log.Fatalf("%s", err)
	}

	var options ostree.PullOptions
	options.OverrideRemoteName = "origin"
	options.Refs = []string{ot.Branch}

	err = dstRepo.PullWithOptions(repoPath, options, nil, nil)
	if err != nil {
		log.Fatalf("%s", err)
	}

	/* Required by ostree to make sure a bunch of information was pulled in  */
	sysroot.Load(nil)

	revision, err := dstRepo.ResolveRev(ot.Branch, false)
	if err != nil {
		log.Fatalf("%s", err)
	}

	cmdline, _ := ioutil.ReadFile(path.Join(context.imageMntDir, "etc/kernel/cmdline"))
	kargs := strings.Split(strings.TrimSpace(string(cmdline)), " ")

	origin := sysroot.OriginNewFromRefspec(ot.Branch)
	deployment, err := sysroot.DeployTree(ot.Os, revision, origin, nil, kargs, nil)
	if err != nil {
		log.Fatalf("%s", err)
	}

	deploymentDir := fmt.Sprintf("ostree/deploy/%s/deploy/%s.%d",
		deployment.Osname(), deployment.Csum(), deployment.Deployserial())

	etcDir := path.Join(context.imageMntDir, deploymentDir, "etc")

	err = os.Mkdir(etcDir, 755)
	if err != nil && !os.IsExist(err) {
		log.Fatalf("%s", err)
	}

	dst, err := os.OpenFile(path.Join(etcDir, "fstab"), os.O_WRONLY|os.O_CREATE, 0755)
	defer dst.Close()
	if err != nil {
		log.Fatalf("%s", err)
	}

	src, err := os.Open(path.Join(context.imageMntDir, "etc", "fstab"))
	defer src.Close()
	if err != nil {
		log.Fatalf("%s", err)
	}

	_, err = io.Copy(dst, src)
	if err != nil {
		log.Fatalf("%s", err)
	}

	err = sysroot.SimpleWriteDeployment(ot.Os, deployment, nil, 0, nil)
	if err != nil {
		log.Fatalf("%s", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
    "image"
    "image/png"
    "os"
    "io"
    "log"
    lib "github.com/johnny-morrice/godelbrot/libgodelbrot"
)

func main() {
    var input io.Reader = os.Stdin
    var output io.Writer = os.Stdout

    frch := lib.ReadInfoStream(input)
    imgch := make(chan image.Image)

    go func() {
        for frpkt := range frch {
            if frpkt.Err != nil {
                log.Fatal(frpkt.Err)
            }
            picture, renderErr := lib.Render(frpkt.Info)

            if renderErr != nil {
                log.Fatal("Render errror:", renderErr)
            }

            imgch<- picture
        }
        close(imgch)
    }()

    for picture := range imgch {
        encodeErr := png.Encode(output, picture)

        if encodeErr != nil {
            log.Fatal("Encoding error:", encodeErr)
        }
    }

}<commit_msg>Pretty formatting<commit_after>package main

import (
    "image"
    "image/png"
    "os"
    "io"
    "log"
    lib "github.com/johnny-morrice/godelbrot/libgodelbrot"
)

func main() {
    var input io.Reader = os.Stdin
    var output io.Writer = os.Stdout

    frch := lib.ReadInfoStream(input)
    imgch := make(chan image.Image)

    go func() {
        for frpkt := range frch {
            if frpkt.Err != nil {
                log.Fatal(frpkt.Err)
            }
            picture, renderErr := lib.Render(frpkt.Info)

            if renderErr != nil {
                log.Fatal("Render errror:", renderErr)
            }

            imgch<- picture
        }
        close(imgch)
    }()

    for picture := range imgch {
        encodeErr := png.Encode(output, picture)

        if encodeErr != nil {
            log.Fatal("Encoding error:", encodeErr)
        }
    }
}
<|endoftext|>


<commit_before>package main

import "fmt"

const (
	cInvalid = iota
	cNorth   = iota
	cSouth   = iota
	cEast    = iota
	cWest    = iota
	cFound   = iota
)

type GuessResult struct {
	VerticalPosition   int
	HorizontalPosition int
	GuessCount         int
}

func (gr GuessResult) String() string {
	v_part := "Invalid"
	h_part := "Invalid"
	if gr.VerticalPosition == cNorth {
		v_part = "North"
	}
	if gr.VerticalPosition == cSouth {
		v_part = "South"
	}
	if gr.VerticalPosition == cFound {
		v_part = "Found"
	}

	if gr.HorizontalPosition == cEast {
		h_part = "East"
	}
	if gr.HorizontalPosition == cWest {
		h_part = "West"
	}
	if gr.HorizontalPosition == cFound {
		h_part = "Found"
	}
	if gr.HorizontalPosition == cFound && gr.VerticalPosition == cFound {
		return fmt.Sprintf("Guess #%v: You got it!", gr.GuessCount)
	}
	return fmt.Sprintf("Guess #%v: Target is %v and %v from your guess.", gr.GuessCount, v_part, h_part)
}
<commit_msg>No need for all this In GuessResult Simpler object.  May not need it at all.<commit_after>package main

type GuessResult struct {
	Message    int
	Found      bool
	GuessCount int
}
<|endoftext|>


<commit_before>package softlayer

import (
	"errors"
	"fmt"
	"log"
	"strconv"
	"strings"
	"time"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
	"github.ibm.com/riethm/gopherlayer.git/datatypes"
	"github.ibm.com/riethm/gopherlayer.git/filter"
	"github.ibm.com/riethm/gopherlayer.git/helpers/location"
	"github.ibm.com/riethm/gopherlayer.git/helpers/product"
	"github.ibm.com/riethm/gopherlayer.git/services"
	"github.ibm.com/riethm/gopherlayer.git/session"
	"github.ibm.com/riethm/gopherlayer.git/sl"
)

const (
	LB_LARGE_150000_CONNECTIONS = 150000
	LB_SMALL_15000_CONNECTIONS  = 15000

	LbLocalPackageType = "ADDITIONAL_SERVICES_LOAD_BALANCER"

	lbMask = "id,connectionLimit,ipAddressId,securityCertificateId,highAvailabilityFlag," +
		"sslEnabledFlag,loadBalancerHardware[datacenter[name]],ipAddress[ipAddress,subnetId]"
)

func resourceSoftLayerLbLocal() *schema.Resource {
	return &schema.Resource{
		Create:   resourceSoftLayerLbLocalCreate,
		Read:     resourceSoftLayerLbLocalRead,
		Update:   resourceSoftLayerLbLocalUpdate,
		Delete:   resourceSoftLayerLbLocalDelete,
		Exists:   resourceSoftLayerLbLocalExists,
		Importer: &schema.ResourceImporter{},

		Schema: map[string]*schema.Schema{
			"connections": {
				Type:     schema.TypeInt,
				Required: true,
				ForceNew: true,
			},
			"datacenter": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"ha_enabled": {
				Type:     schema.TypeBool,
				Required: true,
				ForceNew: true,
			},
			"security_certificate_id": {
				Type:     schema.TypeInt,
				Optional: true,
			},
			"ip_address": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"subnet_id": {
				Type:     schema.TypeInt,
				Computed: true,
			},
		},
	}
}

func resourceSoftLayerLbLocalCreate(d *schema.ResourceData, meta interface{}) error {
	sess := meta.(*session.Session)

	connections := d.Get("connections").(int)
	haEnabled := d.Get("ha_enabled").(bool)

	// SoftLayer capacities don't match the published capacities as seen in the local lb
	// ordering screen in the customer portal. Terraform exposes the published capacities.
	// Create a translation map for those cases where the published capacity does not
	// equal the actual actual capacity on the product_item.
	capacities := map[int]float64{
		15000:  65000.0,
		150000: 130000.0,
	}

	var capacity float64
	if c, ok := capacities[connections]; !ok {
		capacity = float64(connections)
	} else {
		capacity = c
	}

	var keyname string
	if haEnabled {
		keyname = "DEDICATED_LOAD_BALANCER_WITH_HIGH_AVAILABILITY_AND_SSL"
	} else {
		keyname = "LOAD_BALANCER_DEDICATED_WITH_SSL_OFFLOAD"
	}

	keyname = strings.Join([]string{keyname, strconv.Itoa(connections), "CONNECTIONS"}, "_")

	pkg, err := product.GetPackageByType(sess, LbLocalPackageType)
	if err != nil {
		return err
	}

	// Get all prices for ADDITIONAL_SERVICE_LOAD_BALANCER with the given capacity
	productItems, err := product.GetPackageProducts(sess, *pkg.Id)
	if err != nil {
		return err
	}

	// Select only those product items with a matching keyname
	targetItems := []datatypes.Product_Item{}
	for _, item := range productItems {
		if *item.KeyName == keyname {
			targetItems = append(targetItems, item)
		}
	}

	if len(targetItems) == 0 {
		return fmt.Errorf("No product items matching %s could be found", keyname)
	}

	//select prices with the required capacity
	prices := product.SelectProductPricesByCategory(
		targetItems,
		map[string]float64{
			product.DedicatedLoadBalancerCategoryCode: capacity,
		},
	)

	// Lookup the datacenter ID
	dc, err := location.GetDatacenterByName(sess, d.Get("datacenter").(string))

	productOrderContainer := datatypes.Container_Product_Order_Network_LoadBalancer{
		Container_Product_Order: datatypes.Container_Product_Order{
			PackageId: pkg.Id,
			Location:  sl.String(strconv.Itoa(*dc.Id)),
			Prices:    prices[:1],
			Quantity:  sl.Int(1),
		},
	}

	log.Println("[INFO] Creating load balancer")

	receipt, err := services.GetProductOrderService(sess).
		PlaceOrder(&productOrderContainer, sl.Bool(false))
	if err != nil {
		return fmt.Errorf("Error during creation of load balancer: %s", err)
	}

	loadBalancer, err := findLoadBalancerByOrderId(sess, *receipt.OrderId)

	d.SetId(fmt.Sprintf("%d", *loadBalancer.Id))
	d.Set("connections", getConnectionLimit(*loadBalancer.ConnectionLimit))
	d.Set("datacenter", *loadBalancer.LoadBalancerHardware[0].Datacenter.Name)
	d.Set("ip_address", *loadBalancer.IpAddress.IpAddress)
	d.Set("subnet_id", *loadBalancer.IpAddress.SubnetId)
	d.Set("ha_enabled", *loadBalancer.HighAvailabilityFlag)

	log.Printf("[INFO] Load Balancer ID: %s", d.Id())

	return resourceSoftLayerLbLocalUpdate(d, meta)
}

func resourceSoftLayerLbLocalUpdate(d *schema.ResourceData, meta interface{}) error {
	sess := meta.(*session.Session)

	vipID, _ := strconv.Atoi(d.Id())

	vip := datatypes.Network_Application_Delivery_Controller_LoadBalancer_VirtualIpAddress{
		SecurityCertificateId: sl.Int(d.Get("security_certificate_id").(int)),
	}

	success, err := services.GetNetworkApplicationDeliveryControllerLoadBalancerVirtualIpAddressService(sess).
		Id(vipID).
		EditObject(&vip)

	if err != nil {
		return fmt.Errorf("Update load balancer failed: %s", err)
	}

	if !success {
		return fmt.Errorf("Update load balancer failed: %s", err)
	}

	return resourceSoftLayerLbLocalRead(d, meta)
}

func resourceSoftLayerLbLocalRead(d *schema.ResourceData, meta interface{}) error {
	sess := meta.(*session.Session)

	vipID, _ := strconv.Atoi(d.Id())

	vip, err := services.GetNetworkApplicationDeliveryControllerLoadBalancerVirtualIpAddressService(sess).
		Id(vipID).
		Mask(lbMask).
		GetObject()

	if err != nil {
		return fmt.Errorf("Error retrieving load balancer: %s", err)
	}

	d.Set("connections", getConnectionLimit(*vip.ConnectionLimit))
	d.Set("datacenter", *vip.LoadBalancerHardware[0].Datacenter.Name)
	d.Set("ip_address", *vip.IpAddress.IpAddress)
	d.Set("subnet_id", *vip.IpAddress.SubnetId)
	d.Set("ha_enabled", *vip.HighAvailabilityFlag)

	// Optional fields.  Guard against nil pointer dereferences
	if vip.SecurityCertificateId == nil {
		d.Set("security_certificate_id", nil)
	} else {
		d.Set("security_certificate_id", *vip.SecurityCertificateId)
	}

	return nil
}

func resourceSoftLayerLbLocalDelete(d *schema.ResourceData, meta interface{}) error {
	sess := meta.(*session.Session)

	vipID, _ := strconv.Atoi(d.Id())

	// Get billing item associated with the load balancer
	bi, err := services.GetNetworkApplicationDeliveryControllerLoadBalancerVirtualIpAddressService(sess).
		Id(vipID).
		GetDedicatedBillingItem()

	if err != nil {
		return fmt.Errorf("Error while looking up billing item associated with the load balancer: %s", err)
	}

	return cancelService(sess, *bi.Id)
}

func resourceSoftLayerLbLocalExists(d *schema.ResourceData, meta interface{}) (bool, error) {
	sess := meta.(*session.Session)

	vipID, _ := strconv.Atoi(d.Id())

	_, err := services.GetNetworkApplicationDeliveryControllerLoadBalancerVirtualIpAddressService(sess).
		Id(vipID).
		Mask("id").
		GetObject()

	if err != nil {
		return false, err
	}

	return true, nil
}

/* When requesting 15000 SL creates between 15000 and 150000. When requesting 150000 SL creates >= 150000 */
func getConnectionLimit(connectionLimit int) int {
	if connectionLimit >= LB_LARGE_150000_CONNECTIONS {
		return LB_LARGE_150000_CONNECTIONS
	} else if connectionLimit >= LB_SMALL_15000_CONNECTIONS &&
		connectionLimit < LB_LARGE_150000_CONNECTIONS {
		return LB_SMALL_15000_CONNECTIONS
	} else {
		return 0
	}
}

func cancelService(sess *session.Session, billingId int) error {
	stateConf := &resource.StateChangeConf{
		Pending: []string{"transactions_pending"},
		Target:  []string{"complete"},
		Refresh: func() (interface{}, string, error) {
			success, err := services.GetBillingItemService(sess).Id(billingId).CancelService()

			if err != nil {
				if apiErr, ok := err.(sl.Error); ok {
					// TODO this logic depends too heavily on localized strings which could be translated.
					// Need to change this to check for pending transactions in a wait loop, and then
					// cancel the service once, reporting any success/error
					if strings.Index(apiErr.Message, "There is currently an active transaction") != -1 {
						return false, "transactions_pending", nil
					}
				}
				return success, "error", err
			}

			return success, "complete", nil
		},
		Timeout:    10 * time.Minute,
		Delay:      30 * time.Second,
		MinTimeout: 30 * time.Second,
	}

	pendingResult, err := stateConf.WaitForState()

	if err != nil {
		return err
	}

	if pendingResult != nil && !(pendingResult.(bool)) {
		return errors.New("SoftLayer reported an unsuccessful cancellation, but did not provide a reason.")
	}

	return nil
}

func findLoadBalancerByOrderId(sess *session.Session, orderId int) (datatypes.Network_Application_Delivery_Controller_LoadBalancer_VirtualIpAddress, error) {
	stateConf := &resource.StateChangeConf{
		Pending: []string{"pending"},
		Target:  []string{"complete"},
		Refresh: func() (interface{}, string, error) {
			lbs, err := services.GetAccountService(sess).
				Filter(filter.Build(
					filter.Path("adcLoadBalancers.dedicatedBillingItem.orderItem.order.id").
						Eq(strconv.Itoa(orderId)))).
				Mask(lbMask).
				GetAdcLoadBalancers()
			if err != nil {
				return datatypes.Network_Application_Delivery_Controller_LoadBalancer_VirtualIpAddress{}, "", err
			}

			if len(lbs) == 1 {
				return lbs[0], "complete", nil
			} else if len(lbs) == 0 {
				return nil, "pending", nil
			} else {
				return nil, "", fmt.Errorf("Expected one load balancer: %s", err)
			}
		},
		Timeout:    10 * time.Minute,
		Delay:      5 * time.Second,
		MinTimeout: 3 * time.Second,
	}

	pendingResult, err := stateConf.WaitForState()

	if err != nil {
		return datatypes.Network_Application_Delivery_Controller_LoadBalancer_VirtualIpAddress{}, err
	}

	var result, ok = pendingResult.(datatypes.Network_Application_Delivery_Controller_LoadBalancer_VirtualIpAddress)

	if ok {
		return result, nil
	}

	return datatypes.Network_Application_Delivery_Controller_LoadBalancer_VirtualIpAddress{},
		fmt.Errorf("Cannot find Application Delivery Controller Load Balancer with order id '%d'", orderId)
}
<commit_msg>Use Get function to retrieve optional fields in lb_local resource<commit_after>package softlayer

import (
	"errors"
	"fmt"
	"log"
	"strconv"
	"strings"
	"time"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
	"github.ibm.com/riethm/gopherlayer.git/datatypes"
	"github.ibm.com/riethm/gopherlayer.git/filter"
	"github.ibm.com/riethm/gopherlayer.git/helpers/location"
	"github.ibm.com/riethm/gopherlayer.git/helpers/product"
	"github.ibm.com/riethm/gopherlayer.git/services"
	"github.ibm.com/riethm/gopherlayer.git/session"
	"github.ibm.com/riethm/gopherlayer.git/sl"
)

const (
	LB_LARGE_150000_CONNECTIONS = 150000
	LB_SMALL_15000_CONNECTIONS  = 15000

	LbLocalPackageType = "ADDITIONAL_SERVICES_LOAD_BALANCER"

	lbMask = "id,connectionLimit,ipAddressId,securityCertificateId,highAvailabilityFlag," +
		"sslEnabledFlag,loadBalancerHardware[datacenter[name]],ipAddress[ipAddress,subnetId]"
)

func resourceSoftLayerLbLocal() *schema.Resource {
	return &schema.Resource{
		Create:   resourceSoftLayerLbLocalCreate,
		Read:     resourceSoftLayerLbLocalRead,
		Update:   resourceSoftLayerLbLocalUpdate,
		Delete:   resourceSoftLayerLbLocalDelete,
		Exists:   resourceSoftLayerLbLocalExists,
		Importer: &schema.ResourceImporter{},

		Schema: map[string]*schema.Schema{
			"connections": {
				Type:     schema.TypeInt,
				Required: true,
				ForceNew: true,
			},
			"datacenter": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"ha_enabled": {
				Type:     schema.TypeBool,
				Required: true,
				ForceNew: true,
			},
			"security_certificate_id": {
				Type:     schema.TypeInt,
				Optional: true,
			},
			"ip_address": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"subnet_id": {
				Type:     schema.TypeInt,
				Computed: true,
			},
		},
	}
}

func resourceSoftLayerLbLocalCreate(d *schema.ResourceData, meta interface{}) error {
	sess := meta.(*session.Session)

	connections := d.Get("connections").(int)
	haEnabled := d.Get("ha_enabled").(bool)

	// SoftLayer capacities don't match the published capacities as seen in the local lb
	// ordering screen in the customer portal. Terraform exposes the published capacities.
	// Create a translation map for those cases where the published capacity does not
	// equal the actual actual capacity on the product_item.
	capacities := map[int]float64{
		15000:  65000.0,
		150000: 130000.0,
	}

	var capacity float64
	if c, ok := capacities[connections]; !ok {
		capacity = float64(connections)
	} else {
		capacity = c
	}

	var keyname string
	if haEnabled {
		keyname = "DEDICATED_LOAD_BALANCER_WITH_HIGH_AVAILABILITY_AND_SSL"
	} else {
		keyname = "LOAD_BALANCER_DEDICATED_WITH_SSL_OFFLOAD"
	}

	keyname = strings.Join([]string{keyname, strconv.Itoa(connections), "CONNECTIONS"}, "_")

	pkg, err := product.GetPackageByType(sess, LbLocalPackageType)
	if err != nil {
		return err
	}

	// Get all prices for ADDITIONAL_SERVICE_LOAD_BALANCER with the given capacity
	productItems, err := product.GetPackageProducts(sess, *pkg.Id)
	if err != nil {
		return err
	}

	// Select only those product items with a matching keyname
	targetItems := []datatypes.Product_Item{}
	for _, item := range productItems {
		if *item.KeyName == keyname {
			targetItems = append(targetItems, item)
		}
	}

	if len(targetItems) == 0 {
		return fmt.Errorf("No product items matching %s could be found", keyname)
	}

	//select prices with the required capacity
	prices := product.SelectProductPricesByCategory(
		targetItems,
		map[string]float64{
			product.DedicatedLoadBalancerCategoryCode: capacity,
		},
	)

	// Lookup the datacenter ID
	dc, err := location.GetDatacenterByName(sess, d.Get("datacenter").(string))

	productOrderContainer := datatypes.Container_Product_Order_Network_LoadBalancer{
		Container_Product_Order: datatypes.Container_Product_Order{
			PackageId: pkg.Id,
			Location:  sl.String(strconv.Itoa(*dc.Id)),
			Prices:    prices[:1],
			Quantity:  sl.Int(1),
		},
	}

	log.Println("[INFO] Creating load balancer")

	receipt, err := services.GetProductOrderService(sess).
		PlaceOrder(&productOrderContainer, sl.Bool(false))
	if err != nil {
		return fmt.Errorf("Error during creation of load balancer: %s", err)
	}

	loadBalancer, err := findLoadBalancerByOrderId(sess, *receipt.OrderId)

	d.SetId(fmt.Sprintf("%d", *loadBalancer.Id))
	d.Set("connections", getConnectionLimit(*loadBalancer.ConnectionLimit))
	d.Set("datacenter", *loadBalancer.LoadBalancerHardware[0].Datacenter.Name)
	d.Set("ip_address", *loadBalancer.IpAddress.IpAddress)
	d.Set("subnet_id", *loadBalancer.IpAddress.SubnetId)
	d.Set("ha_enabled", *loadBalancer.HighAvailabilityFlag)

	log.Printf("[INFO] Load Balancer ID: %s", d.Id())

	return resourceSoftLayerLbLocalUpdate(d, meta)
}

func resourceSoftLayerLbLocalUpdate(d *schema.ResourceData, meta interface{}) error {
	sess := meta.(*session.Session)

	vipID, _ := strconv.Atoi(d.Id())

	vip := datatypes.Network_Application_Delivery_Controller_LoadBalancer_VirtualIpAddress{
		SecurityCertificateId: sl.Int(d.Get("security_certificate_id").(int)),
	}

	success, err := services.GetNetworkApplicationDeliveryControllerLoadBalancerVirtualIpAddressService(sess).
		Id(vipID).
		EditObject(&vip)

	if err != nil {
		return fmt.Errorf("Update load balancer failed: %s", err)
	}

	if !success {
		return fmt.Errorf("Update load balancer failed: %s", err)
	}

	return resourceSoftLayerLbLocalRead(d, meta)
}

func resourceSoftLayerLbLocalRead(d *schema.ResourceData, meta interface{}) error {
	sess := meta.(*session.Session)

	vipID, _ := strconv.Atoi(d.Id())

	vip, err := services.GetNetworkApplicationDeliveryControllerLoadBalancerVirtualIpAddressService(sess).
		Id(vipID).
		Mask(lbMask).
		GetObject()

	if err != nil {
		return fmt.Errorf("Error retrieving load balancer: %s", err)
	}

	d.Set("connections", getConnectionLimit(*vip.ConnectionLimit))
	d.Set("datacenter", *vip.LoadBalancerHardware[0].Datacenter.Name)
	d.Set("ip_address", *vip.IpAddress.IpAddress)
	d.Set("subnet_id", *vip.IpAddress.SubnetId)
	d.Set("ha_enabled", *vip.HighAvailabilityFlag)

	// Optional fields.  Guard against nil pointer dereferences
	d.Set("security_certificate_id", sl.Get(vip.SecurityCertificateId, nil))

	return nil
}

func resourceSoftLayerLbLocalDelete(d *schema.ResourceData, meta interface{}) error {
	sess := meta.(*session.Session)

	vipID, _ := strconv.Atoi(d.Id())

	// Get billing item associated with the load balancer
	bi, err := services.GetNetworkApplicationDeliveryControllerLoadBalancerVirtualIpAddressService(sess).
		Id(vipID).
		GetDedicatedBillingItem()

	if err != nil {
		return fmt.Errorf("Error while looking up billing item associated with the load balancer: %s", err)
	}

	return cancelService(sess, *bi.Id)
}

func resourceSoftLayerLbLocalExists(d *schema.ResourceData, meta interface{}) (bool, error) {
	sess := meta.(*session.Session)

	vipID, _ := strconv.Atoi(d.Id())

	_, err := services.GetNetworkApplicationDeliveryControllerLoadBalancerVirtualIpAddressService(sess).
		Id(vipID).
		Mask("id").
		GetObject()

	if err != nil {
		return false, err
	}

	return true, nil
}

/* When requesting 15000 SL creates between 15000 and 150000. When requesting 150000 SL creates >= 150000 */
func getConnectionLimit(connectionLimit int) int {
	if connectionLimit >= LB_LARGE_150000_CONNECTIONS {
		return LB_LARGE_150000_CONNECTIONS
	} else if connectionLimit >= LB_SMALL_15000_CONNECTIONS &&
		connectionLimit < LB_LARGE_150000_CONNECTIONS {
		return LB_SMALL_15000_CONNECTIONS
	} else {
		return 0
	}
}

func cancelService(sess *session.Session, billingId int) error {
	stateConf := &resource.StateChangeConf{
		Pending: []string{"transactions_pending"},
		Target:  []string{"complete"},
		Refresh: func() (interface{}, string, error) {
			success, err := services.GetBillingItemService(sess).Id(billingId).CancelService()

			if err != nil {
				if apiErr, ok := err.(sl.Error); ok {
					// TODO this logic depends too heavily on localized strings which could be translated.
					// Need to change this to check for pending transactions in a wait loop, and then
					// cancel the service once, reporting any success/error
					if strings.Index(apiErr.Message, "There is currently an active transaction") != -1 {
						return false, "transactions_pending", nil
					}
				}
				return success, "error", err
			}

			return success, "complete", nil
		},
		Timeout:    10 * time.Minute,
		Delay:      30 * time.Second,
		MinTimeout: 30 * time.Second,
	}

	pendingResult, err := stateConf.WaitForState()

	if err != nil {
		return err
	}

	if pendingResult != nil && !(pendingResult.(bool)) {
		return errors.New("SoftLayer reported an unsuccessful cancellation, but did not provide a reason.")
	}

	return nil
}

func findLoadBalancerByOrderId(sess *session.Session, orderId int) (datatypes.Network_Application_Delivery_Controller_LoadBalancer_VirtualIpAddress, error) {
	stateConf := &resource.StateChangeConf{
		Pending: []string{"pending"},
		Target:  []string{"complete"},
		Refresh: func() (interface{}, string, error) {
			lbs, err := services.GetAccountService(sess).
				Filter(filter.Build(
					filter.Path("adcLoadBalancers.dedicatedBillingItem.orderItem.order.id").
						Eq(strconv.Itoa(orderId)))).
				Mask(lbMask).
				GetAdcLoadBalancers()
			if err != nil {
				return datatypes.Network_Application_Delivery_Controller_LoadBalancer_VirtualIpAddress{}, "", err
			}

			if len(lbs) == 1 {
				return lbs[0], "complete", nil
			} else if len(lbs) == 0 {
				return nil, "pending", nil
			} else {
				return nil, "", fmt.Errorf("Expected one load balancer: %s", err)
			}
		},
		Timeout:    10 * time.Minute,
		Delay:      5 * time.Second,
		MinTimeout: 3 * time.Second,
	}

	pendingResult, err := stateConf.WaitForState()

	if err != nil {
		return datatypes.Network_Application_Delivery_Controller_LoadBalancer_VirtualIpAddress{}, err
	}

	var result, ok = pendingResult.(datatypes.Network_Application_Delivery_Controller_LoadBalancer_VirtualIpAddress)

	if ok {
		return result, nil
	}

	return datatypes.Network_Application_Delivery_Controller_LoadBalancer_VirtualIpAddress{},
		fmt.Errorf("Cannot find Application Delivery Controller Load Balancer with order id '%d'", orderId)
}
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"bytes"
	"context"
	"crypto"
	"crypto/x509"
	"fmt"
	"math/rand"
	"net"
	"sort"
	"time"

	log "github.com/golang/glog"
	"github.com/golang/protobuf/proto"

	"github.com/google/fleetspeak/fleetspeak/src/common"
	"github.com/google/fleetspeak/fleetspeak/src/server/authorizer"
	"github.com/google/fleetspeak/fleetspeak/src/server/comms"
	"github.com/google/fleetspeak/fleetspeak/src/server/db"
	"github.com/google/fleetspeak/fleetspeak/src/server/internal/signatures"
	"github.com/google/fleetspeak/fleetspeak/src/server/stats"

	fspb "github.com/google/fleetspeak/fleetspeak/src/common/proto/fleetspeak"
)

const maxMessagesPerContact = 100
const processingChunkSize = 10

type commsContext struct {
	s *Server
}

func (c commsContext) InitializeConnection(ctx context.Context, addr net.Addr, key crypto.PublicKey, wcd *fspb.WrappedContactData) (*comms.ConnectionInfo, *fspb.ContactData, error) {
	id, err := common.MakeClientID(key)
	if err != nil {
		return nil, nil, err
	}

	contactInfo := authorizer.ContactInfo{
		ID:           id,
		ContactSize:  len(wcd.ContactData),
		ClientLabels: wcd.ClientLabels,
	}
	if !c.s.authorizer.Allow2(addr, contactInfo) {
		return nil, nil, comms.NotAuthorizedError
	}

	ci, err := c.getClientInfo(ctx, id)
	if err != nil {
		return nil, nil, err
	}

	res := comms.ConnectionInfo{
		Addr: addr,
	}
	if ci == nil {
		res.AuthClientInfo.New = true
	} else {
		res.Client = *ci
		res.AuthClientInfo.Labels = ci.Labels
	}

	if !c.s.authorizer.Allow3(addr, contactInfo, res.AuthClientInfo) {
		return nil, nil, comms.NotAuthorizedError
	}

	if ci == nil {
		if err := c.addClient(ctx, id, key); err != nil {
			return nil, nil, err
		}
		res.Client.ID = id
		res.Client.Key = key
		// Set initial labels for the client according to the contact data. Going
		// forward, labels will be adjusted when the client sends a ClientInfo
		// message (see system_service.go).
		for _, l := range wcd.ClientLabels {
			cl := &fspb.Label{ServiceName: "client", Label: l}

			// Ignore errors - if this fails, the first ClientInfo message will try again
			// in a context where we can retry easily.
			c.s.dataStore.AddClientLabel(ctx, id, cl)

			res.Client.Labels = append(res.Client.Labels, cl)
		}
		res.AuthClientInfo.Labels = res.Client.Labels
	}

	sigs, err := signatures.ValidateWrappedContactData(id, wcd)
	if err != nil {
		return nil, nil, err
	}
	accept, validationInfo := c.s.authorizer.Allow4(
		addr,
		contactInfo,
		res.AuthClientInfo,
		sigs)
	if !accept {
		return nil, nil, comms.NotAuthorizedError
	}

	var cd fspb.ContactData
	if err = proto.Unmarshal(wcd.ContactData, &cd); err != nil {
		return nil, nil, fmt.Errorf("unable to parse contact_data: %v", err)
	}
	if len(cd.Messages) > maxMessagesPerContact {
		return nil, nil, fmt.Errorf("contact_data contains %d messages, only %d allowed", len(cd.Messages), maxMessagesPerContact)
	}
	res.NonceReceived = cd.SequencingNonce
	toSend := fspb.ContactData{SequencingNonce: uint64(rand.Int63())}
	res.NonceSent = toSend.SequencingNonce
	res.ContactID, err = c.s.dataStore.RecordClientContact(ctx,
		db.ContactData{
			ClientID:      id,
			NonceSent:     toSend.SequencingNonce,
			NonceReceived: cd.SequencingNonce,
			Addr:          addr.String(),
			ClientClock:   cd.ClientClock,
		})
	if err != nil {
		return nil, nil, err
	}

	err = c.handleMessagesFromClient(ctx, &res.Client, res.ContactID, &cd, validationInfo)
	if err != nil {
		return nil, nil, err
	}

	toSend.Messages, err = c.FindMessagesForClient(ctx, &res.Client, res.ContactID, 100)
	if err != nil {
		return nil, nil, err
	}

	res.AuthClientInfo.New = false
	return &res, &toSend, nil
}

// getClientInfo loads basic information about a client. Returns nil if the client does
// not exist in the datastore.
func (c commsContext) getClientInfo(ctx context.Context, id common.ClientID) (*comms.ClientInfo, error) {
	cld, cacheHit, err := c.s.clientCache.GetOrRead(ctx, id, c.s.dataStore)
	if err != nil {
		if c.s.dataStore.IsNotFound(err) {
			return nil, nil
		}
		return nil, err
	}

	k, err := x509.ParsePKIXPublicKey(cld.Key)
	if err != nil {
		return nil, err
	}
	return &comms.ClientInfo{
		ID:          id,
		Key:         k,
		Labels:      cld.Labels,
		Blacklisted: cld.Blacklisted,
		Cached:      cacheHit}, nil
}

func (c commsContext) HandleMessagesFromClient(ctx context.Context, info *comms.ConnectionInfo, wcd *fspb.WrappedContactData) error {
	sigs, err := signatures.ValidateWrappedContactData(info.Client.ID, wcd)
	if err != nil {
		return err
	}

	accept, validationInfo := c.s.authorizer.Allow4(
		info.Addr,
		authorizer.ContactInfo{
			ID:           info.Client.ID,
			ContactSize:  len(wcd.ContactData),
			ClientLabels: wcd.ClientLabels,
		},
		info.AuthClientInfo,
		sigs)
	if !accept {
		return comms.NotAuthorizedError
	}

	var cd fspb.ContactData
	if err = proto.Unmarshal(wcd.ContactData, &cd); err != nil {
		return fmt.Errorf("unable to parse contact_data: %v", err)
	}
	if len(cd.Messages) > maxMessagesPerContact {
		return fmt.Errorf("contact_data contains %d messages, only %d allowed", len(cd.Messages), maxMessagesPerContact)
	}

	err = c.handleMessagesFromClient(ctx, &info.Client, info.ContactID, &cd, validationInfo)
	if err != nil {
		return err
	}
	return nil
}

func (c commsContext) GetMessagesForClient(ctx context.Context, info *comms.ConnectionInfo) (*fspb.ContactData, error) {
	toSend := fspb.ContactData{
		SequencingNonce: info.NonceSent,
	}
	var err error
	toSend.Messages, err = c.FindMessagesForClient(ctx, &info.Client, info.ContactID, 100)
	if err != nil || len(toSend.Messages) == 0 {
		return nil, err
	}
	return &toSend, nil
}

// addClient adds a new client to the system.
func (c commsContext) addClient(ctx context.Context, id common.ClientID, key crypto.PublicKey) error {
	k, err := x509.MarshalPKIXPublicKey(key)
	if err != nil {
		return err
	}
	if err := c.s.dataStore.AddClient(ctx, id, &db.ClientData{Key: k}); err != nil {
		return err
	}
	return nil
}

// FindMessagesForClient finds unprocessed messages for a given client and
// reserves them for processing.
func (c commsContext) FindMessagesForClient(ctx context.Context, info *comms.ClientInfo, contactID db.ContactID, maxMessages int) ([]*fspb.Message, error) {
	if info.Blacklisted {
		log.Warningf("Contact from blacklisted id [%v], creating RekeyRequest.", info.ID)
		m, err := c.MakeBlacklistMessage(ctx, info, contactID)
		return []*fspb.Message{m}, err
	}
	msgs, err := c.s.dataStore.ClientMessagesForProcessing(ctx, info.ID, maxMessages)
	if err != nil {
		if len(msgs) == 0 {
			return nil, err
		}
		log.Warning("Got %v messages along with error, continuing: %v", len(msgs), err)
	}

	// If the client recently contacted us, the broadcast situation is unlikely to
	// have changed, so we skip checking for broadcasts. To keep this from delaying
	// broadcast distribution, the broadcast manager clears the client cache when it
	// finds more broadcasts.
	if !info.Cached {
		bms, err := c.s.broadcastManager.MakeBroadcastMessagesForClient(ctx, info.ID, info.Labels)
		if err != nil {
			return nil, err
		}
		msgs = append(msgs, bms...)
	}

	if len(msgs) == 0 {
		return msgs, nil
	}

	mids := make([]common.MessageID, 0, len(msgs))
	for _, m := range msgs {
		id, err := common.BytesToMessageID(m.MessageId)
		if err != nil {
			return nil, err
		}
		mids = append(mids, id)
	}
	err = c.s.dataStore.LinkMessagesToContact(ctx, contactID, mids)
	if err != nil {
		return nil, err
	}
	return msgs, nil
}

func (c commsContext) MakeBlacklistMessage(ctx context.Context, info *comms.ClientInfo, contactID db.ContactID) (*fspb.Message, error) {
	mid, err := common.RandomMessageID()
	if err != nil {
		return nil, fmt.Errorf("unable to create message id: %v", err)
	}
	msg := &fspb.Message{
		MessageId: mid.Bytes(),
		Source: &fspb.Address{
			ServiceName: "system",
		},
		Destination: &fspb.Address{
			ServiceName: "system",
			ClientId:    info.ID.Bytes(),
		},
		MessageType:  "RekeyRequest",
		CreationTime: db.NowProto(),
	}
	if err = c.s.dataStore.StoreMessages(ctx, []*fspb.Message{msg}, contactID); err != nil {
		return nil, fmt.Errorf("unable to store RekeyRequest: %v", err)
	}
	return msg, nil
}

func (c commsContext) validateMessageFromClient(id common.ClientID, m *fspb.Message, validationInfo *fspb.ValidationInfo) error {
	if m.Destination == nil {
		return fmt.Errorf("message must have Destination")
	}
	if m.Destination.ClientId != nil {
		return fmt.Errorf("cannot send a message directly to another client [%v]", m.Destination.ClientId)
	}
	if m.Source == nil || m.Source.ServiceName == "" {
		return fmt.Errorf("message must have a source with a ServiceName, got: %v", m.Source)
	}
	if m.SourceMessageId == nil {
		return fmt.Errorf("source message id cannot be empty")
	}

	m.Source.ClientId = id.Bytes()
	m.ValidationInfo = validationInfo
	m.MessageId = common.MakeMessageID(m.Source, m.SourceMessageId).Bytes()
	return nil
}

// handleMessagesFromClient processes a block of messages from a particular
// client. It saves them to the database, associates them with the contact
// identified by contactTime, and processes them.
func (c commsContext) handleMessagesFromClient(ctx context.Context, info *comms.ClientInfo, contactID db.ContactID, received *fspb.ContactData, validationInfo *fspb.ValidationInfo) error {
	msgs := make([]*fspb.Message, 0, len(received.Messages))
	for _, m := range received.Messages {
		err := c.validateMessageFromClient(info.ID, m, validationInfo)
		if err != nil {
			log.Errorf("Dropping invalid message from [%v]: %v", info.ID, err)
			continue
		}
		msgs = append(msgs, m)
	}
	if len(msgs) == 0 {
		return nil
	}

	sort.Slice(msgs, func(a, b int) bool {
		return bytes.Compare(msgs[a].MessageId, msgs[b].MessageId) == -1
	})

	for {
		if len(msgs) <= processingChunkSize {
			return c.s.serviceConfig.HandleNewMessages(ctx, msgs, contactID)
		}

		if err := c.s.serviceConfig.HandleNewMessages(ctx, msgs[:processingChunkSize], contactID); err != nil {
			return err
		}
		msgs = msgs[processingChunkSize:]
	}
}

// ReadFile returns the data and modification time of file. Caller is
// responsible for closing data.
//
// Calls to data are permitted to fail if ctx is canceled or expired.
func (c commsContext) ReadFile(ctx context.Context, service, name string) (data db.ReadSeekerCloser, modtime time.Time, err error) {
	return c.s.dataStore.ReadFile(ctx, service, name)
}

// IsNotFound returns whether an error returned by ReadFile indicates that the
// file was not found.
func (c commsContext) IsNotFound(err error) bool {
	return c.s.dataStore.IsNotFound(err)
}

// StatsCollector returns the stats.Collector used by the Fleetspeak
// system. Access is provided to allow collection of stats relating to the
// client communication.
func (c commsContext) StatsCollector() stats.Collector {
	return c.s.statsCollector
}

func (c commsContext) Authorizer() authorizer.Authorizer {
	return c.s.authorizer
}
<commit_msg>Use crypto rand for nonce generation.<commit_after>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"bytes"
	"context"
	"crypto"
	"crypto/rand"
	"crypto/x509"
	"encoding/binary"
	"fmt"
	"net"
	"sort"
	"time"

	log "github.com/golang/glog"
	"github.com/golang/protobuf/proto"

	"github.com/google/fleetspeak/fleetspeak/src/common"
	"github.com/google/fleetspeak/fleetspeak/src/server/authorizer"
	"github.com/google/fleetspeak/fleetspeak/src/server/comms"
	"github.com/google/fleetspeak/fleetspeak/src/server/db"
	"github.com/google/fleetspeak/fleetspeak/src/server/internal/signatures"
	"github.com/google/fleetspeak/fleetspeak/src/server/stats"

	fspb "github.com/google/fleetspeak/fleetspeak/src/common/proto/fleetspeak"
)

const maxMessagesPerContact = 100
const processingChunkSize = 10

type commsContext struct {
	s *Server
}

func randUint64() uint64 {
	var b [8]byte
	_, err := rand.Read(b[:])
	if err != nil {
		// Random numbers are required for the correct operation of a FS server.
		panic(fmt.Errorf("unable to read random bytes: %v", err))
	}
	return binary.LittleEndian.Uint64(b[:])
}

func (c commsContext) InitializeConnection(ctx context.Context, addr net.Addr, key crypto.PublicKey, wcd *fspb.WrappedContactData) (*comms.ConnectionInfo, *fspb.ContactData, error) {
	id, err := common.MakeClientID(key)
	if err != nil {
		return nil, nil, err
	}

	contactInfo := authorizer.ContactInfo{
		ID:           id,
		ContactSize:  len(wcd.ContactData),
		ClientLabels: wcd.ClientLabels,
	}
	if !c.s.authorizer.Allow2(addr, contactInfo) {
		return nil, nil, comms.NotAuthorizedError
	}

	ci, err := c.getClientInfo(ctx, id)
	if err != nil {
		return nil, nil, err
	}

	res := comms.ConnectionInfo{
		Addr: addr,
	}
	if ci == nil {
		res.AuthClientInfo.New = true
	} else {
		res.Client = *ci
		res.AuthClientInfo.Labels = ci.Labels
	}

	if !c.s.authorizer.Allow3(addr, contactInfo, res.AuthClientInfo) {
		return nil, nil, comms.NotAuthorizedError
	}

	if ci == nil {
		if err := c.addClient(ctx, id, key); err != nil {
			return nil, nil, err
		}
		res.Client.ID = id
		res.Client.Key = key
		// Set initial labels for the client according to the contact data. Going
		// forward, labels will be adjusted when the client sends a ClientInfo
		// message (see system_service.go).
		for _, l := range wcd.ClientLabels {
			cl := &fspb.Label{ServiceName: "client", Label: l}

			// Ignore errors - if this fails, the first ClientInfo message will try again
			// in a context where we can retry easily.
			c.s.dataStore.AddClientLabel(ctx, id, cl)

			res.Client.Labels = append(res.Client.Labels, cl)
		}
		res.AuthClientInfo.Labels = res.Client.Labels
	}

	sigs, err := signatures.ValidateWrappedContactData(id, wcd)
	if err != nil {
		return nil, nil, err
	}
	accept, validationInfo := c.s.authorizer.Allow4(
		addr,
		contactInfo,
		res.AuthClientInfo,
		sigs)
	if !accept {
		return nil, nil, comms.NotAuthorizedError
	}

	var cd fspb.ContactData
	if err = proto.Unmarshal(wcd.ContactData, &cd); err != nil {
		return nil, nil, fmt.Errorf("unable to parse contact_data: %v", err)
	}
	if len(cd.Messages) > maxMessagesPerContact {
		return nil, nil, fmt.Errorf("contact_data contains %d messages, only %d allowed", len(cd.Messages), maxMessagesPerContact)
	}
	res.NonceReceived = cd.SequencingNonce
	toSend := fspb.ContactData{SequencingNonce: randUint64()}
	res.NonceSent = toSend.SequencingNonce
	res.ContactID, err = c.s.dataStore.RecordClientContact(ctx,
		db.ContactData{
			ClientID:      id,
			NonceSent:     toSend.SequencingNonce,
			NonceReceived: cd.SequencingNonce,
			Addr:          addr.String(),
			ClientClock:   cd.ClientClock,
		})
	if err != nil {
		return nil, nil, err
	}

	err = c.handleMessagesFromClient(ctx, &res.Client, res.ContactID, &cd, validationInfo)
	if err != nil {
		return nil, nil, err
	}

	toSend.Messages, err = c.FindMessagesForClient(ctx, &res.Client, res.ContactID, 100)
	if err != nil {
		return nil, nil, err
	}

	res.AuthClientInfo.New = false
	return &res, &toSend, nil
}

// getClientInfo loads basic information about a client. Returns nil if the client does
// not exist in the datastore.
func (c commsContext) getClientInfo(ctx context.Context, id common.ClientID) (*comms.ClientInfo, error) {
	cld, cacheHit, err := c.s.clientCache.GetOrRead(ctx, id, c.s.dataStore)
	if err != nil {
		if c.s.dataStore.IsNotFound(err) {
			return nil, nil
		}
		return nil, err
	}

	k, err := x509.ParsePKIXPublicKey(cld.Key)
	if err != nil {
		return nil, err
	}
	return &comms.ClientInfo{
		ID:          id,
		Key:         k,
		Labels:      cld.Labels,
		Blacklisted: cld.Blacklisted,
		Cached:      cacheHit}, nil
}

func (c commsContext) HandleMessagesFromClient(ctx context.Context, info *comms.ConnectionInfo, wcd *fspb.WrappedContactData) error {
	sigs, err := signatures.ValidateWrappedContactData(info.Client.ID, wcd)
	if err != nil {
		return err
	}

	accept, validationInfo := c.s.authorizer.Allow4(
		info.Addr,
		authorizer.ContactInfo{
			ID:           info.Client.ID,
			ContactSize:  len(wcd.ContactData),
			ClientLabels: wcd.ClientLabels,
		},
		info.AuthClientInfo,
		sigs)
	if !accept {
		return comms.NotAuthorizedError
	}

	var cd fspb.ContactData
	if err = proto.Unmarshal(wcd.ContactData, &cd); err != nil {
		return fmt.Errorf("unable to parse contact_data: %v", err)
	}
	if len(cd.Messages) > maxMessagesPerContact {
		return fmt.Errorf("contact_data contains %d messages, only %d allowed", len(cd.Messages), maxMessagesPerContact)
	}

	err = c.handleMessagesFromClient(ctx, &info.Client, info.ContactID, &cd, validationInfo)
	if err != nil {
		return err
	}
	return nil
}

func (c commsContext) GetMessagesForClient(ctx context.Context, info *comms.ConnectionInfo) (*fspb.ContactData, error) {
	toSend := fspb.ContactData{
		SequencingNonce: info.NonceSent,
	}
	var err error
	toSend.Messages, err = c.FindMessagesForClient(ctx, &info.Client, info.ContactID, 100)
	if err != nil || len(toSend.Messages) == 0 {
		return nil, err
	}
	return &toSend, nil
}

// addClient adds a new client to the system.
func (c commsContext) addClient(ctx context.Context, id common.ClientID, key crypto.PublicKey) error {
	k, err := x509.MarshalPKIXPublicKey(key)
	if err != nil {
		return err
	}
	if err := c.s.dataStore.AddClient(ctx, id, &db.ClientData{Key: k}); err != nil {
		return err
	}
	return nil
}

// FindMessagesForClient finds unprocessed messages for a given client and
// reserves them for processing.
func (c commsContext) FindMessagesForClient(ctx context.Context, info *comms.ClientInfo, contactID db.ContactID, maxMessages int) ([]*fspb.Message, error) {
	if info.Blacklisted {
		log.Warningf("Contact from blacklisted id [%v], creating RekeyRequest.", info.ID)
		m, err := c.MakeBlacklistMessage(ctx, info, contactID)
		return []*fspb.Message{m}, err
	}
	msgs, err := c.s.dataStore.ClientMessagesForProcessing(ctx, info.ID, maxMessages)
	if err != nil {
		if len(msgs) == 0 {
			return nil, err
		}
		log.Warning("Got %v messages along with error, continuing: %v", len(msgs), err)
	}

	// If the client recently contacted us, the broadcast situation is unlikely to
	// have changed, so we skip checking for broadcasts. To keep this from delaying
	// broadcast distribution, the broadcast manager clears the client cache when it
	// finds more broadcasts.
	if !info.Cached {
		bms, err := c.s.broadcastManager.MakeBroadcastMessagesForClient(ctx, info.ID, info.Labels)
		if err != nil {
			return nil, err
		}
		msgs = append(msgs, bms...)
	}

	if len(msgs) == 0 {
		return msgs, nil
	}

	mids := make([]common.MessageID, 0, len(msgs))
	for _, m := range msgs {
		id, err := common.BytesToMessageID(m.MessageId)
		if err != nil {
			return nil, err
		}
		mids = append(mids, id)
	}
	err = c.s.dataStore.LinkMessagesToContact(ctx, contactID, mids)
	if err != nil {
		return nil, err
	}
	return msgs, nil
}

func (c commsContext) MakeBlacklistMessage(ctx context.Context, info *comms.ClientInfo, contactID db.ContactID) (*fspb.Message, error) {
	mid, err := common.RandomMessageID()
	if err != nil {
		return nil, fmt.Errorf("unable to create message id: %v", err)
	}
	msg := &fspb.Message{
		MessageId: mid.Bytes(),
		Source: &fspb.Address{
			ServiceName: "system",
		},
		Destination: &fspb.Address{
			ServiceName: "system",
			ClientId:    info.ID.Bytes(),
		},
		MessageType:  "RekeyRequest",
		CreationTime: db.NowProto(),
	}
	if err = c.s.dataStore.StoreMessages(ctx, []*fspb.Message{msg}, contactID); err != nil {
		return nil, fmt.Errorf("unable to store RekeyRequest: %v", err)
	}
	return msg, nil
}

func (c commsContext) validateMessageFromClient(id common.ClientID, m *fspb.Message, validationInfo *fspb.ValidationInfo) error {
	if m.Destination == nil {
		return fmt.Errorf("message must have Destination")
	}
	if m.Destination.ClientId != nil {
		return fmt.Errorf("cannot send a message directly to another client [%v]", m.Destination.ClientId)
	}
	if m.Source == nil || m.Source.ServiceName == "" {
		return fmt.Errorf("message must have a source with a ServiceName, got: %v", m.Source)
	}
	if m.SourceMessageId == nil {
		return fmt.Errorf("source message id cannot be empty")
	}

	m.Source.ClientId = id.Bytes()
	m.ValidationInfo = validationInfo
	m.MessageId = common.MakeMessageID(m.Source, m.SourceMessageId).Bytes()
	return nil
}

// handleMessagesFromClient processes a block of messages from a particular
// client. It saves them to the database, associates them with the contact
// identified by contactTime, and processes them.
func (c commsContext) handleMessagesFromClient(ctx context.Context, info *comms.ClientInfo, contactID db.ContactID, received *fspb.ContactData, validationInfo *fspb.ValidationInfo) error {
	msgs := make([]*fspb.Message, 0, len(received.Messages))
	for _, m := range received.Messages {
		err := c.validateMessageFromClient(info.ID, m, validationInfo)
		if err != nil {
			log.Errorf("Dropping invalid message from [%v]: %v", info.ID, err)
			continue
		}
		msgs = append(msgs, m)
	}
	if len(msgs) == 0 {
		return nil
	}

	sort.Slice(msgs, func(a, b int) bool {
		return bytes.Compare(msgs[a].MessageId, msgs[b].MessageId) == -1
	})

	for {
		if len(msgs) <= processingChunkSize {
			return c.s.serviceConfig.HandleNewMessages(ctx, msgs, contactID)
		}

		if err := c.s.serviceConfig.HandleNewMessages(ctx, msgs[:processingChunkSize], contactID); err != nil {
			return err
		}
		msgs = msgs[processingChunkSize:]
	}
}

// ReadFile returns the data and modification time of file. Caller is
// responsible for closing data.
//
// Calls to data are permitted to fail if ctx is canceled or expired.
func (c commsContext) ReadFile(ctx context.Context, service, name string) (data db.ReadSeekerCloser, modtime time.Time, err error) {
	return c.s.dataStore.ReadFile(ctx, service, name)
}

// IsNotFound returns whether an error returned by ReadFile indicates that the
// file was not found.
func (c commsContext) IsNotFound(err error) bool {
	return c.s.dataStore.IsNotFound(err)
}

// StatsCollector returns the stats.Collector used by the Fleetspeak
// system. Access is provided to allow collection of stats relating to the
// client communication.
func (c commsContext) StatsCollector() stats.Collector {
	return c.s.statsCollector
}

func (c commsContext) Authorizer() authorizer.Authorizer {
	return c.s.authorizer
}
<|endoftext|>


<commit_before>package palette

import (
	"log"
	"net/http"
	"testing"
	"time"
)

func init() {
	go func() {
		log.Fatal(http.ListenAndServe(":9595", http.FileServer(http.Dir("test"))))
	}()
	time.Sleep(time.Millisecond * 20)
}

func TestNew(t *testing.T) {
	p, err := New("http://localhost:9595/mixed.html", &SumScore{})
	if err != nil {
		t.Error(err)
	}
	var exp = []struct {
		hex   string
		score int
	}{
		{"#000102", 2},
		{"#ff0000", 1},
		{"#c0c0c0", 1},
	}
	for i, c := range p {
		if c.Score != exp[i].score {
			t.Errorf("Expecting score %d for color %s, got %d", exp[i].score, c.Color.Hex(), c.Score)
		}
		if c.Color.Hex() != exp[i].hex {
			t.Errorf("Expecting color %s, got %s", exp[i].hex, c.Color.Hex())
		}
	}
}
<commit_msg>Improve palette tests<commit_after>package palette

import (
	"github.com/lucasb-eyer/go-colorful"
	"github.com/nochso/colourl/css"
	"log"
	"net/http"
	"testing"
	"time"
)

func init() {
	go func() {
		log.Fatal(http.ListenAndServe(":9595", http.FileServer(http.Dir("test"))))
	}()
	time.Sleep(time.Millisecond * 20)
}

func TestNew(t *testing.T) {
	p, err := New("http://localhost:9595/mixed.html", &SumScore{})
	if err != nil {
		t.Error(err)
	}
	var exp = []struct {
		hex   string
		score int
	}{
		{"#000102", 2},
		{"#ff0000", 1},
		{"#c0c0c0", 1},
	}
	for i, c := range p {
		if c.Score != exp[i].score {
			t.Errorf("Expecting score %d for color %s, got %d", exp[i].score, c.Color.Hex(), c.Score)
		}
		if c.Color.Hex() != exp[i].hex {
			t.Errorf("Expecting color %s, got %s", exp[i].hex, c.Color.Hex())
		}
	}
}

func TestNew_ErrorGET(t *testing.T) {
	_, err := New("invalid url", nil)
	if err == nil {
		t.Error("Expecting error because of invalid URL")
	}
}

func TestGroup_ScorerDefaultsToSumScorer(t *testing.T) {
	c := colorful.FastHappyColor()
	cml := &css.CML{
		Mentions: []*css.ColorMention{&css.ColorMention{Color: &c}},
	}
	p := Group(cml, nil)
	if p[0].Score != 1 {
		t.Fatalf("Expecting score of 1, got %d", p[0].Score)
	}
}
<|endoftext|>