<commit_before>package update

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"runtime"

	"github.com/jobtalk/pnzr/vars"
)

const (
	GITHUB_API = "https://api.github.com/repos/jobtalk/pnzr/tags"
)

var client = &http.Client{}

type commit struct {
	Sha string `json:"sha"`
	URL string `json:"url"`
}

type tag struct {
	Name       string `json:"name"`
	ZipballURL string `json:"zipball_url"`
	TarballURL string `json:"tarball_url"`
	Commit     commit `json:"commit"`
}

func (t tag) String() string {
	bin, err := json.MarshalIndent(t, "", "    ")
	if err != nil {
		return ""
	}
	return string(bin)
}

func checkENV() bool {
	if runtime.GOOS != "darwin" && runtime.GOOS != "linux" {
		return false
	}
	if runtime.GOARCH != "amd64" {
		return false
	}
	return true
}

type Update struct{}

func (c *Update) Run(args []string) int {
	var platform string
	tags := []tag{}
	resp, err := client.Get(GITHUB_API)
	if err != nil {
		log.Println(err)
		return 255
	}
	bin, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Println(err)
		return 255
	}
	defer resp.Body.Close()
	if err := json.Unmarshal(bin, &tags); err != nil {
		log.Println(err)
		return 255
	}

	if !checkENV() {
		fmt.Printf("Sorry, this architecture not supported (%s %s).\n", runtime.GOOS, runtime.GOARCH)
		fmt.Println("Please try manual update.")
		return 255
	}

	if runtime.GOOS == "darwin" {
		platform = "darwin-amd64"
	} else if runtime.GOOS == "linux" {
		platform = "linux-amd64"
	}
	latest := tags[0].Name
	if vars.VERSION == latest {
		fmt.Println("this version is latest")
		return 0
	}
	if latest == "" {
		fmt.Println("can not get latest version")
		return 255
	}
	binaryURL := fmt.Sprintf("https://github.com/jobtalk/pnzr/releases/download/%s/pnzr-%s", latest, platform)

	dir, err := os.Executable()
	if err != nil {
		log.Println(err)
		return 255
	}

	latestResp, err := client.Get(binaryURL)
	if err != nil {
		log.Println(err)
		return 255
	}
	latestBin, err := ioutil.ReadAll(latestResp.Body)
	if err != nil {
		log.Println(err)
		return 255
	}

	if err := ioutil.WriteFile(dir, latestBin, 0755); err != nil {
		fmt.Printf("Check parmission: %s\n", dir)
		return 255
	}

	fmt.Println("update successed")

	return 0
}

func (c *Update) Synopsis() string {
	msg := "Upgrade apex to the latest stable release\n"
	msg += "options:\n"
	msg += "    <node>\n"
	msg += "===================================================\n"
	return msg
}

func (c *Update) Help() string {
	return c.Synopsis()
}
<commit_msg>helpの文言を修正<commit_after>package update

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"runtime"

	"github.com/jobtalk/pnzr/vars"
)

const (
	GITHUB_API = "https://api.github.com/repos/jobtalk/pnzr/tags"
)

var client = &http.Client{}

type commit struct {
	Sha string `json:"sha"`
	URL string `json:"url"`
}

type tag struct {
	Name       string `json:"name"`
	ZipballURL string `json:"zipball_url"`
	TarballURL string `json:"tarball_url"`
	Commit     commit `json:"commit"`
}

func (t tag) String() string {
	bin, err := json.MarshalIndent(t, "", "    ")
	if err != nil {
		return ""
	}
	return string(bin)
}

func checkENV() bool {
	if runtime.GOOS != "darwin" && runtime.GOOS != "linux" {
		return false
	}
	if runtime.GOARCH != "amd64" {
		return false
	}
	return true
}

type Update struct{}

func (c *Update) Run(args []string) int {
	var platform string
	tags := []tag{}
	resp, err := client.Get(GITHUB_API)
	if err != nil {
		log.Println(err)
		return 255
	}
	bin, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Println(err)
		return 255
	}
	defer resp.Body.Close()
	if err := json.Unmarshal(bin, &tags); err != nil {
		log.Println(err)
		return 255
	}

	if !checkENV() {
		fmt.Printf("Sorry, this architecture not supported (%s %s).\n", runtime.GOOS, runtime.GOARCH)
		fmt.Println("Please try manual update.")
		return 255
	}

	if runtime.GOOS == "darwin" {
		platform = "darwin-amd64"
	} else if runtime.GOOS == "linux" {
		platform = "linux-amd64"
	}
	latest := tags[0].Name
	if vars.VERSION == latest {
		fmt.Println("this version is latest")
		return 0
	}
	if latest == "" {
		fmt.Println("can not get latest version")
		return 255
	}
	binaryURL := fmt.Sprintf("https://github.com/jobtalk/pnzr/releases/download/%s/pnzr-%s", latest, platform)

	dir, err := os.Executable()
	if err != nil {
		log.Println(err)
		return 255
	}

	latestResp, err := client.Get(binaryURL)
	if err != nil {
		log.Println(err)
		return 255
	}
	latestBin, err := ioutil.ReadAll(latestResp.Body)
	if err != nil {
		log.Println(err)
		return 255
	}

	if err := ioutil.WriteFile(dir, latestBin, 0755); err != nil {
		fmt.Printf("Check parmission: %s\n", dir)
		return 255
	}

	fmt.Println("update successed")

	return 0
}

func (c *Update) Synopsis() string {
	msg := "Upgrade pnzr to the latest stable release\n"
	msg += "options:\n"
	msg += "    <node>\n"
	msg += "===================================================\n"
	return msg
}

func (c *Update) Help() string {
	return c.Synopsis()
}
<|endoftext|>


<commit_before>package cfg

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"time"

	"github.com/voxelbrain/goptions"
	"gopkg.in/ini.v0"
)

const VERSION = "0.2"

type Options struct {
	Env     string        `goptions:"-e, --env, description='<env> Application environment, defines config'"`
	Rebuild bool          `goptions:"-r, --rebuild, description='Rebuild index only, do not run web server'"`
	RootDir string        `goptions:"-d, --root, description='<dir> Force set root dir'"`
	Update  string        `goptions:"-u, --update, description='<uuid> Update post UUID field with new value (works with --field and --value set), do not run web server'"`
	Field   string        `goptions:"-f, --field, description='<field> Update post UUID field with new value (works with --update and --value set), do not run web server'"`
	Value   string        `goptions:"-v, --value, description='<value> Update post UUID field with new value (works with --update and --field set), do not run web server'"`
	Help    goptions.Help `goptions:"-h, --help, description='Show this help'"`
}

var config *ini.File
var options Options
var stdLogger *log.Logger
var instanceId string
var rootDir string
var runWebServer bool

func configKey(key string) *ini.Key {
	return config.Section("").Key(key)
}

func String(key string) string {
	return configKey(key).String()
}

func Int(key string) int {
	if val, err := configKey(key).Int(); err != nil {
		panic(err)
	} else {
		return val
	}
}

func Bool(key string) bool {
	if val, err := configKey(key).Bool(); err != nil {
		panic(err)
	} else {
		return val
	}
}

func Log(msg string) {
	stdLogger.Printf("[LOG] %v | %s\n", time.Now().Format("2006/01/02 - 15:04:05"), msg)
}

func GetOptions() Options {
	return options
}

func GetInstanceId() string {
	return instanceId
}

func GetVersion() string {
	return VERSION
}

func GetRootDir() string {
	return rootDir
}

func GetRootUrl(r *http.Request) *url.URL {
	scheme := String("ui.root_url.scheme")
	host := String("ui.root_url.host")
	path := String("ui.root_url.path")
	if len(host) < 1 {
		host = r.Host
	}
	return &url.URL{Scheme: scheme, Host: host, Path: path}
}

func GetRunWebServer() bool {
	return runWebServer
}

func init() {
	options = Options{
		Env:     "prod",
		Rebuild: false,
	}
	goptions.ParseAndFail(&options)

	runWebServer = !options.Rebuild && len(options.Update) < 1

	stdLogger = log.New(os.Stdout, "", 0)
	Log(fmt.Sprintf("Initializing application ver %s", VERSION))

	if len(options.RootDir) > 0 {
		rootDir = options.RootDir
	} else {
		var err error
		if rootDir, err = filepath.Abs(fmt.Sprintf("%s/../", filepath.Dir(os.Args[0]))); err != nil {
			panic(err)
		}
		options.RootDir = rootDir
	}

	var err error
	configPath := fmt.Sprintf("%s/config.ini", GetRootDir())
	Log(fmt.Sprintf("Loading base config from %s", configPath))
	if config, err = ini.Load(configPath); err != nil {
		panic(err)
	}

	configPath = fmt.Sprintf("%s/%s.ini", GetRootDir(), options.Env)
	Log(fmt.Sprintf("Loading env config from %s", configPath))
	if _, err := os.Stat(configPath); os.IsNotExist(err) {
		Log("Env config not found")
	} else {
		if err := config.Append(configPath); err != nil {
			panic(err)
		}
	}

	hasher := md5.New()
	hasher.Write([]byte(time.Now().Format("2006/01/02 - 15:04:05")))
	instanceId = hex.EncodeToString(hasher.Sum(nil))[:5]
	Log(fmt.Sprintf("Initialized application instance ID %s", instanceId))
}
<commit_msg>get version from accessor rather than direct access to variable<commit_after>package cfg

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"time"

	"github.com/voxelbrain/goptions"
	"gopkg.in/ini.v0"
)

const VERSION = "0.2"

type Options struct {
	Env     string        `goptions:"-e, --env, description='<env> Application environment, defines config'"`
	Rebuild bool          `goptions:"-r, --rebuild, description='Rebuild index only, do not run web server'"`
	RootDir string        `goptions:"-d, --root, description='<dir> Force set root dir'"`
	Update  string        `goptions:"-u, --update, description='<uuid> Update post UUID field with new value (works with --field and --value set), do not run web server'"`
	Field   string        `goptions:"-f, --field, description='<field> Update post UUID field with new value (works with --update and --value set), do not run web server'"`
	Value   string        `goptions:"-v, --value, description='<value> Update post UUID field with new value (works with --update and --field set), do not run web server'"`
	Help    goptions.Help `goptions:"-h, --help, description='Show this help'"`
}

var config *ini.File
var options Options
var stdLogger *log.Logger
var instanceId string
var rootDir string
var runWebServer bool

func configKey(key string) *ini.Key {
	return config.Section("").Key(key)
}

func String(key string) string {
	return configKey(key).String()
}

func Int(key string) int {
	if val, err := configKey(key).Int(); err != nil {
		panic(err)
	} else {
		return val
	}
}

func Bool(key string) bool {
	if val, err := configKey(key).Bool(); err != nil {
		panic(err)
	} else {
		return val
	}
}

func Log(msg string) {
	stdLogger.Printf("[LOG] %v | %s\n", time.Now().Format("2006/01/02 - 15:04:05"), msg)
}

func GetOptions() Options {
	return options
}

func GetInstanceId() string {
	return instanceId
}

func GetVersion() string {
	return VERSION
}

func GetRootDir() string {
	return rootDir
}

func GetRootUrl(r *http.Request) *url.URL {
	scheme := String("ui.root_url.scheme")
	host := String("ui.root_url.host")
	path := String("ui.root_url.path")
	if len(host) < 1 {
		host = r.Host
	}
	return &url.URL{Scheme: scheme, Host: host, Path: path}
}

func GetRunWebServer() bool {
	return runWebServer
}

func init() {
	options = Options{
		Env:     "prod",
		Rebuild: false,
	}
	goptions.ParseAndFail(&options)

	runWebServer = !options.Rebuild && len(options.Update) < 1

	stdLogger = log.New(os.Stdout, "", 0)
	Log(fmt.Sprintf("Initializing application ver %s", GetVersion()))

	if len(options.RootDir) > 0 {
		rootDir = options.RootDir
	} else {
		var err error
		if rootDir, err = filepath.Abs(fmt.Sprintf("%s/../", filepath.Dir(os.Args[0]))); err != nil {
			panic(err)
		}
		options.RootDir = rootDir
	}

	var err error
	configPath := fmt.Sprintf("%s/config.ini", GetRootDir())
	Log(fmt.Sprintf("Loading base config from %s", configPath))
	if config, err = ini.Load(configPath); err != nil {
		panic(err)
	}

	configPath = fmt.Sprintf("%s/%s.ini", GetRootDir(), options.Env)
	Log(fmt.Sprintf("Loading env config from %s", configPath))
	if _, err := os.Stat(configPath); os.IsNotExist(err) {
		Log("Env config not found")
	} else {
		if err := config.Append(configPath); err != nil {
			panic(err)
		}
	}

	hasher := md5.New()
	hasher.Write([]byte(time.Now().Format("2006/01/02 - 15:04:05")))
	instanceId = hex.EncodeToString(hasher.Sum(nil))[:5]
	Log(fmt.Sprintf("Initialized application instance ID %s", instanceId))
}
<|endoftext|>


<commit_before>package misc

import (
    //"unsafe"
    "encoding/binary"
    "jvmgo/jvm/rtda"
)

func init() {
    _unsafe(allocateMemory, "allocateMemory",   "(J)J")
    _unsafe(freeMemory,     "freeMemory",       "(J)V")
    _unsafe(putLong,        "putLong",          "(JJ)V")
    _unsafe(getByte,        "getByte",          "(J)B")
}

// public native long allocateMemory(long l);
// (J)J
func allocateMemory(frame *rtda.Frame) {
    stack := frame.OperandStack()
    size := stack.PopLong()
    stack.PopRef() // this

    address := allocate(size)
    stack.PushLong(address)
}

// public native void freeMemory(long l);
// (J)V
func freeMemory(frame *rtda.Frame) {
    stack := frame.OperandStack()
    address := stack.PopLong()
    stack.PopRef() // this
    free(address)
}

// public native byte getByte(long l);
// (J)B
func getByte(frame *rtda.Frame) {
    stack := frame.OperandStack()
    address := stack.PopLong()
    stack.PopRef() // this

    mem := memoryAt(address)
    b := mem[0]
    stack.PushInt(int32(b))
}

// public native void putLong(long l, long l1);
// (JJ)V
func putLong(frame *rtda.Frame) {
    stack := frame.OperandStack()
    value := stack.PopLong()
    address := stack.PopLong()
    stack.PopRef() // this

    mem := memoryAt(address)
    binary.BigEndian.PutUint64(mem, uint64(value))
}
<commit_msg>code refactor<commit_after>package misc

import (
    //"unsafe"
    "encoding/binary"
    "jvmgo/jvm/rtda"
)

func init() {
    _unsafe(allocateMemory, "allocateMemory",   "(J)J")
    _unsafe(freeMemory,     "freeMemory",       "(J)V")
    _unsafe(putLong,        "putLong",          "(JJ)V")
    _unsafe(getByte,        "getByte",          "(J)B")
}

// public native long allocateMemory(long bytes);
// (J)J
func allocateMemory(frame *rtda.Frame) {
    stack := frame.OperandStack()
    bytes := stack.PopLong()
    stack.PopRef() // this

    address := allocate(bytes)
    stack.PushLong(address)
}

// public native void freeMemory(long address);
// (J)V
func freeMemory(frame *rtda.Frame) {
    stack := frame.OperandStack()
    address := stack.PopLong()
    stack.PopRef() // this
    free(address)
}

// public native void putByte(long offset, byte b);

// public native byte getByte(long offset);
// (J)B
func getByte(frame *rtda.Frame) {
    stack := frame.OperandStack()
    address := stack.PopLong()
    stack.PopRef() // this

    mem := memoryAt(address)
    b := mem[0]
    stack.PushInt(int32(b))
}

// public native void putLong(long offset, long x);
// (JJ)V
func putLong(frame *rtda.Frame) {
    stack := frame.OperandStack()
    value := stack.PopLong()
    address := stack.PopLong()
    stack.PopRef() // this

    mem := memoryAt(address)
    binary.BigEndian.PutUint64(mem, uint64(value))
}
<|endoftext|>


<commit_before>package main

const primeIndexSize = 1 << 24

type Octree struct {
	N        int
	leafSize int
	mask     int64
	p        [][]uint16
	v        []uint16
}

// N must be power of 2.
func NewOctree(N int) *Octree {
	indexSize := N
	if N > primeIndexSize {
		indexSize = primeIndexSize
	}
	return &Octree{
		N:        N,
		leafSize: N / indexSize,
		mask:     int64(indexSize - 1),
		p:        make([][]uint16, indexSize),
		v:        make([]uint16, indexSize),
	}
}

func step(p, base [3]int, l int, index int64) (nbase [3]int, nl int, nindex int64) {
	nbase, nl, nindex = base, l>>1, index
	for i := 0; i < 3; i++ {
		nindex <<= 1
		if p[i] >= nbase[i]+nl {
			nindex++
			nbase[i] += nl
		}
	}
	return
}

func (t *Octree) GetV(x, y, z int) uint16 {
	p := [3]int{x, y, z}
	base := [3]int{0, 0, 0}
	var index int64
	l := t.N
	count := uint(1)
	base, l, index = step(p, base, l, index)
	if t.p != nil {
		panic("extended part of octree is not implemented")
	}
	cur := t.v[int(index&int64(t.mask))]
	if cur == 0 {
		return 0
	}

	// We need to check the neighbour if it has the value of 0 or not.
	// If no, we need to go deeper.
	// This is because we reuse the cell after an expansion.
	another := index + (1 << (count * 3))
	if another > t.mask || t.v[int(another&t.mask)] == 0 {
		// This is the leaf
		return cur
	}

	// The node is not leaf
	panic("not implemented")
}
<commit_msg>Fix small bug: wrong check<commit_after>package main

const primeIndexSize = 1 << 24

type Octree struct {
	N        int
	leafSize int
	mask     int64
	p        [][]uint16
	v        []uint16
}

// N must be power of 2.
func NewOctree(N int) *Octree {
	indexSize := N
	if N > primeIndexSize {
		indexSize = primeIndexSize
	}
	return &Octree{
		N:        N,
		leafSize: N / indexSize,
		mask:     int64(indexSize - 1),
		p:        make([][]uint16, indexSize),
		v:        make([]uint16, indexSize),
	}
}

func step(p, base [3]int, l int, index int64) (nbase [3]int, nl int, nindex int64) {
	nbase, nl, nindex = base, l>>1, index
	for i := 0; i < 3; i++ {
		nindex <<= 1
		if p[i] >= nbase[i]+nl {
			nindex++
			nbase[i] += nl
		}
	}
	return
}

func (t *Octree) GetV(x, y, z int) uint16 {
	p := [3]int{x, y, z}
	base := [3]int{0, 0, 0}
	var index int64
	l := t.N
	count := uint(1)
	base, l, index = step(p, base, l, index)
	if t.p[int(index&int64(t.mask))] != nil {
		panic("extended part of octree is not implemented")
	}
	cur := t.v[int(index&int64(t.mask))]
	if cur == 0 {
		return 0
	}

	// We need to check the neighbour if it has the value of 0 or not.
	// If no, we need to go deeper.
	// This is because we reuse the cell after an expansion.
	another := index + (1 << (count * 3))
	if another > t.mask || t.v[int(another&t.mask)] == 0 {
		// This is the leaf
		return cur
	}

	// The node is not leaf
	panic("not implemented")
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build linux

package main

import (
	"fmt"
	"os"
)

/*
#include <signal.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>

int *sigfwdP;
static void sigsegv() {
	*sigfwdP = 1;
	fprintf(stderr, "ERROR: C SIGSEGV not thrown on caught?.\n");
	exit(2);
}

static void segvhandler(int signum) {
	if (signum == SIGSEGV) {
		fprintf(stdout, "OK\n");
		exit(0);  // success
	}
}

static void __attribute__ ((constructor)) sigsetup(void) {
	if (getenv("GO_TEST_CGOSIGFWD") == NULL) {
		return;
	}

	struct sigaction act;

	memset(&act, 0, sizeof act);
	act.sa_handler = segvhandler;
	sigaction(SIGSEGV, &act, NULL);
}
*/
import "C"

func init() {
	register("CgoSigfwd", CgoSigfwd)
}

var nilPtr *byte

func f() (ret bool) {
	defer func() {
		if recover() == nil {
			fmt.Errorf("ERROR: couldn't raise SIGSEGV in Go.")
			C.exit(2)
		}
		ret = true
	}()
	*nilPtr = 1
	return false
}

func CgoSigfwd() {
	if os.Getenv("GO_TEST_CGOSIGFWD") == "" {
		fmt.Fprintf(os.Stderr, "test must be run with GO_TEST_CGOSIGFWD set\n")
		os.Exit(1)
	}

	// Test that the signal originating in Go is handled (and recovered) by Go.
	if !f() {
		fmt.Fprintf(os.Stderr, "couldn't recover from SIGSEGV in Go.\n")
		C.exit(2)
	}

	// Test that the signal originating in C is handled by C.
	C.sigsegv()
}
<commit_msg>runtime: improve coverage of TestCgoSigfwd<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build linux

package main

import (
	"fmt"
	"os"
)

/*
#include <signal.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>

sig_atomic_t expectCSigsegv;
int *sigfwdP;

static void sigsegv() {
	expectCSigsegv = 1;
	*sigfwdP = 1;
	fprintf(stderr, "ERROR: C SIGSEGV not thrown on caught?.\n");
	exit(2);
}

static void segvhandler(int signum) {
	if (signum == SIGSEGV) {
		if (expectCSigsegv == 0) {
			fprintf(stderr, "SIGSEGV caught in C unexpectedly\n");
			exit(1);
		}
		fprintf(stdout, "OK\n");
		exit(0);  // success
	}
}

static void __attribute__ ((constructor)) sigsetup(void) {
	if (getenv("GO_TEST_CGOSIGFWD") == NULL) {
		return;
	}

	struct sigaction act;

	memset(&act, 0, sizeof act);
	act.sa_handler = segvhandler;
	sigaction(SIGSEGV, &act, NULL);
}
*/
import "C"

func init() {
	register("CgoSigfwd", CgoSigfwd)
}

var nilPtr *byte

func f() (ret bool) {
	defer func() {
		if recover() == nil {
			fmt.Errorf("ERROR: couldn't raise SIGSEGV in Go.")
			C.exit(2)
		}
		ret = true
	}()
	*nilPtr = 1
	return false
}

func CgoSigfwd() {
	if os.Getenv("GO_TEST_CGOSIGFWD") == "" {
		fmt.Fprintf(os.Stderr, "test must be run with GO_TEST_CGOSIGFWD set\n")
		os.Exit(1)
	}

	// Test that the signal originating in Go is handled (and recovered) by Go.
	if !f() {
		fmt.Fprintf(os.Stderr, "couldn't recover from SIGSEGV in Go.\n")
		C.exit(2)
	}

	// Test that the signal originating in C is handled by C.
	C.sigsegv()
}
<|endoftext|>


<commit_before>package terraform

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"reflect"
	"strings"

	"terraform-resource/models"
	"terraform-resource/storage"
)

type Client struct {
	Model         models.Terraform
	StorageDriver storage.Storage
	LogWriter     io.Writer
}

func (c Client) Apply() error {
	var sourcePath string
	if c.Model.PlanRun {
		sourcePath = c.Model.PlanFileLocalPath
	} else {
		var err error
		sourcePath, err = c.fetchSource()
		if err != nil {
			return err
		}
	}

	applyArgs := []string{
		"apply",
		"-backup='-'",  // no need to backup state file
		"-input=false", // do not prompt for inputs
	}
	if c.Model.PlanRun == false {
		applyArgs = append(applyArgs, c.varFlags()...)
		applyArgs = append(applyArgs, fmt.Sprintf("-state=%s", c.Model.StateFileLocalPath))
	} else {
		applyArgs = append(applyArgs, fmt.Sprintf("-state-out=%s", c.Model.StateFileLocalPath))
	}

	applyArgs = append(applyArgs, sourcePath)

	applyCmd := c.terraformCmd(applyArgs)
	applyCmd.Stdout = c.LogWriter
	applyCmd.Stderr = c.LogWriter
	err := applyCmd.Run()
	if err != nil {
		return fmt.Errorf("Failed to run Terraform command: %s", err)
	}

	return nil
}

func (c Client) Destroy() error {
	sourcePath, err := c.fetchSource()
	if err != nil {
		return err
	}

	destroyArgs := []string{
		"destroy",
		"-backup='-'", // no need to backup state file
		"-force",      // do not prompt for confirmation
		fmt.Sprintf("-state=%s", c.Model.StateFileLocalPath),
	}
	destroyArgs = append(destroyArgs, c.varFlags()...)
	destroyArgs = append(destroyArgs, sourcePath)

	destroyCmd := c.terraformCmd(destroyArgs)
	destroyCmd.Stdout = c.LogWriter
	destroyCmd.Stderr = c.LogWriter
	err = destroyCmd.Run()
	if err != nil {
		return fmt.Errorf("Failed to run Terraform command: %s", err)
	}

	return nil
}

func (c Client) Plan() error {
	sourcePath, err := c.fetchSource()
	if err != nil {
		return err
	}

	planArgs := []string{
		"plan",
		"-input=false", // do not prompt for inputs
		fmt.Sprintf("-out=%s", c.Model.PlanFileLocalPath),
		fmt.Sprintf("-state=%s", c.Model.StateFileLocalPath),
	}
	planArgs = append(planArgs, c.varFlags()...)
	planArgs = append(planArgs, sourcePath)

	planCmd := c.terraformCmd(planArgs)
	planCmd.Stdout = c.LogWriter
	planCmd.Stderr = c.LogWriter
	err = planCmd.Run()
	if err != nil {
		return fmt.Errorf("Failed to run Terraform command: %s", err)
	}

	return nil
}

func (c Client) Output() (map[string]map[string]interface{}, error) {
	outputArgs := []string{
		"output",
		"-json",
		fmt.Sprintf("-state=%s", c.Model.StateFileLocalPath),
	}

	if c.Model.OutputModule != "" {
		outputArgs = append(outputArgs, fmt.Sprintf("-module=%s", c.Model.OutputModule))
	}

	outputCmd := c.terraformCmd(outputArgs)

	rawOutput, err := outputCmd.CombinedOutput()
	if err != nil {
		// TF CLI currently doesn't provide a nice way to detect an empty set of outputs
		// https://github.com/hashicorp/terraform/issues/11696
		if strings.Contains(string(rawOutput), "no outputs defined") {
			rawOutput = []byte("{}")
		} else {
			return nil, fmt.Errorf("Failed to retrieve output.\nError: %s\nOutput: %s", err, rawOutput)
		}
	}

	tfOutput := map[string]map[string]interface{}{}
	if err = json.Unmarshal(rawOutput, &tfOutput); err != nil {
		return nil, fmt.Errorf("Failed to unmarshal JSON output.\nError: %s\nOutput: %s", err, rawOutput)
	}

	return tfOutput, nil
}

func (c Client) Version() (string, error) {
	outputCmd := c.terraformCmd([]string{
		"-v",
	})
	output, err := outputCmd.CombinedOutput()
	if err != nil {
		return "", fmt.Errorf("Failed to retrieve version.\nError: %s\nOutput: %s", err, output)
	}

	return strings.TrimSpace(string(output)), nil
}

func (c Client) Import() error {
	if len(c.Model.Imports) == 0 {
		return nil
	}

	sourcePath, err := c.fetchSource()
	if err != nil {
		return err
	}

	for importType, importName := range c.Model.Imports {
		importArgs := []string{
			"import",
			fmt.Sprintf("-state=%s", c.Model.StateFileLocalPath),
			fmt.Sprintf("-config=%s", sourcePath),
		}
		importArgs = append(importArgs, c.varFlags()...)
		importArgs = append(importArgs, importType)
		importArgs = append(importArgs, importName)

		c.LogWriter.Write([]byte(fmt.Sprintf("Importing `%s: %s`...\n", importType, importName)))
		importCmd := c.terraformCmd(importArgs)
		rawOutput, err := importCmd.CombinedOutput()
		if err != nil {
			return fmt.Errorf("Failed to import resource %s %s.\nError: %s\nOutput: %s", importType, importName, err, rawOutput)
		}
	}

	return nil
}

func (c Client) terraformCmd(args []string) *exec.Cmd {
	cmd := exec.Command("/bin/sh", "-c", fmt.Sprintf("terraform %s", strings.Join(args, " ")))

	cmd.Env = []string{
		fmt.Sprintf("PATH=%s", os.Getenv("PATH")),
	}
	for key, value := range c.Model.Env {
		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", key, value))
	}

	return cmd
}

func (c Client) fetchSource() (string, error) {
	var sourceDir string
	if c.useLocalSource() {
		sourceDir = c.Model.Source
	} else {
		tmpDir, err := ioutil.TempDir(os.TempDir(), "terraform-resource-client")
		if err != nil {
			return "", fmt.Errorf("Failed to create temporary working dir at '%s'", os.TempDir())
		}
		initCmd := c.terraformCmd([]string{
			"init",
			c.Model.Source,
			tmpDir,
		})
		if initOutput, initErr := initCmd.CombinedOutput(); initErr != nil {
			return "", fmt.Errorf("terraform init command failed.\nError: %s\nOutput: %s", initErr, initOutput)
		}
		sourceDir = tmpDir
	}

	getCmd := c.terraformCmd([]string{
		"get",
		"-update",
		sourceDir,
	})
	if getOutput, getErr := getCmd.CombinedOutput(); getErr != nil {
		return "", fmt.Errorf("terraform get command failed.\nError: %s\nOutput: %s", getErr, getOutput)
	}

	return sourceDir, nil
}

func (c Client) useLocalSource() bool {
	if info, err := os.Stat(c.Model.Source); err == nil && info.IsDir() {
		return true
	}
	return false
}

func (c Client) varFlags() []string {
	args := []string{}
	for key, val := range c.Model.Vars {
		args = append(args, "-var", fmt.Sprintf("'%s=%s'", key, formatVar(val)))
	}
	return args
}

func formatVar(value interface{}) string {
	valType := reflect.TypeOf(value)
	if valType == nil {
		return "null"
	}

	switch valType.Kind() {
	case reflect.Slice:
		valSlice, _ := value.([]interface{})
		sliceVars := []string{}
		for _, v := range valSlice {
			sliceVars = append(sliceVars, formatVar(v))
		}
		return fmt.Sprintf("[%s]", strings.Join(sliceVars, ","))
	case reflect.Map:
		valMap, _ := value.(map[string]interface{})
		mapVars := []string{}
		for k, v := range valMap {
			mapVars = append(mapVars, fmt.Sprintf("%s=%s", k, formatVar(v)))
		}
		return fmt.Sprintf("{%s}", strings.Join(mapVars, ","))
	}
	return fmt.Sprintf("%#v", value)
}
<commit_msg>Make `terraform import` idempotent<commit_after>package terraform

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"reflect"
	"strings"

	"terraform-resource/models"
	"terraform-resource/storage"
)

type Client struct {
	Model         models.Terraform
	StorageDriver storage.Storage
	LogWriter     io.Writer
}

func (c Client) Apply() error {
	var sourcePath string
	if c.Model.PlanRun {
		sourcePath = c.Model.PlanFileLocalPath
	} else {
		var err error
		sourcePath, err = c.fetchSource()
		if err != nil {
			return err
		}
	}

	applyArgs := []string{
		"apply",
		"-backup='-'",  // no need to backup state file
		"-input=false", // do not prompt for inputs
	}
	if c.Model.PlanRun == false {
		applyArgs = append(applyArgs, c.varFlags()...)
		applyArgs = append(applyArgs, fmt.Sprintf("-state=%s", c.Model.StateFileLocalPath))
	} else {
		applyArgs = append(applyArgs, fmt.Sprintf("-state-out=%s", c.Model.StateFileLocalPath))
	}

	applyArgs = append(applyArgs, sourcePath)

	applyCmd := c.terraformCmd(applyArgs)
	applyCmd.Stdout = c.LogWriter
	applyCmd.Stderr = c.LogWriter
	err := applyCmd.Run()
	if err != nil {
		return fmt.Errorf("Failed to run Terraform command: %s", err)
	}

	return nil
}

func (c Client) Destroy() error {
	sourcePath, err := c.fetchSource()
	if err != nil {
		return err
	}

	destroyArgs := []string{
		"destroy",
		"-backup='-'", // no need to backup state file
		"-force",      // do not prompt for confirmation
		fmt.Sprintf("-state=%s", c.Model.StateFileLocalPath),
	}
	destroyArgs = append(destroyArgs, c.varFlags()...)
	destroyArgs = append(destroyArgs, sourcePath)

	destroyCmd := c.terraformCmd(destroyArgs)
	destroyCmd.Stdout = c.LogWriter
	destroyCmd.Stderr = c.LogWriter
	err = destroyCmd.Run()
	if err != nil {
		return fmt.Errorf("Failed to run Terraform command: %s", err)
	}

	return nil
}

func (c Client) Plan() error {
	sourcePath, err := c.fetchSource()
	if err != nil {
		return err
	}

	planArgs := []string{
		"plan",
		"-input=false", // do not prompt for inputs
		fmt.Sprintf("-out=%s", c.Model.PlanFileLocalPath),
		fmt.Sprintf("-state=%s", c.Model.StateFileLocalPath),
	}
	planArgs = append(planArgs, c.varFlags()...)
	planArgs = append(planArgs, sourcePath)

	planCmd := c.terraformCmd(planArgs)
	planCmd.Stdout = c.LogWriter
	planCmd.Stderr = c.LogWriter
	err = planCmd.Run()
	if err != nil {
		return fmt.Errorf("Failed to run Terraform command: %s", err)
	}

	return nil
}

func (c Client) Output() (map[string]map[string]interface{}, error) {
	outputArgs := []string{
		"output",
		"-json",
		fmt.Sprintf("-state=%s", c.Model.StateFileLocalPath),
	}

	if c.Model.OutputModule != "" {
		outputArgs = append(outputArgs, fmt.Sprintf("-module=%s", c.Model.OutputModule))
	}

	outputCmd := c.terraformCmd(outputArgs)

	rawOutput, err := outputCmd.CombinedOutput()
	if err != nil {
		// TF CLI currently doesn't provide a nice way to detect an empty set of outputs
		// https://github.com/hashicorp/terraform/issues/11696
		if strings.Contains(string(rawOutput), "no outputs defined") {
			rawOutput = []byte("{}")
		} else {
			return nil, fmt.Errorf("Failed to retrieve output.\nError: %s\nOutput: %s", err, rawOutput)
		}
	}

	tfOutput := map[string]map[string]interface{}{}
	if err = json.Unmarshal(rawOutput, &tfOutput); err != nil {
		return nil, fmt.Errorf("Failed to unmarshal JSON output.\nError: %s\nOutput: %s", err, rawOutput)
	}

	return tfOutput, nil
}

func (c Client) Version() (string, error) {
	outputCmd := c.terraformCmd([]string{
		"-v",
	})
	output, err := outputCmd.CombinedOutput()
	if err != nil {
		return "", fmt.Errorf("Failed to retrieve version.\nError: %s\nOutput: %s", err, output)
	}

	return strings.TrimSpace(string(output)), nil
}

func (c Client) Import() error {
	if len(c.Model.Imports) == 0 {
		return nil
	}

	sourcePath, err := c.fetchSource()
	if err != nil {
		return err
	}

	for tfID, iaasID := range c.Model.Imports {
		exists, err := c.resourceExists(tfID)
		if err != nil {
			return fmt.Errorf("Failed to check for existence of resource %s %s.\nError: %s", tfID, iaasID, err)
		}
		if exists {
			c.LogWriter.Write([]byte(fmt.Sprintf("Skipping import of `%s: %s` as it already exists in the statefile...\n", tfID, iaasID)))
			continue
		}

		c.LogWriter.Write([]byte(fmt.Sprintf("Importing `%s: %s`...\n", tfID, iaasID)))
		importArgs := []string{
			"import",
			fmt.Sprintf("-state=%s", c.Model.StateFileLocalPath),
			fmt.Sprintf("-config=%s", sourcePath),
		}
		importArgs = append(importArgs, c.varFlags()...)
		importArgs = append(importArgs, tfID)
		importArgs = append(importArgs, iaasID)

		importCmd := c.terraformCmd(importArgs)
		rawOutput, err := importCmd.CombinedOutput()
		if err != nil {
			return fmt.Errorf("Failed to import resource %s %s.\nError: %s\nOutput: %s", tfID, iaasID, err, rawOutput)
		}
	}

	return nil
}

func (c Client) resourceExists(tfID string) (bool, error) {
	if _, err := os.Stat(c.Model.StateFileLocalPath); os.IsNotExist(err) {
		return false, nil
	}

	cmd := c.terraformCmd([]string{
		"state",
		"list",
		fmt.Sprintf("-state=%s", c.Model.StateFileLocalPath),
		tfID,
	})
	rawOutput, err := cmd.CombinedOutput()
	if err != nil {
		return false, fmt.Errorf("Error: %s, Output: %s", err, rawOutput)
	}

	// command returns the ID of the resource if it exists
	return (len(strings.TrimSpace(string(rawOutput))) > 0), nil
}

func (c Client) terraformCmd(args []string) *exec.Cmd {
	cmd := exec.Command("/bin/sh", "-c", fmt.Sprintf("terraform %s", strings.Join(args, " ")))

	cmd.Env = []string{
		fmt.Sprintf("PATH=%s", os.Getenv("PATH")),
	}
	for key, value := range c.Model.Env {
		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", key, value))
	}

	return cmd
}

func (c Client) fetchSource() (string, error) {
	var sourceDir string
	if c.useLocalSource() {
		sourceDir = c.Model.Source
	} else {
		tmpDir, err := ioutil.TempDir(os.TempDir(), "terraform-resource-client")
		if err != nil {
			return "", fmt.Errorf("Failed to create temporary working dir at '%s'", os.TempDir())
		}
		initCmd := c.terraformCmd([]string{
			"init",
			c.Model.Source,
			tmpDir,
		})
		if initOutput, initErr := initCmd.CombinedOutput(); initErr != nil {
			return "", fmt.Errorf("terraform init command failed.\nError: %s\nOutput: %s", initErr, initOutput)
		}
		sourceDir = tmpDir
	}

	getCmd := c.terraformCmd([]string{
		"get",
		"-update",
		sourceDir,
	})
	if getOutput, getErr := getCmd.CombinedOutput(); getErr != nil {
		return "", fmt.Errorf("terraform get command failed.\nError: %s\nOutput: %s", getErr, getOutput)
	}

	return sourceDir, nil
}

func (c Client) useLocalSource() bool {
	if info, err := os.Stat(c.Model.Source); err == nil && info.IsDir() {
		return true
	}
	return false
}

func (c Client) varFlags() []string {
	args := []string{}
	for key, val := range c.Model.Vars {
		args = append(args, "-var", fmt.Sprintf("'%s=%s'", key, formatVar(val)))
	}
	return args
}

func formatVar(value interface{}) string {
	valType := reflect.TypeOf(value)
	if valType == nil {
		return "null"
	}

	switch valType.Kind() {
	case reflect.Slice:
		valSlice, _ := value.([]interface{})
		sliceVars := []string{}
		for _, v := range valSlice {
			sliceVars = append(sliceVars, formatVar(v))
		}
		return fmt.Sprintf("[%s]", strings.Join(sliceVars, ","))
	case reflect.Map:
		valMap, _ := value.(map[string]interface{})
		mapVars := []string{}
		for k, v := range valMap {
			mapVars = append(mapVars, fmt.Sprintf("%s=%s", k, formatVar(v)))
		}
		return fmt.Sprintf("{%s}", strings.Join(mapVars, ","))
	}
	return fmt.Sprintf("%#v", value)
}
<|endoftext|>


<commit_before>package harvesterd

import (
	"harvesterd/intf"
	"sync"
	"sync/atomic"
)

type RecordsChan chan intf.Record
type CloseChan chan bool

type WriterConfig struct {
	Output  []string
	Reader  []string
	Threads int
}

type Writer struct {
	outputs     OutputsFactory
	readers     []*Reader
	failed      int32
	created     int32
	transferred int32
	maxThreads  int32
	threads     int32
	mutex       sync.Mutex
	recordsChan RecordsChan
	closeChan   CloseChan
}

func NewWriter() *Writer {
	writer := new(Writer)

	return writer
}

func (self *Writer) SetReaders(readers []*Reader) {
	self.readers = readers
}

func (self *Writer) SetOutputsFactory(factory OutputsFactory) {
	self.outputs = factory
}

func (self *Writer) SetThreads(threads int) {
	self.maxThreads = int32(threads)
}

func (self *Writer) GetChannels() (RecordsChan, CloseChan) {
	return self.recordsChan, self.closeChan
}

func (self *Writer) IsAlive() bool {
	return atomic.LoadInt32(&self.threads) != 0
}

func (self *Writer) Setup() {
	self.createChannels()
	self.setupReaders()
}

func (self *Writer) createChannels() {
	self.recordsChan = make(RecordsChan, self.maxThreads)
	self.closeChan = make(CloseChan, 1)
}

func (self *Writer) setupReaders() {
	for _, reader := range self.readers {
		reader.SetChannels(self.recordsChan, self.closeChan)
		reader.GoRead()
	}
}

func (self *Writer) Boot() {
	self.goWaitForReadersClose()
	self.goWriteFromChannel()
}

func (self *Writer) goWriteFromChannel() {
	for i := int32(0); i < self.maxThreads; i++ {
		atomic.AddInt32(&self.threads, 1)
		go self.doWriteFromChannel()
	}
}

func (self *Writer) goWaitForReadersClose() {
	go func() {
		readersClosed := 0
		readersCount := len(self.readers)
		for _ = range self.closeChan {
			readersClosed++

			if readersClosed >= readersCount {
				close(self.recordsChan)
				break
			}
		}
	}()
}

func (self *Writer) doWriteFromChannel() {
	outputs := self.outputs()
	for record := range self.recordsChan {
		self.writeRecordFromChannel(outputs, record)
	}

	atomic.AddInt32(&self.threads, -1)
}

func (self *Writer) writeRecordFromChannel(outputs []intf.Output, record intf.Record) {
	var wait sync.WaitGroup

	for _, output := range outputs {
		wait.Add(1)
		go self.writeRecordIntoOutput(output, record, &wait)
	}

	wait.Wait()
}

func (self *Writer) writeRecordIntoOutput(output intf.Output, record intf.Record, wait *sync.WaitGroup) {
	if output.PutRecord(record) {
		self.created++
	} else {
		self.failed++
	}

	wait.Done()
}

func (self *Writer) GetCounters() (int32, int32, int32, int32) {
	return self.created, self.failed, self.transferred, self.threads
}

func (self *Writer) ResetCounters() {
	self.created = 0
	self.failed = 0
	self.transferred = 0
}

func (self *Writer) Teardown() {
	self.teardownReaders()
}

func (self *Writer) teardownReaders() {
	for _, reader := range self.readers {
		reader.Teardown()
	}
}
<commit_msg>fix: runtime/cgo: pthread_create failed: Resource temporarily unavailable<commit_after>package harvesterd

import (
	"harvesterd/intf"
	"sync"
	"sync/atomic"
)

type RecordsChan chan intf.Record
type CloseChan chan bool

type WriterConfig struct {
	Output  []string
	Reader  []string
	Threads int
}

type Writer struct {
	outputs     OutputsFactory
	readers     []*Reader
	failed      int32
	created     int32
	transferred int32
	maxThreads  int32
	threads     int32
	mutex       sync.Mutex
	recordsChan RecordsChan
	closeChan   CloseChan
}

func NewWriter() *Writer {
	writer := new(Writer)

	return writer
}

func (self *Writer) SetReaders(readers []*Reader) {
	self.readers = readers
}

func (self *Writer) SetOutputsFactory(factory OutputsFactory) {
	self.outputs = factory
}

func (self *Writer) SetThreads(threads int) {
	self.maxThreads = int32(threads)
}

func (self *Writer) GetChannels() (RecordsChan, CloseChan) {
	return self.recordsChan, self.closeChan
}

func (self *Writer) IsAlive() bool {
	return atomic.LoadInt32(&self.threads) != 0
}

func (self *Writer) Setup() {
	self.createChannels()
	self.setupReaders()
}

func (self *Writer) createChannels() {
	self.recordsChan = make(RecordsChan, self.maxThreads)
	self.closeChan = make(CloseChan, 1)
}

func (self *Writer) setupReaders() {
	for _, reader := range self.readers {
		reader.SetChannels(self.recordsChan, self.closeChan)
		reader.GoRead()
	}
}

func (self *Writer) Boot() {
	self.goWaitForReadersClose()
	self.goWriteFromChannel()
}

func (self *Writer) goWriteFromChannel() {
	for i := int32(0); i < self.maxThreads; i++ {
		atomic.AddInt32(&self.threads, 1)
		go self.doWriteFromChannel()
	}
}

func (self *Writer) goWaitForReadersClose() {
	go func() {
		readersClosed := 0
		readersCount := len(self.readers)
		for _ = range self.closeChan {
			readersClosed++

			if readersClosed >= readersCount {
				close(self.recordsChan)
				break
			}
		}
	}()
}

func (self *Writer) doWriteFromChannel() {
	outputs := self.outputs()
	for record := range self.recordsChan {
		self.writeRecordFromChannel(outputs, record)
	}

	atomic.AddInt32(&self.threads, -1)
}

func (self *Writer) writeRecordFromChannel(outputs []intf.Output, record intf.Record) {
	var wait sync.WaitGroup

	for _, output := range outputs {
		self.writeRecordIntoOutput(output, record, &wait)
	}
}

func (self *Writer) writeRecordIntoOutput(output intf.Output, record intf.Record, wait *sync.WaitGroup) {
	if output.PutRecord(record) {
		self.created++
	} else {
		self.failed++
	}

	wait.Done()
}

func (self *Writer) GetCounters() (int32, int32, int32, int32) {
	return self.created, self.failed, self.transferred, self.threads
}

func (self *Writer) ResetCounters() {
	self.created = 0
	self.failed = 0
	self.transferred = 0
}

func (self *Writer) Teardown() {
	self.teardownReaders()
}

func (self *Writer) teardownReaders() {
	for _, reader := range self.readers {
		reader.Teardown()
	}
}
<|endoftext|>


<commit_before>// +build appengine

package urlfetch

import (
	"io"
	"net/http"
	"net/url"
	"time"

	"appengine"
	"appengine/urlfetch"
	"github.com/blueoceans/go-common/errors"
	u "github.com/blueoceans/go-common/logutil"
)

const (
	reasonUrlfetchClosed               = "urlfetch: CLOSED"
	reasonUrlfetchFetchErr             = "urlfetch: FETCH_ERROR"
	reasonUrlfetchConnectionErr        = "urlfetch: CONNECTION_ERROR"
	reasonUrlfetchInternalTransientErr = "urlfetch: INTERNAL_TRANSIENT_ERROR"
	reasonUrlfetchDeadlineExceeded     = "urlfetch: DEADLINE_EXCEEDED"
	reasonDeadlineExceededTimeout      = "Deadline exceeded (timeout)"

	sec1 = 1 * time.Second
)

var (
	errNoWait = []string{
		reasonUrlfetchClosed,
		reasonUrlfetchFetchErr,
		reasonUrlfetchConnectionErr,
		reasonUrlfetchDeadlineExceeded,
		reasonDeadlineExceededTimeout,
	}
	errWait = []string{
		reasonUrlfetchInternalTransientErr,
	}
)

func ClientGet(
	c appengine.Context,
	url string,
) (
	*http.Response,
	error,
) {
	return clientDo(
		c,
		func() (
			*http.Response,
			error,
		) {
			return urlfetch.Client(c).Get(url)
		},
	)
}

func ClientPost(
	c appengine.Context,
	url string,
	mimeType string,
	body io.Reader,
) (
	*http.Response,
	error,
) {
	return clientDo(
		c,
		func() (
			*http.Response,
			error,
		) {
			return urlfetch.Client(c).Post(url, mimeType, body)
		},
	)
}

func ClientPostForm(
	c appengine.Context,
	url string,
	data url.Values,
) (
	*http.Response,
	error,
) {
	return clientDo(
		c,
		func() (
			*http.Response,
			error,
		) {
			return urlfetch.Client(c).PostForm(url, data)
		},
	)
}

func clientDo(
	c appengine.Context,
	do func() (*http.Response, error),
) (
	*http.Response,
	error,
) {
retry:
	response, err := do()
	switch {
	case err == nil:
		return response, nil
	case IsErrNoWait(err):
		u.Infof(c, "%q", err)
		goto retry
	case IsErrWait(err):
		u.Infof(c, "%q", err)
		time.Sleep(sec1)
		goto retry
	default:
		return nil, err
	}
}

// IsErrNoWait returns is whether it can re-try soon or not.
func IsErrNoWait(
	err error,
) bool {
	return errors.ContainsErrorMessage(err, errNoWait)
}

// IsErrWait returns is whether it must wait for next re-try or not.
func IsErrWait(
	err error,
) bool {
	return errors.ContainsErrorMessage(err, errWait)
}
<commit_msg>refactor<commit_after>// +build appengine

package urlfetch

import (
	"io"
	"net/http"
	"net/url"
	"time"

	"appengine"
	"appengine/urlfetch"
	"github.com/blueoceans/go-common/errors"
	u "github.com/blueoceans/go-common/logutil"
)

const (
	reasonDeadlineExceededTimeout = "Deadline exceeded (timeout)"

	reasonUrlfetchClosed               = "urlfetch: CLOSED"
	reasonUrlfetchConnectionErr        = "urlfetch: CONNECTION_ERROR"
	reasonUrlfetchDeadlineExceeded     = "urlfetch: DEADLINE_EXCEEDED"
	reasonUrlfetchFetchErr             = "urlfetch: FETCH_ERROR"
	reasonUrlfetchInternalTransientErr = "urlfetch: INTERNAL_TRANSIENT_ERROR"

	sec1 = 1 * time.Second
)

var (
	errNoWait = []string{
		reasonDeadlineExceededTimeout,
		reasonUrlfetchClosed,
		reasonUrlfetchConnectionErr,
		reasonUrlfetchDeadlineExceeded,
		reasonUrlfetchFetchErr,
	}
	errWait = []string{
		reasonUrlfetchInternalTransientErr,
	}
)

func ClientGet(
	c appengine.Context,
	url string,
) (
	*http.Response,
	error,
) {
	return clientDo(
		c,
		func() (
			*http.Response,
			error,
		) {
			return urlfetch.Client(c).Get(url)
		},
	)
}

func ClientPost(
	c appengine.Context,
	url string,
	mimeType string,
	body io.Reader,
) (
	*http.Response,
	error,
) {
	return clientDo(
		c,
		func() (
			*http.Response,
			error,
		) {
			return urlfetch.Client(c).Post(url, mimeType, body)
		},
	)
}

func ClientPostForm(
	c appengine.Context,
	url string,
	data url.Values,
) (
	*http.Response,
	error,
) {
	return clientDo(
		c,
		func() (
			*http.Response,
			error,
		) {
			return urlfetch.Client(c).PostForm(url, data)
		},
	)
}

func clientDo(
	c appengine.Context,
	do func() (*http.Response, error),
) (
	*http.Response,
	error,
) {
retry:
	response, err := do()
	switch {
	case err == nil:
		return response, nil
	case IsErrNoWait(err):
		u.Infof(c, "%q", err)
		goto retry
	case IsErrWait(err):
		u.Infof(c, "%q", err)
		time.Sleep(sec1)
		goto retry
	default:
		return nil, err
	}
}

// IsErrNoWait returns is whether it can re-try soon or not.
func IsErrNoWait(
	err error,
) bool {
	return errors.ContainsErrorMessage(err, errNoWait)
}

// IsErrWait returns is whether it must wait for next re-try or not.
func IsErrWait(
	err error,
) bool {
	return errors.ContainsErrorMessage(err, errWait)
}
<|endoftext|>


<commit_before>/*
 Copyright 2018 Padduck, LLC
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 	http://www.apache.org/licenses/LICENSE-2.0
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package database

import (
	"github.com/jinzhu/gorm"
	_ "github.com/jinzhu/gorm/dialects/mysql"
	"github.com/pufferpanel/apufferi/config"
)

var dbConn *gorm.DB

func Load() error {
	err := openConnection()
	if err != nil {
		return err
	}

	return nil
}

func openConnection() (error) {
	dialect := config.GetStringOrDefault("database.dialect", "mysql")
	connString := config.GetString("database.url")

	//attempt to open database connection to validate
	var err error
	dbConn, err = gorm.Open(dialect, connString)
	return err
}

func GetConnection() (*gorm.DB, error) {
	var err error
	if dbConn == nil {
		err = openConnection()
	}

	return dbConn, err
}

func Close() {
	dbConn.Close()
}<commit_msg>Have database be created<commit_after>/*
 Copyright 2018 Padduck, LLC
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 	http://www.apache.org/licenses/LICENSE-2.0
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package database

import (
	"github.com/jinzhu/gorm"
	_ "github.com/jinzhu/gorm/dialects/mysql"
	"github.com/pufferpanel/apufferi/config"
	"github.com/pufferpanel/pufferpanel/models"
)

var dbConn *gorm.DB

func Load() error {
	err := openConnection()
	if err != nil {
		return err
	}

	dbObjects := []interface{} {
		&models.Node{},
		&models.Server{},
		&models.User{},
	}

	for _, v := range dbObjects {
		dbConn.AutoMigrate(v)
	}

	return nil
}

func openConnection() (error) {
	dialect := config.GetStringOrDefault("database.dialect", "mysql")
	connString := config.GetString("database.url")

	//attempt to open database connection to validate
	var err error
	dbConn, err = gorm.Open(dialect, connString)
	return err
}

func GetConnection() (*gorm.DB, error) {
	var err error
	if dbConn == nil {
		err = openConnection()
	}

	return dbConn, err
}

func Close() {
	dbConn.Close()
}<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cipher_test

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"encoding/hex"
	"fmt"
	"io"
	"os"
)

func ExampleNewCBCDecrypter() {
	key := []byte("example key 1234")
	ciphertext, _ := hex.DecodeString("f363f3ccdcb12bb883abf484ba77d9cd7d32b5baecb3d4b1b3e0e4beffdb3ded")

	block, err := aes.NewCipher(key)
	if err != nil {
		panic(err)
	}

	// The IV needs to be unique, but not secure. Therefore it's common to
	// include it at the beginning of the ciphertext.
	if len(ciphertext) < aes.BlockSize {
		panic("ciphertext too short")
	}
	iv := ciphertext[:aes.BlockSize]
	ciphertext = ciphertext[aes.BlockSize:]

	// CBC mode always works in whole blocks.
	if len(ciphertext)%aes.BlockSize != 0 {
		panic("ciphertext is not a multiple of the block size")
	}

	mode := cipher.NewCBCDecrypter(block, iv)

	// CryptBlocks can work in-place if the two arguments are the same.
	mode.CryptBlocks(ciphertext, ciphertext)

	// If the original plaintext lengths are not a multiple of the block
	// size, padding would have to be added when encrypting, which would be
	// removed at this point. For an example, see
	// https://tools.ietf.org/html/rfc5246#section-6.2.3.2. However, it's
	// critical to note that ciphertexts must be authenticated (i.e. by
	// using crypto/hmac) before being decrypted in order to avoid creating
	// a padding oracle.

	fmt.Printf("%s\n", ciphertext)
	// Output: exampleplaintext
}

func ExampleNewCBCEncrypter() {
	key := []byte("example key 1234")
	plaintext := []byte("exampleplaintext")

	// CBC mode works on blocks so plaintexts may need to be padded to the
	// next whole block. For an example of such padding, see
	// https://tools.ietf.org/html/rfc5246#section-6.2.3.2. Here we'll
	// assume that the plaintext is already of the correct length.
	if len(plaintext)%aes.BlockSize != 0 {
		panic("plaintext is not a multiple of the block size")
	}

	block, err := aes.NewCipher(key)
	if err != nil {
		panic(err)
	}

	// The IV needs to be unique, but not secure. Therefore it's common to
	// include it at the beginning of the ciphertext.
	ciphertext := make([]byte, aes.BlockSize+len(plaintext))
	iv := ciphertext[:aes.BlockSize]
	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
		panic(err)
	}

	mode := cipher.NewCBCEncrypter(block, iv)
	mode.CryptBlocks(ciphertext[aes.BlockSize:], plaintext)

	// It's important to remember that ciphertexts must be authenticated
	// (i.e. by using crypto/hmac) as well as being encrypted in order to
	// be secure.

	fmt.Printf("%x\n", ciphertext)
}

func ExampleNewCFBDecrypter() {
	key := []byte("example key 1234")
	ciphertext, _ := hex.DecodeString("22277966616d9bc47177bd02603d08c9a67d5380d0fe8cf3b44438dff7b9")

	block, err := aes.NewCipher(key)
	if err != nil {
		panic(err)
	}

	// The IV needs to be unique, but not secure. Therefore it's common to
	// include it at the beginning of the ciphertext.
	if len(ciphertext) < aes.BlockSize {
		panic("ciphertext too short")
	}
	iv := ciphertext[:aes.BlockSize]
	ciphertext = ciphertext[aes.BlockSize:]

	stream := cipher.NewCFBDecrypter(block, iv)

	// XORKeyStream can work in-place if the two arguments are the same.
	stream.XORKeyStream(ciphertext, ciphertext)
	fmt.Printf("%s", ciphertext)
	// Output: some plaintext
}

func ExampleNewCFBEncrypter() {
	key := []byte("example key 1234")
	plaintext := []byte("some plaintext")

	block, err := aes.NewCipher(key)
	if err != nil {
		panic(err)
	}

	// The IV needs to be unique, but not secure. Therefore it's common to
	// include it at the beginning of the ciphertext.
	ciphertext := make([]byte, aes.BlockSize+len(plaintext))
	iv := ciphertext[:aes.BlockSize]
	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
		panic(err)
	}

	stream := cipher.NewCFBEncrypter(block, iv)
	stream.XORKeyStream(ciphertext[aes.BlockSize:], plaintext)

	// It's important to remember that ciphertexts must be authenticated
	// (i.e. by using crypto/hmac) as well as being encrypted in order to
	// be secure.
}

func ExampleNewCTR() {
	key := []byte("example key 1234")
	plaintext := []byte("some plaintext")

	block, err := aes.NewCipher(key)
	if err != nil {
		panic(err)
	}

	// The IV needs to be unique, but not secure. Therefore it's common to
	// include it at the beginning of the ciphertext.
	ciphertext := make([]byte, aes.BlockSize+len(plaintext))
	iv := ciphertext[:aes.BlockSize]
	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
		panic(err)
	}

	stream := cipher.NewCTR(block, iv)
	stream.XORKeyStream(ciphertext[aes.BlockSize:], plaintext)

	// It's important to remember that ciphertexts must be authenticated
	// (i.e. by using crypto/hmac) as well as being encrypted in order to
	// be secure.

	// CTR mode is the same for both encryption and decryption, so we can
	// also decrypt that ciphertext with NewCTR.

	plaintext2 := make([]byte, len(plaintext))
	stream = cipher.NewCTR(block, iv)
	stream.XORKeyStream(plaintext2, ciphertext[aes.BlockSize:])

	fmt.Printf("%s\n", plaintext2)
	// Output: some plaintext
}

func ExampleNewOFB() {
	key := []byte("example key 1234")
	plaintext := []byte("some plaintext")

	block, err := aes.NewCipher(key)
	if err != nil {
		panic(err)
	}

	// The IV needs to be unique, but not secure. Therefore it's common to
	// include it at the beginning of the ciphertext.
	ciphertext := make([]byte, aes.BlockSize+len(plaintext))
	iv := ciphertext[:aes.BlockSize]
	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
		panic(err)
	}

	stream := cipher.NewOFB(block, iv)
	stream.XORKeyStream(ciphertext[aes.BlockSize:], plaintext)

	// It's important to remember that ciphertexts must be authenticated
	// (i.e. by using crypto/hmac) as well as being encrypted in order to
	// be secure.

	// OFB mode is the same for both encryption and decryption, so we can
	// also decrypt that ciphertext with NewOFB.

	plaintext2 := make([]byte, len(plaintext))
	stream = cipher.NewOFB(block, iv)
	stream.XORKeyStream(plaintext2, ciphertext[aes.BlockSize:])

	fmt.Printf("%s\n", plaintext2)
	// Output: some plaintext
}

func ExampleStreamReader() {
	key := []byte("example key 1234")

	inFile, err := os.Open("encrypted-file")
	if err != nil {
		panic(err)
	}
	defer inFile.Close()

	block, err := aes.NewCipher(key)
	if err != nil {
		panic(err)
	}

	// If the key is unique for each ciphertext, then it's ok to use a zero
	// IV.
	var iv [aes.BlockSize]byte
	stream := cipher.NewOFB(block, iv[:])

	outFile, err := os.OpenFile("decrypted-file", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		panic(err)
	}
	defer outFile.Close()

	reader := &cipher.StreamReader{stream, inFile}
	// Copy the input file to the output file, decrypting as we go.
	if _, err := io.Copy(outFile, reader); err != nil {
		panic(err)
	}

	// Note that this example is simplistic in that it omits any
	// authentication of the encrypted data. It you were actually to use
	// StreamReader in this manner, an attacker could flip arbitrary bits in
	// the output.
}

func ExampleStreamWriter() {
	key := []byte("example key 1234")

	inFile, err := os.Open("plaintext-file")
	if err != nil {
		panic(err)
	}
	defer inFile.Close()

	block, err := aes.NewCipher(key)
	if err != nil {
		panic(err)
	}

	// If the key is unique for each ciphertext, then it's ok to use a zero
	// IV.
	var iv [aes.BlockSize]byte
	stream := cipher.NewOFB(block, iv[:])

	outFile, err := os.OpenFile("encrypted-file", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		panic(err)
	}
	defer outFile.Close()

	writer := &cipher.StreamWriter{stream, outFile, nil}
	// Copy the input file to the output file, encrypting as we go.
	if _, err := io.Copy(writer, inFile); err != nil {
		panic(err)
	}

	// Note that this example is simplistic in that it omits any
	// authentication of the encrypted data. It you were actually to use
	// StreamReader in this manner, an attacker could flip arbitrary bits in
	// the decrypted result.
}
<commit_msg>crypto/cipher: fix vet warning<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cipher_test

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"encoding/hex"
	"fmt"
	"io"
	"os"
)

func ExampleNewCBCDecrypter() {
	key := []byte("example key 1234")
	ciphertext, _ := hex.DecodeString("f363f3ccdcb12bb883abf484ba77d9cd7d32b5baecb3d4b1b3e0e4beffdb3ded")

	block, err := aes.NewCipher(key)
	if err != nil {
		panic(err)
	}

	// The IV needs to be unique, but not secure. Therefore it's common to
	// include it at the beginning of the ciphertext.
	if len(ciphertext) < aes.BlockSize {
		panic("ciphertext too short")
	}
	iv := ciphertext[:aes.BlockSize]
	ciphertext = ciphertext[aes.BlockSize:]

	// CBC mode always works in whole blocks.
	if len(ciphertext)%aes.BlockSize != 0 {
		panic("ciphertext is not a multiple of the block size")
	}

	mode := cipher.NewCBCDecrypter(block, iv)

	// CryptBlocks can work in-place if the two arguments are the same.
	mode.CryptBlocks(ciphertext, ciphertext)

	// If the original plaintext lengths are not a multiple of the block
	// size, padding would have to be added when encrypting, which would be
	// removed at this point. For an example, see
	// https://tools.ietf.org/html/rfc5246#section-6.2.3.2. However, it's
	// critical to note that ciphertexts must be authenticated (i.e. by
	// using crypto/hmac) before being decrypted in order to avoid creating
	// a padding oracle.

	fmt.Printf("%s\n", ciphertext)
	// Output: exampleplaintext
}

func ExampleNewCBCEncrypter() {
	key := []byte("example key 1234")
	plaintext := []byte("exampleplaintext")

	// CBC mode works on blocks so plaintexts may need to be padded to the
	// next whole block. For an example of such padding, see
	// https://tools.ietf.org/html/rfc5246#section-6.2.3.2. Here we'll
	// assume that the plaintext is already of the correct length.
	if len(plaintext)%aes.BlockSize != 0 {
		panic("plaintext is not a multiple of the block size")
	}

	block, err := aes.NewCipher(key)
	if err != nil {
		panic(err)
	}

	// The IV needs to be unique, but not secure. Therefore it's common to
	// include it at the beginning of the ciphertext.
	ciphertext := make([]byte, aes.BlockSize+len(plaintext))
	iv := ciphertext[:aes.BlockSize]
	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
		panic(err)
	}

	mode := cipher.NewCBCEncrypter(block, iv)
	mode.CryptBlocks(ciphertext[aes.BlockSize:], plaintext)

	// It's important to remember that ciphertexts must be authenticated
	// (i.e. by using crypto/hmac) as well as being encrypted in order to
	// be secure.

	fmt.Printf("%x\n", ciphertext)
}

func ExampleNewCFBDecrypter() {
	key := []byte("example key 1234")
	ciphertext, _ := hex.DecodeString("22277966616d9bc47177bd02603d08c9a67d5380d0fe8cf3b44438dff7b9")

	block, err := aes.NewCipher(key)
	if err != nil {
		panic(err)
	}

	// The IV needs to be unique, but not secure. Therefore it's common to
	// include it at the beginning of the ciphertext.
	if len(ciphertext) < aes.BlockSize {
		panic("ciphertext too short")
	}
	iv := ciphertext[:aes.BlockSize]
	ciphertext = ciphertext[aes.BlockSize:]

	stream := cipher.NewCFBDecrypter(block, iv)

	// XORKeyStream can work in-place if the two arguments are the same.
	stream.XORKeyStream(ciphertext, ciphertext)
	fmt.Printf("%s", ciphertext)
	// Output: some plaintext
}

func ExampleNewCFBEncrypter() {
	key := []byte("example key 1234")
	plaintext := []byte("some plaintext")

	block, err := aes.NewCipher(key)
	if err != nil {
		panic(err)
	}

	// The IV needs to be unique, but not secure. Therefore it's common to
	// include it at the beginning of the ciphertext.
	ciphertext := make([]byte, aes.BlockSize+len(plaintext))
	iv := ciphertext[:aes.BlockSize]
	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
		panic(err)
	}

	stream := cipher.NewCFBEncrypter(block, iv)
	stream.XORKeyStream(ciphertext[aes.BlockSize:], plaintext)

	// It's important to remember that ciphertexts must be authenticated
	// (i.e. by using crypto/hmac) as well as being encrypted in order to
	// be secure.
}

func ExampleNewCTR() {
	key := []byte("example key 1234")
	plaintext := []byte("some plaintext")

	block, err := aes.NewCipher(key)
	if err != nil {
		panic(err)
	}

	// The IV needs to be unique, but not secure. Therefore it's common to
	// include it at the beginning of the ciphertext.
	ciphertext := make([]byte, aes.BlockSize+len(plaintext))
	iv := ciphertext[:aes.BlockSize]
	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
		panic(err)
	}

	stream := cipher.NewCTR(block, iv)
	stream.XORKeyStream(ciphertext[aes.BlockSize:], plaintext)

	// It's important to remember that ciphertexts must be authenticated
	// (i.e. by using crypto/hmac) as well as being encrypted in order to
	// be secure.

	// CTR mode is the same for both encryption and decryption, so we can
	// also decrypt that ciphertext with NewCTR.

	plaintext2 := make([]byte, len(plaintext))
	stream = cipher.NewCTR(block, iv)
	stream.XORKeyStream(plaintext2, ciphertext[aes.BlockSize:])

	fmt.Printf("%s\n", plaintext2)
	// Output: some plaintext
}

func ExampleNewOFB() {
	key := []byte("example key 1234")
	plaintext := []byte("some plaintext")

	block, err := aes.NewCipher(key)
	if err != nil {
		panic(err)
	}

	// The IV needs to be unique, but not secure. Therefore it's common to
	// include it at the beginning of the ciphertext.
	ciphertext := make([]byte, aes.BlockSize+len(plaintext))
	iv := ciphertext[:aes.BlockSize]
	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
		panic(err)
	}

	stream := cipher.NewOFB(block, iv)
	stream.XORKeyStream(ciphertext[aes.BlockSize:], plaintext)

	// It's important to remember that ciphertexts must be authenticated
	// (i.e. by using crypto/hmac) as well as being encrypted in order to
	// be secure.

	// OFB mode is the same for both encryption and decryption, so we can
	// also decrypt that ciphertext with NewOFB.

	plaintext2 := make([]byte, len(plaintext))
	stream = cipher.NewOFB(block, iv)
	stream.XORKeyStream(plaintext2, ciphertext[aes.BlockSize:])

	fmt.Printf("%s\n", plaintext2)
	// Output: some plaintext
}

func ExampleStreamReader() {
	key := []byte("example key 1234")

	inFile, err := os.Open("encrypted-file")
	if err != nil {
		panic(err)
	}
	defer inFile.Close()

	block, err := aes.NewCipher(key)
	if err != nil {
		panic(err)
	}

	// If the key is unique for each ciphertext, then it's ok to use a zero
	// IV.
	var iv [aes.BlockSize]byte
	stream := cipher.NewOFB(block, iv[:])

	outFile, err := os.OpenFile("decrypted-file", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		panic(err)
	}
	defer outFile.Close()

	reader := &cipher.StreamReader{S: stream, R: inFile}
	// Copy the input file to the output file, decrypting as we go.
	if _, err := io.Copy(outFile, reader); err != nil {
		panic(err)
	}

	// Note that this example is simplistic in that it omits any
	// authentication of the encrypted data. It you were actually to use
	// StreamReader in this manner, an attacker could flip arbitrary bits in
	// the output.
}

func ExampleStreamWriter() {
	key := []byte("example key 1234")

	inFile, err := os.Open("plaintext-file")
	if err != nil {
		panic(err)
	}
	defer inFile.Close()

	block, err := aes.NewCipher(key)
	if err != nil {
		panic(err)
	}

	// If the key is unique for each ciphertext, then it's ok to use a zero
	// IV.
	var iv [aes.BlockSize]byte
	stream := cipher.NewOFB(block, iv[:])

	outFile, err := os.OpenFile("encrypted-file", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		panic(err)
	}
	defer outFile.Close()

	writer := &cipher.StreamWriter{S: stream, W: outFile}
	// Copy the input file to the output file, encrypting as we go.
	if _, err := io.Copy(writer, inFile); err != nil {
		panic(err)
	}

	// Note that this example is simplistic in that it omits any
	// authentication of the encrypted data. It you were actually to use
	// StreamReader in this manner, an attacker could flip arbitrary bits in
	// the decrypted result.
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package json

import (
	"bytes"
	"math"
	"math/rand"
	"reflect"
	"testing"
)

// Tests of simple examples.

type example struct {
	compact string
	indent  string
}

var examples = []example{
	{`1`, `1`},
	{`{}`, `{}`},
	{`[]`, `[]`},
	{`{"":2}`, "{\n\t\"\": 2\n}"},
	{`[3]`, "[\n\t3\n]"},
	{`[1,2,3]`, "[\n\t1,\n\t2,\n\t3\n]"},
	{`{"x":1}`, "{\n\t\"x\": 1\n}"},
	{ex1, ex1i},
}

var ex1 = `[true,false,null,"x",1,1.5,0,-5e+2]`

var ex1i = `[
	true,
	false,
	null,
	"x",
	1,
	1.5,
	0,
	-5e+2
]`

func TestCompact(t *testing.T) {
	var buf bytes.Buffer
	for _, tt := range examples {
		buf.Reset()
		if err := Compact(&buf, []byte(tt.compact)); err != nil {
			t.Errorf("Compact(%#q): %v", tt.compact, err)
		} else if s := buf.String(); s != tt.compact {
			t.Errorf("Compact(%#q) = %#q, want original", tt.compact, s)
		}

		buf.Reset()
		if err := Compact(&buf, []byte(tt.indent)); err != nil {
			t.Errorf("Compact(%#q): %v", tt.indent, err)
			continue
		} else if s := buf.String(); s != tt.compact {
			t.Errorf("Compact(%#q) = %#q, want %#q", tt.indent, s, tt.compact)
		}
	}
}

func TestIndent(t *testing.T) {
	var buf bytes.Buffer
	for _, tt := range examples {
		buf.Reset()
		if err := Indent(&buf, []byte(tt.indent), "", "\t"); err != nil {
			t.Errorf("Indent(%#q): %v", tt.indent, err)
		} else if s := buf.String(); s != tt.indent {
			t.Errorf("Indent(%#q) = %#q, want original", tt.indent, s)
		}

		buf.Reset()
		if err := Indent(&buf, []byte(tt.compact), "", "\t"); err != nil {
			t.Errorf("Indent(%#q): %v", tt.compact, err)
			continue
		} else if s := buf.String(); s != tt.indent {
			t.Errorf("Indent(%#q) = %#q, want %#q", tt.compact, s, tt.indent)
		}
	}
}

// Tests of a large random structure.

func TestCompactBig(t *testing.T) {
	initBig()
	var buf bytes.Buffer
	if err := Compact(&buf, jsonBig); err != nil {
		t.Fatalf("Compact: %v", err)
	}
	b := buf.Bytes()
	if bytes.Compare(b, jsonBig) != 0 {
		t.Error("Compact(jsonBig) != jsonBig")
		diff(t, b, jsonBig)
		return
	}
}

func TestIndentBig(t *testing.T) {
	initBig()
	var buf bytes.Buffer
	if err := Indent(&buf, jsonBig, "", "\t"); err != nil {
		t.Fatalf("Indent1: %v", err)
	}
	b := buf.Bytes()
	if len(b) == len(jsonBig) {
		// jsonBig is compact (no unnecessary spaces);
		// indenting should make it bigger
		t.Fatalf("Indent(jsonBig) did not get bigger")
	}

	// should be idempotent
	var buf1 bytes.Buffer
	if err := Indent(&buf1, b, "", "\t"); err != nil {
		t.Fatalf("Indent2: %v", err)
	}
	b1 := buf1.Bytes()
	if bytes.Compare(b1, b) != 0 {
		t.Error("Indent(Indent(jsonBig)) != Indent(jsonBig)")
		diff(t, b1, b)
		return
	}

	// should get back to original
	buf1.Reset()
	if err := Compact(&buf1, b); err != nil {
		t.Fatalf("Compact: %v", err)
	}
	b1 = buf1.Bytes()
	if bytes.Compare(b1, jsonBig) != 0 {
		t.Error("Compact(Indent(jsonBig)) != jsonBig")
		diff(t, b1, jsonBig)
		return
	}
}

type indentErrorTest struct {
	in  string
	err error
}

var indentErrorTests = []indentErrorTest{
	{`{"X": "foo", "Y"}`, &SyntaxError{"invalid character '}' after object key", 17}},
	{`{"X": "foo" "Y": "bar"}`, &SyntaxError{"invalid character '\"' after object key:value pair", 13}},
}

func TestIndentErrors(t *testing.T) {
	for i, tt := range indentErrorTests {
		slice := make([]uint8, 0)
		buf := bytes.NewBuffer(slice)
		if err := Indent(buf, []uint8(tt.in), "", ""); err != nil {
			if !reflect.DeepEqual(err, tt.err) {
				t.Errorf("#%d: Indent: %#v", i, err)
				continue
			}
		}
	}
}

func TestNextValueBig(t *testing.T) {
	initBig()
	var scan scanner
	item, rest, err := nextValue(jsonBig, &scan)
	if err != nil {
		t.Fatalf("nextValue: %s", err)
	}
	if len(item) != len(jsonBig) || &item[0] != &jsonBig[0] {
		t.Errorf("invalid item: %d %d", len(item), len(jsonBig))
	}
	if len(rest) != 0 {
		t.Errorf("invalid rest: %d", len(rest))
	}

	item, rest, err = nextValue(append(jsonBig, "HELLO WORLD"...), &scan)
	if err != nil {
		t.Fatalf("nextValue extra: %s", err)
	}
	if len(item) != len(jsonBig) {
		t.Errorf("invalid item: %d %d", len(item), len(jsonBig))
	}
	if string(rest) != "HELLO WORLD" {
		t.Errorf("invalid rest: %d", len(rest))
	}
}

func BenchmarkSkipValue(b *testing.B) {
	initBig()
	var scan scanner
	for i := 0; i < b.N; i++ {
		nextValue(jsonBig, &scan)
	}
	b.SetBytes(int64(len(jsonBig)))
}

func diff(t *testing.T, a, b []byte) {
	for i := 0; ; i++ {
		if i >= len(a) || i >= len(b) || a[i] != b[i] {
			j := i - 10
			if j < 0 {
				j = 0
			}
			t.Errorf("diverge at %d: «%s» vs «%s»", i, trim(a[j:]), trim(b[j:]))
			return
		}
	}
}

func trim(b []byte) []byte {
	if len(b) > 20 {
		return b[0:20]
	}
	return b
}

// Generate a random JSON object.

var jsonBig []byte

const (
	big   = 10000
	small = 100
)

func initBig() {
	n := big
	if testing.Short() {
		n = small
	}
	if len(jsonBig) != n {
		b, err := Marshal(genValue(n))
		if err != nil {
			panic(err)
		}
		jsonBig = b
	}
}

func genValue(n int) interface{} {
	if n > 1 {
		switch rand.Intn(2) {
		case 0:
			return genArray(n)
		case 1:
			return genMap(n)
		}
	}
	switch rand.Intn(3) {
	case 0:
		return rand.Intn(2) == 0
	case 1:
		return rand.NormFloat64()
	case 2:
		return genString(30)
	}
	panic("unreachable")
}

func genString(stddev float64) string {
	n := int(math.Abs(rand.NormFloat64()*stddev + stddev/2))
	c := make([]rune, n)
	for i := range c {
		f := math.Abs(rand.NormFloat64()*64 + 32)
		if f > 0x10ffff {
			f = 0x10ffff
		}
		c[i] = rune(f)
	}
	return string(c)
}

func genArray(n int) []interface{} {
	f := int(math.Abs(rand.NormFloat64()) * math.Min(10, float64(n/2)))
	if f > n {
		f = n
	}
	if n > 0 && f == 0 {
		f = 1
	}
	x := make([]interface{}, f)
	for i := range x {
		x[i] = genValue(((i+1)*n)/f - (i*n)/f)
	}
	return x
}

func genMap(n int) map[string]interface{} {
	f := int(math.Abs(rand.NormFloat64()) * math.Min(10, float64(n/2)))
	if f > n {
		f = n
	}
	if n > 0 && f == 0 {
		f = 1
	}
	x := make(map[string]interface{})
	for i := 0; i < f; i++ {
		x[genString(10)] = genValue(((i+1)*n)/f - (i*n)/f)
	}
	return x
}
<commit_msg>encoding/json: make BenchmarkSkipValue more consistent<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package json

import (
	"bytes"
	"math"
	"math/rand"
	"reflect"
	"testing"
)

// Tests of simple examples.

type example struct {
	compact string
	indent  string
}

var examples = []example{
	{`1`, `1`},
	{`{}`, `{}`},
	{`[]`, `[]`},
	{`{"":2}`, "{\n\t\"\": 2\n}"},
	{`[3]`, "[\n\t3\n]"},
	{`[1,2,3]`, "[\n\t1,\n\t2,\n\t3\n]"},
	{`{"x":1}`, "{\n\t\"x\": 1\n}"},
	{ex1, ex1i},
}

var ex1 = `[true,false,null,"x",1,1.5,0,-5e+2]`

var ex1i = `[
	true,
	false,
	null,
	"x",
	1,
	1.5,
	0,
	-5e+2
]`

func TestCompact(t *testing.T) {
	var buf bytes.Buffer
	for _, tt := range examples {
		buf.Reset()
		if err := Compact(&buf, []byte(tt.compact)); err != nil {
			t.Errorf("Compact(%#q): %v", tt.compact, err)
		} else if s := buf.String(); s != tt.compact {
			t.Errorf("Compact(%#q) = %#q, want original", tt.compact, s)
		}

		buf.Reset()
		if err := Compact(&buf, []byte(tt.indent)); err != nil {
			t.Errorf("Compact(%#q): %v", tt.indent, err)
			continue
		} else if s := buf.String(); s != tt.compact {
			t.Errorf("Compact(%#q) = %#q, want %#q", tt.indent, s, tt.compact)
		}
	}
}

func TestIndent(t *testing.T) {
	var buf bytes.Buffer
	for _, tt := range examples {
		buf.Reset()
		if err := Indent(&buf, []byte(tt.indent), "", "\t"); err != nil {
			t.Errorf("Indent(%#q): %v", tt.indent, err)
		} else if s := buf.String(); s != tt.indent {
			t.Errorf("Indent(%#q) = %#q, want original", tt.indent, s)
		}

		buf.Reset()
		if err := Indent(&buf, []byte(tt.compact), "", "\t"); err != nil {
			t.Errorf("Indent(%#q): %v", tt.compact, err)
			continue
		} else if s := buf.String(); s != tt.indent {
			t.Errorf("Indent(%#q) = %#q, want %#q", tt.compact, s, tt.indent)
		}
	}
}

// Tests of a large random structure.

func TestCompactBig(t *testing.T) {
	initBig()
	var buf bytes.Buffer
	if err := Compact(&buf, jsonBig); err != nil {
		t.Fatalf("Compact: %v", err)
	}
	b := buf.Bytes()
	if bytes.Compare(b, jsonBig) != 0 {
		t.Error("Compact(jsonBig) != jsonBig")
		diff(t, b, jsonBig)
		return
	}
}

func TestIndentBig(t *testing.T) {
	initBig()
	var buf bytes.Buffer
	if err := Indent(&buf, jsonBig, "", "\t"); err != nil {
		t.Fatalf("Indent1: %v", err)
	}
	b := buf.Bytes()
	if len(b) == len(jsonBig) {
		// jsonBig is compact (no unnecessary spaces);
		// indenting should make it bigger
		t.Fatalf("Indent(jsonBig) did not get bigger")
	}

	// should be idempotent
	var buf1 bytes.Buffer
	if err := Indent(&buf1, b, "", "\t"); err != nil {
		t.Fatalf("Indent2: %v", err)
	}
	b1 := buf1.Bytes()
	if bytes.Compare(b1, b) != 0 {
		t.Error("Indent(Indent(jsonBig)) != Indent(jsonBig)")
		diff(t, b1, b)
		return
	}

	// should get back to original
	buf1.Reset()
	if err := Compact(&buf1, b); err != nil {
		t.Fatalf("Compact: %v", err)
	}
	b1 = buf1.Bytes()
	if bytes.Compare(b1, jsonBig) != 0 {
		t.Error("Compact(Indent(jsonBig)) != jsonBig")
		diff(t, b1, jsonBig)
		return
	}
}

type indentErrorTest struct {
	in  string
	err error
}

var indentErrorTests = []indentErrorTest{
	{`{"X": "foo", "Y"}`, &SyntaxError{"invalid character '}' after object key", 17}},
	{`{"X": "foo" "Y": "bar"}`, &SyntaxError{"invalid character '\"' after object key:value pair", 13}},
}

func TestIndentErrors(t *testing.T) {
	for i, tt := range indentErrorTests {
		slice := make([]uint8, 0)
		buf := bytes.NewBuffer(slice)
		if err := Indent(buf, []uint8(tt.in), "", ""); err != nil {
			if !reflect.DeepEqual(err, tt.err) {
				t.Errorf("#%d: Indent: %#v", i, err)
				continue
			}
		}
	}
}

func TestNextValueBig(t *testing.T) {
	initBig()
	var scan scanner
	item, rest, err := nextValue(jsonBig, &scan)
	if err != nil {
		t.Fatalf("nextValue: %s", err)
	}
	if len(item) != len(jsonBig) || &item[0] != &jsonBig[0] {
		t.Errorf("invalid item: %d %d", len(item), len(jsonBig))
	}
	if len(rest) != 0 {
		t.Errorf("invalid rest: %d", len(rest))
	}

	item, rest, err = nextValue(append(jsonBig, "HELLO WORLD"...), &scan)
	if err != nil {
		t.Fatalf("nextValue extra: %s", err)
	}
	if len(item) != len(jsonBig) {
		t.Errorf("invalid item: %d %d", len(item), len(jsonBig))
	}
	if string(rest) != "HELLO WORLD" {
		t.Errorf("invalid rest: %d", len(rest))
	}
}

var benchScan scanner

func BenchmarkSkipValue(b *testing.B) {
	initBig()
	for i := 0; i < b.N; i++ {
		nextValue(jsonBig, &benchScan)
	}
	b.SetBytes(int64(len(jsonBig)))
}

func diff(t *testing.T, a, b []byte) {
	for i := 0; ; i++ {
		if i >= len(a) || i >= len(b) || a[i] != b[i] {
			j := i - 10
			if j < 0 {
				j = 0
			}
			t.Errorf("diverge at %d: «%s» vs «%s»", i, trim(a[j:]), trim(b[j:]))
			return
		}
	}
}

func trim(b []byte) []byte {
	if len(b) > 20 {
		return b[0:20]
	}
	return b
}

// Generate a random JSON object.

var jsonBig []byte

const (
	big   = 10000
	small = 100
)

func initBig() {
	n := big
	if testing.Short() {
		n = small
	}
	if len(jsonBig) != n {
		b, err := Marshal(genValue(n))
		if err != nil {
			panic(err)
		}
		jsonBig = b
	}
}

func genValue(n int) interface{} {
	if n > 1 {
		switch rand.Intn(2) {
		case 0:
			return genArray(n)
		case 1:
			return genMap(n)
		}
	}
	switch rand.Intn(3) {
	case 0:
		return rand.Intn(2) == 0
	case 1:
		return rand.NormFloat64()
	case 2:
		return genString(30)
	}
	panic("unreachable")
}

func genString(stddev float64) string {
	n := int(math.Abs(rand.NormFloat64()*stddev + stddev/2))
	c := make([]rune, n)
	for i := range c {
		f := math.Abs(rand.NormFloat64()*64 + 32)
		if f > 0x10ffff {
			f = 0x10ffff
		}
		c[i] = rune(f)
	}
	return string(c)
}

func genArray(n int) []interface{} {
	f := int(math.Abs(rand.NormFloat64()) * math.Min(10, float64(n/2)))
	if f > n {
		f = n
	}
	if n > 0 && f == 0 {
		f = 1
	}
	x := make([]interface{}, f)
	for i := range x {
		x[i] = genValue(((i+1)*n)/f - (i*n)/f)
	}
	return x
}

func genMap(n int) map[string]interface{} {
	f := int(math.Abs(rand.NormFloat64()) * math.Min(10, float64(n/2)))
	if f > n {
		f = n
	}
	if n > 0 && f == 0 {
		f = 1
	}
	x := make(map[string]interface{})
	for i := 0; i < f; i++ {
		x[genString(10)] = genValue(((i+1)*n)/f - (i*n)/f)
	}
	return x
}
<|endoftext|>


<commit_before>//  Copyright (c) 2012 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package db

import (
	"encoding/json"
	"log"
	"testing"
	"time"

	"github.com/couchbaselabs/go.assert"

	"github.com/couchbase/sync_gateway/base"
	"github.com/couchbase/sync_gateway/channels"
)

// Unit test for bug #314
func TestChangesAfterChannelAdded(t *testing.T) {
	db := setupTestDB(t)
	defer tearDownTestDB(t, db)
	_testChangesAfterChannelAdded(t, db)
}

func printChanges(changes []*ChangeEntry) {
	for _, change := range changes {
		log.Printf("Change:%+v", change)
	}
}

func getLastSeq(changes []*ChangeEntry) SequenceID {
	if len(changes) > 0 {
		return changes[len(changes)-1].Seq
	}
	return SequenceID{}
}

func getZeroSequence(db *Database) ChangesOptions {
	if db.SequenceType == IntSequenceType {
		return ChangesOptions{Since: SequenceID{Seq: 0}}
	} else {
		return ChangesOptions{Since: SequenceID{Clock: base.NewSequenceClockImpl()}}
	}
}

func _testChangesAfterChannelAdded(t *testing.T, db *Database) {
	base.EnableLogKey("IndexChanges")
	base.EnableLogKey("Hash+")
	db.ChannelMapper = channels.NewDefaultChannelMapper()

	// Create a user with access to channel ABC
	authenticator := db.Authenticator()
	user, _ := authenticator.NewUser("naomi", "letmein", channels.SetOf("ABC"))
	authenticator.Save(user)

	// Create a doc on two channels (sequence 1):
	revid, _ := db.Put("doc1", Body{"channels": []string{"ABC", "PBS"}})
	db.changeCache.waitForSequence(1)
	time.Sleep(100 * time.Millisecond)

	// Modify user to have access to both channels (sequence 2):
	userInfo, err := db.GetPrincipal("naomi", true)
	assert.True(t, userInfo != nil)
	userInfo.ExplicitChannels = base.SetOf("ABC", "PBS")
	_, err = db.UpdatePrincipal(*userInfo, true, true)
	assertNoError(t, err, "UpdatePrincipal failed")

	// Check the _changes feed:
	db.changeCache.waitForSequence(1)
	time.Sleep(100 * time.Millisecond)
	db.Bucket.Dump()
	if changeCache, ok := db.changeCache.(*kvChangeIndex); ok {
		changeCache.reader.indexReadBucket.Dump()
	}
	db.user, _ = authenticator.GetUser("naomi")
	changes, err := db.GetChanges(base.SetOf("*"), getZeroSequence(db))
	assertNoError(t, err, "Couldn't GetChanges")
	printChanges(changes)
	time.Sleep(1000 * time.Millisecond)
	assert.Equals(t, len(changes), 3)
	assert.DeepEquals(t, changes[0], &ChangeEntry{ // Seq 1, from ABC
		Seq:     SequenceID{Seq: 1},
		ID:      "doc1",
		Changes: []ChangeRev{{"rev": revid}}})
	assert.DeepEquals(t, changes[1], &ChangeEntry{ // Seq 1, from PBS backfill
		Seq:     SequenceID{Seq: 1, TriggeredBy: 2},
		ID:      "doc1",
		Changes: []ChangeRev{{"rev": revid}}})
	assert.DeepEquals(t, changes[2], &ChangeEntry{ // Seq 2, from ABC and PBS
		Seq:     SequenceID{Seq: 2},
		ID:      "_user/naomi",
		Changes: []ChangeRev{}})
	lastSeq := getLastSeq(changes)
	lastSeq, _ = db.ParseSequenceID(lastSeq.String())

	// Add a new doc (sequence 3):
	revid, _ = db.Put("doc2", Body{"channels": []string{"PBS"}})

	// Check the _changes feed -- this is to make sure the changeCache properly received
	// sequence 2 (the user doc) and isn't stuck waiting for it.
	db.changeCache.waitForSequence(3)
	changes, err = db.GetChanges(base.SetOf("*"), ChangesOptions{Since: lastSeq})

	assertNoError(t, err, "Couldn't GetChanges (2nd)")

	assert.Equals(t, len(changes), 1)
	assert.DeepEquals(t, changes[0], &ChangeEntry{
		Seq:     SequenceID{Seq: 3},
		ID:      "doc2",
		Changes: []ChangeRev{{"rev": revid}}})

	// validate from zero
	changes, err = db.GetChanges(base.SetOf("*"), getZeroSequence(db))
	assertNoError(t, err, "Couldn't GetChanges")
	printChanges(changes)
}

func TestDocDeletionFromChannelCoalescedRemoved(t *testing.T) {
	base.EnableLogKey("*")
	db := setupTestDB(t)
	defer tearDownTestDB(t, db)

	db.ChannelMapper = channels.NewDefaultChannelMapper()

	// Create a user with access to channel A
	authenticator := db.Authenticator()
	user, _ := authenticator.NewUser("alice", "letmein", channels.SetOf("A"))
	authenticator.Save(user)

	// Create a doc on two channels (sequence 1):
	revid, _ := db.Put("alpha", Body{"channels": []string{"A", "B"}})
	db.changeCache.waitForSequence(1)
	time.Sleep(100 * time.Millisecond)

	if changeCache, ok := db.changeCache.(*kvChangeIndex); ok {
		changeCache.reader.indexReadBucket.Dump()
	}
	db.user, _ = authenticator.GetUser("alice")
	changes, err := db.GetChanges(base.SetOf("*"), getZeroSequence(db))
	assertNoError(t, err, "Couldn't GetChanges")
	printChanges(changes)
	time.Sleep(1000 * time.Millisecond)
	assert.Equals(t, len(changes), 1)
	assert.DeepEquals(t, changes[0], &ChangeEntry{ // Seq 1, from A
		Seq:     SequenceID{Seq: 1},
		ID:      "alpha",
		Changes: []ChangeRev{{"rev": revid}}})
	lastSeq := getLastSeq(changes)
	lastSeq, _ = db.ParseSequenceID(lastSeq.String())

	// Get raw document from the bucket
	rv, _, _ := db.Bucket.GetRaw("alpha") // cas, err

	//Unmarshall into nested maps
	var x map[string]interface{}
	json.Unmarshal(rv, &x)
	assert.True(t, err == nil)

	sync := x["_sync"].(map[string]interface{})
	sync["sequence"] = 3
	sync["rev"] = "3-e99405a23fa102238fa8c3fd499b15bc"
	sync["recent_sequences"] = []uint64{1, 2, 3}

	cm := make(channels.ChannelMap)
	cm["A"] = &channels.ChannelRemoval{Seq: 2, RevID: "2-e99405a23fa102238fa8c3fd499b15bc"}
	sync["channels"] = cm

	history := sync["history"].(map[string]interface{})
	history["revs"] = []string{revid, "2-e99405a23fa102238fa8c3fd499b15bc", "3-e99405a23fa102238fa8c3fd499b15bc"}
	history["parents"] = []int{-1, 0, 1}
	history["channels"] = []base.Set{base.SetOf("A", "B"), base.SetOf("B"), base.SetOf("B")}

	//Marshall back to JSON
	b, err := json.Marshal(x)

	// Update raw document in the bucket
	db.Bucket.SetRaw("alpha", 0, b)

	// Check the _changes feed -- this is to make sure the changeCache properly received
	// sequence 3 and isn't stuck waiting for it.
	db.changeCache.waitForSequence(3)
	changes, err = db.GetChanges(base.SetOf("*"), ChangesOptions{Since: lastSeq})

	assertNoError(t, err, "Couldn't GetChanges (2nd)")

	assert.Equals(t, len(changes), 1)
	assert.DeepEquals(t, changes[0], &ChangeEntry{
		Seq:     SequenceID{Seq: 2},
		ID:      "alpha",
		Removed: base.SetOf("A"),
		allRemoved: true,
		Changes: []ChangeRev{{"rev": "2-e99405a23fa102238fa8c3fd499b15bc"}}})

	printChanges(changes)
}

func TestDocDeletionFromChannelCoalesced(t *testing.T) {
	db := setupTestDB(t)
	defer tearDownTestDB(t, db)

	db.ChannelMapper = channels.NewDefaultChannelMapper()

	// Create a user with access to channel A
	authenticator := db.Authenticator()
	user, _ := authenticator.NewUser("alice", "letmein", channels.SetOf("A"))
	authenticator.Save(user)

	// Create a doc on two channels (sequence 1):
	revid, _ := db.Put("alpha", Body{"channels": []string{"A", "B"}})
	db.changeCache.waitForSequence(1)
	time.Sleep(100 * time.Millisecond)

	if changeCache, ok := db.changeCache.(*kvChangeIndex); ok {
		changeCache.reader.indexReadBucket.Dump()
	}
	db.user, _ = authenticator.GetUser("alice")
	changes, err := db.GetChanges(base.SetOf("*"), getZeroSequence(db))
	assertNoError(t, err, "Couldn't GetChanges")
	printChanges(changes)
	time.Sleep(1000 * time.Millisecond)

	assert.Equals(t, len(changes), 1)
	assert.DeepEquals(t, changes[0], &ChangeEntry{ // Seq 1, from A
		Seq:     SequenceID{Seq: 1},
		ID:      "alpha",
		Changes: []ChangeRev{{"rev": revid}}})
	lastSeq := getLastSeq(changes)
	lastSeq, _ = db.ParseSequenceID(lastSeq.String())

	// Get raw document from the bucket
	rv, _, _ := db.Bucket.GetRaw("alpha") // cas, err

	//Unmarshall into nested maps
	var x map[string]interface{}
	json.Unmarshal(rv, &x)

	assert.True(t, err == nil)

	sync := x["_sync"].(map[string]interface{})
	sync["sequence"] = 3
	sync["rev"] = "3-e99405a23fa102238fa8c3fd499b15bc"
	sync["recent_sequences"] = []uint64{1, 2, 3}

	history := sync["history"].(map[string]interface{})
	history["revs"] = []string{revid, "2-e99405a23fa102238fa8c3fd499b15bc", "3-e99405a23fa102238fa8c3fd499b15bc"}
	history["parents"] = []int{-1, 0, 1}
	history["channels"] = []base.Set{base.SetOf("A", "B"), base.SetOf("A", "B"), base.SetOf("A", "B")}

	//Marshall back to JSON
	b, err := json.Marshal(x)

	// Update raw document in the bucket
	db.Bucket.SetRaw("alpha", 0, b)

	// Check the _changes feed -- this is to make sure the changeCache properly received
	// sequence 3 (the modified document) and isn't stuck waiting for it.
	db.changeCache.waitForSequence(3)

	changes, err = db.GetChanges(base.SetOf("*"), ChangesOptions{Since: lastSeq})

	assertNoError(t, err, "Couldn't GetChanges (2nd)")

	assert.Equals(t, len(changes), 1)
	assert.DeepEquals(t, changes[0], &ChangeEntry{
		Seq:     SequenceID{Seq: 3},
		ID:      "alpha",
		Changes: []ChangeRev{{"rev": "3-e99405a23fa102238fa8c3fd499b15bc"}}})

	printChanges(changes)
}
<commit_msg>Added pseudoDoc: true to expected result of failing unit test (#2442)<commit_after>//  Copyright (c) 2012 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package db

import (
	"encoding/json"
	"log"
	"testing"
	"time"

	"github.com/couchbaselabs/go.assert"

	"github.com/couchbase/sync_gateway/base"
	"github.com/couchbase/sync_gateway/channels"
)

// Unit test for bug #314
func TestChangesAfterChannelAdded(t *testing.T) {
	db := setupTestDB(t)
	defer tearDownTestDB(t, db)
	_testChangesAfterChannelAdded(t, db)
}

func printChanges(changes []*ChangeEntry) {
	for _, change := range changes {
		log.Printf("Change:%+v", change)
	}
}

func getLastSeq(changes []*ChangeEntry) SequenceID {
	if len(changes) > 0 {
		return changes[len(changes)-1].Seq
	}
	return SequenceID{}
}

func getZeroSequence(db *Database) ChangesOptions {
	if db.SequenceType == IntSequenceType {
		return ChangesOptions{Since: SequenceID{Seq: 0}}
	} else {
		return ChangesOptions{Since: SequenceID{Clock: base.NewSequenceClockImpl()}}
	}
}

func _testChangesAfterChannelAdded(t *testing.T, db *Database) {
	base.EnableLogKey("IndexChanges")
	base.EnableLogKey("Hash+")
	db.ChannelMapper = channels.NewDefaultChannelMapper()

	// Create a user with access to channel ABC
	authenticator := db.Authenticator()
	user, _ := authenticator.NewUser("naomi", "letmein", channels.SetOf("ABC"))
	authenticator.Save(user)

	// Create a doc on two channels (sequence 1):
	revid, _ := db.Put("doc1", Body{"channels": []string{"ABC", "PBS"}})
	db.changeCache.waitForSequence(1)
	time.Sleep(100 * time.Millisecond)

	// Modify user to have access to both channels (sequence 2):
	userInfo, err := db.GetPrincipal("naomi", true)
	assert.True(t, userInfo != nil)
	userInfo.ExplicitChannels = base.SetOf("ABC", "PBS")
	_, err = db.UpdatePrincipal(*userInfo, true, true)
	assertNoError(t, err, "UpdatePrincipal failed")

	// Check the _changes feed:
	db.changeCache.waitForSequence(1)
	time.Sleep(100 * time.Millisecond)
	db.Bucket.Dump()
	if changeCache, ok := db.changeCache.(*kvChangeIndex); ok {
		changeCache.reader.indexReadBucket.Dump()
	}
	db.user, _ = authenticator.GetUser("naomi")
	changes, err := db.GetChanges(base.SetOf("*"), getZeroSequence(db))
	assertNoError(t, err, "Couldn't GetChanges")
	printChanges(changes)
	time.Sleep(1000 * time.Millisecond)
	assert.Equals(t, len(changes), 3)
	assert.DeepEquals(t, changes[0], &ChangeEntry{ // Seq 1, from ABC
		Seq:     SequenceID{Seq: 1},
		ID:      "doc1",
		Changes: []ChangeRev{{"rev": revid}}})
	assert.DeepEquals(t, changes[1], &ChangeEntry{ // Seq 1, from PBS backfill
		Seq:     SequenceID{Seq: 1, TriggeredBy: 2},
		ID:      "doc1",
		Changes: []ChangeRev{{"rev": revid}}})
	assert.DeepEquals(t, changes[2], &ChangeEntry{ // Seq 2, from ABC and PBS
		Seq:     SequenceID{Seq: 2},
		ID:      "_user/naomi",
		Changes: []ChangeRev{},
		pseudoDoc: true})
	lastSeq := getLastSeq(changes)
	lastSeq, _ = db.ParseSequenceID(lastSeq.String())

	// Add a new doc (sequence 3):
	revid, _ = db.Put("doc2", Body{"channels": []string{"PBS"}})

	// Check the _changes feed -- this is to make sure the changeCache properly received
	// sequence 2 (the user doc) and isn't stuck waiting for it.
	db.changeCache.waitForSequence(3)
	changes, err = db.GetChanges(base.SetOf("*"), ChangesOptions{Since: lastSeq})

	assertNoError(t, err, "Couldn't GetChanges (2nd)")

	assert.Equals(t, len(changes), 1)
	assert.DeepEquals(t, changes[0], &ChangeEntry{
		Seq:     SequenceID{Seq: 3},
		ID:      "doc2",
		Changes: []ChangeRev{{"rev": revid}}})

	// validate from zero
	changes, err = db.GetChanges(base.SetOf("*"), getZeroSequence(db))
	assertNoError(t, err, "Couldn't GetChanges")
	printChanges(changes)
}

func TestDocDeletionFromChannelCoalescedRemoved(t *testing.T) {
	base.EnableLogKey("*")
	db := setupTestDB(t)
	defer tearDownTestDB(t, db)

	db.ChannelMapper = channels.NewDefaultChannelMapper()

	// Create a user with access to channel A
	authenticator := db.Authenticator()
	user, _ := authenticator.NewUser("alice", "letmein", channels.SetOf("A"))
	authenticator.Save(user)

	// Create a doc on two channels (sequence 1):
	revid, _ := db.Put("alpha", Body{"channels": []string{"A", "B"}})
	db.changeCache.waitForSequence(1)
	time.Sleep(100 * time.Millisecond)

	if changeCache, ok := db.changeCache.(*kvChangeIndex); ok {
		changeCache.reader.indexReadBucket.Dump()
	}
	db.user, _ = authenticator.GetUser("alice")
	changes, err := db.GetChanges(base.SetOf("*"), getZeroSequence(db))
	assertNoError(t, err, "Couldn't GetChanges")
	printChanges(changes)
	time.Sleep(1000 * time.Millisecond)
	assert.Equals(t, len(changes), 1)
	assert.DeepEquals(t, changes[0], &ChangeEntry{ // Seq 1, from A
		Seq:     SequenceID{Seq: 1},
		ID:      "alpha",
		Changes: []ChangeRev{{"rev": revid}}})
	lastSeq := getLastSeq(changes)
	lastSeq, _ = db.ParseSequenceID(lastSeq.String())

	// Get raw document from the bucket
	rv, _, _ := db.Bucket.GetRaw("alpha") // cas, err

	//Unmarshall into nested maps
	var x map[string]interface{}
	json.Unmarshal(rv, &x)
	assert.True(t, err == nil)

	sync := x["_sync"].(map[string]interface{})
	sync["sequence"] = 3
	sync["rev"] = "3-e99405a23fa102238fa8c3fd499b15bc"
	sync["recent_sequences"] = []uint64{1, 2, 3}

	cm := make(channels.ChannelMap)
	cm["A"] = &channels.ChannelRemoval{Seq: 2, RevID: "2-e99405a23fa102238fa8c3fd499b15bc"}
	sync["channels"] = cm

	history := sync["history"].(map[string]interface{})
	history["revs"] = []string{revid, "2-e99405a23fa102238fa8c3fd499b15bc", "3-e99405a23fa102238fa8c3fd499b15bc"}
	history["parents"] = []int{-1, 0, 1}
	history["channels"] = []base.Set{base.SetOf("A", "B"), base.SetOf("B"), base.SetOf("B")}

	//Marshall back to JSON
	b, err := json.Marshal(x)

	// Update raw document in the bucket
	db.Bucket.SetRaw("alpha", 0, b)

	// Check the _changes feed -- this is to make sure the changeCache properly received
	// sequence 3 and isn't stuck waiting for it.
	db.changeCache.waitForSequence(3)
	changes, err = db.GetChanges(base.SetOf("*"), ChangesOptions{Since: lastSeq})

	assertNoError(t, err, "Couldn't GetChanges (2nd)")

	assert.Equals(t, len(changes), 1)
	assert.DeepEquals(t, changes[0], &ChangeEntry{
		Seq:     SequenceID{Seq: 2},
		ID:      "alpha",
		Removed: base.SetOf("A"),
		allRemoved: true,
		Changes: []ChangeRev{{"rev": "2-e99405a23fa102238fa8c3fd499b15bc"}}})

	printChanges(changes)
}

func TestDocDeletionFromChannelCoalesced(t *testing.T) {
	db := setupTestDB(t)
	defer tearDownTestDB(t, db)

	db.ChannelMapper = channels.NewDefaultChannelMapper()

	// Create a user with access to channel A
	authenticator := db.Authenticator()
	user, _ := authenticator.NewUser("alice", "letmein", channels.SetOf("A"))
	authenticator.Save(user)

	// Create a doc on two channels (sequence 1):
	revid, _ := db.Put("alpha", Body{"channels": []string{"A", "B"}})
	db.changeCache.waitForSequence(1)
	time.Sleep(100 * time.Millisecond)

	if changeCache, ok := db.changeCache.(*kvChangeIndex); ok {
		changeCache.reader.indexReadBucket.Dump()
	}
	db.user, _ = authenticator.GetUser("alice")
	changes, err := db.GetChanges(base.SetOf("*"), getZeroSequence(db))
	assertNoError(t, err, "Couldn't GetChanges")
	printChanges(changes)
	time.Sleep(1000 * time.Millisecond)

	assert.Equals(t, len(changes), 1)
	assert.DeepEquals(t, changes[0], &ChangeEntry{ // Seq 1, from A
		Seq:     SequenceID{Seq: 1},
		ID:      "alpha",
		Changes: []ChangeRev{{"rev": revid}}})
	lastSeq := getLastSeq(changes)
	lastSeq, _ = db.ParseSequenceID(lastSeq.String())

	// Get raw document from the bucket
	rv, _, _ := db.Bucket.GetRaw("alpha") // cas, err

	//Unmarshall into nested maps
	var x map[string]interface{}
	json.Unmarshal(rv, &x)

	assert.True(t, err == nil)

	sync := x["_sync"].(map[string]interface{})
	sync["sequence"] = 3
	sync["rev"] = "3-e99405a23fa102238fa8c3fd499b15bc"
	sync["recent_sequences"] = []uint64{1, 2, 3}

	history := sync["history"].(map[string]interface{})
	history["revs"] = []string{revid, "2-e99405a23fa102238fa8c3fd499b15bc", "3-e99405a23fa102238fa8c3fd499b15bc"}
	history["parents"] = []int{-1, 0, 1}
	history["channels"] = []base.Set{base.SetOf("A", "B"), base.SetOf("A", "B"), base.SetOf("A", "B")}

	//Marshall back to JSON
	b, err := json.Marshal(x)

	// Update raw document in the bucket
	db.Bucket.SetRaw("alpha", 0, b)

	// Check the _changes feed -- this is to make sure the changeCache properly received
	// sequence 3 (the modified document) and isn't stuck waiting for it.
	db.changeCache.waitForSequence(3)

	changes, err = db.GetChanges(base.SetOf("*"), ChangesOptions{Since: lastSeq})

	assertNoError(t, err, "Couldn't GetChanges (2nd)")

	assert.Equals(t, len(changes), 1)
	assert.DeepEquals(t, changes[0], &ChangeEntry{
		Seq:     SequenceID{Seq: 3},
		ID:      "alpha",
		Changes: []ChangeRev{{"rev": "3-e99405a23fa102238fa8c3fd499b15bc"}}})

	printChanges(changes)
}
<|endoftext|>