<commit_before>// service registration and discovery

package server

import (
	"errors"
	"github.com/coreos/etcd/client"
	"golang.org/x/net/context"
	"os"
	"strings"
	"time"
)

const (
	EtcdServersPrefix    = "/pando/servers/"
	EtcdServersPrefixCnt = 2
	EnvTCPProxy          = "TCP_PROXY"
	EnvHTTPProxy         = "HTTP_PROXY"
)

type ServerManager struct {
	serverName string
	// servername -> hosttype -> hostlist
	// eg. var hosts []string = mapServers["testserver"]["rpchost"]
	mapServers map[string](map[string][]string)
	etcdHosts  []string
}

// etcd hosts is config as http://ip1:port1;http://ip2:port2;http://ip3:port3
func NewServerManager(name string, etcd string) (*ServerManager, error) {
	if etcd == "" {
		return nil, errors.New("no etcd host found!")
	}
	return &ServerManager{
		serverName: name,
		etcdHosts:  strings.Split(etcd, ";"),
		mapServers: make(map[string](map[string][]string)),
	}, nil
}

// register server to etcd
func (mgr *ServerManager) RegisterServer() error {
	if serverInstance == nil {
		return errorf(errServerNotInit)
	}
	cfg := client.Config{
		Endpoints: mgr.etcdHosts,
		Transport: client.DefaultTransport,
		// set timeout per request to fail fast when the target endpoint is unavailable
		HeaderTimeoutPerRequest: time.Second,
	}
	c, err := client.New(cfg)
	if err != nil {
		return err
	}
	kapi := client.NewKeysAPI(c)
	prefix := EtcdServersPrefix + mgr.serverName + "/"
	var response *client.Response
	opt := &client.SetOptions{TTL: 90 * time.Second}
	if serverInstance.tcpsvr != nil {
		addr := os.Getenv(EnvTCPProxy)
		if addr == "" {
			addr, _ = fixHostIp(*confTCPHost)
		}
		response, err = kapi.Set(context.Background(), prefix+FlagTCPHost+"/"+addr, addr, opt)
	}
	if serverInstance.rpcsvr != nil {
		addr, _ := fixHostIp(*confRPCHost)
		response, err = kapi.Set(context.Background(), prefix+FlagRPCHost+"/"+addr, addr, opt)
	}
	if serverInstance.httpsvr != nil {
		addr := os.Getenv(EnvHTTPProxy)
		if addr == "" {
			addr, _ = fixHostIp(*confHTTPHost)
		}
		response, err = kapi.Set(context.Background(), prefix+FlagHTTPHost+"/"+addr, addr, opt)
	}
	if err != nil {
		return err
	}
	// print common key info
	Log.Infof("RegisterServer is done. Metadata is %q\n", response)

	return nil
}

// update server hosts
func (mgr *ServerManager) UpdateServerHosts() error {
	if serverInstance == nil {
		return errorf(errServerNotInit)
	}

	cfg := client.Config{
		Endpoints: mgr.etcdHosts,
		Transport: client.DefaultTransport,
		// set timeout per request to fail fast when the target endpoint is unavailable
		HeaderTimeoutPerRequest: time.Second,
	}
	c, err := client.New(cfg)
	if err != nil {
		return err
	}

	kapi := client.NewKeysAPI(c)
	prefix := EtcdServersPrefix
	opt := &client.GetOptions{Recursive: true}
	response, err := kapi.Get(context.Background(), prefix, opt)
	if err != nil {
		return err
	}

	servers := make(map[string](map[string][]string))

	root := response.Node
	if root.Dir != true {
		return errorf(errWrongEtcdPath, root.Key)
	}
	for _, server := range root.Nodes {
		if server.Dir != true {
			return errorf(errWrongEtcdPath, server.Key)
		}
		name := strings.Split(server.Key, "/")[EtcdServersPrefixCnt+1]
		servers[name] = make(map[string][]string)
		for _, hosttype := range server.Nodes {
			if hosttype.Dir != true {
				return errorf(errWrongEtcdPath, hosttype.Key)
			}
			host := strings.Split(hosttype.Key, "/")[EtcdServersPrefixCnt+2]
			servers[name][host] = []string{}
			for _, hostaddr := range hosttype.Nodes {
				addr := strings.Split(hostaddr.Key, "/")[EtcdServersPrefixCnt+3]
				servers[name][host] = append(servers[name][host], addr)
			}
		}
	}

	mgr.mapServers = servers

	Log.Infof("UpdateServerHosts is done: %v", mgr.mapServers)
	return nil

}

// get host ips for the server, now return all hosts
func (mgr *ServerManager) GetServerHosts(serverName string, hostType string) ([]string, error) {
	server, ok := mgr.mapServers[serverName]
	if !ok {
		// try update server hosts mannually
		mgr.UpdateServerHosts()
	}
	server, ok = mgr.mapServers[serverName]
	if !ok {
		return nil, errorf("no server for %s", serverName)
	}
	hosts, ok := server[hostType]
	if !ok || len(hosts) == 0 {
		return nil, errorf("no hosts for %s:%s", serverName, hostType)
	}

	return hosts, nil
}
<commit_msg>change PROXY evironment name<commit_after>// service registration and discovery

package server

import (
	"errors"
	"github.com/coreos/etcd/client"
	"golang.org/x/net/context"
	"os"
	"strings"
	"time"
)

const (
	EtcdServersPrefix    = "/pando/servers/"
	EtcdServersPrefixCnt = 2
	EnvTCPProxy          = "TCP_PROXY_ADDR"
	EnvHTTPProxy         = "HTTP_PROXY_ADDR"
)

type ServerManager struct {
	serverName string
	// servername -> hosttype -> hostlist
	// eg. var hosts []string = mapServers["testserver"]["rpchost"]
	mapServers map[string](map[string][]string)
	etcdHosts  []string
}

// etcd hosts is config as http://ip1:port1;http://ip2:port2;http://ip3:port3
func NewServerManager(name string, etcd string) (*ServerManager, error) {
	if etcd == "" {
		return nil, errors.New("no etcd host found!")
	}
	return &ServerManager{
		serverName: name,
		etcdHosts:  strings.Split(etcd, ";"),
		mapServers: make(map[string](map[string][]string)),
	}, nil
}

// register server to etcd
func (mgr *ServerManager) RegisterServer() error {
	if serverInstance == nil {
		return errorf(errServerNotInit)
	}
	cfg := client.Config{
		Endpoints: mgr.etcdHosts,
		Transport: client.DefaultTransport,
		// set timeout per request to fail fast when the target endpoint is unavailable
		HeaderTimeoutPerRequest: time.Second,
	}
	c, err := client.New(cfg)
	if err != nil {
		return err
	}
	kapi := client.NewKeysAPI(c)
	prefix := EtcdServersPrefix + mgr.serverName + "/"
	var response *client.Response
	opt := &client.SetOptions{TTL: 90 * time.Second}
	if serverInstance.tcpsvr != nil {
		addr := os.Getenv(EnvTCPProxy)
		if addr == "" {
			addr, _ = fixHostIp(*confTCPHost)
		}
		response, err = kapi.Set(context.Background(), prefix+FlagTCPHost+"/"+addr, addr, opt)
	}
	if serverInstance.rpcsvr != nil {
		addr, _ := fixHostIp(*confRPCHost)
		response, err = kapi.Set(context.Background(), prefix+FlagRPCHost+"/"+addr, addr, opt)
	}
	if serverInstance.httpsvr != nil {
		addr := os.Getenv(EnvHTTPProxy)
		if addr == "" {
			addr, _ = fixHostIp(*confHTTPHost)
		}
		response, err = kapi.Set(context.Background(), prefix+FlagHTTPHost+"/"+addr, addr, opt)
	}
	if err != nil {
		return err
	}
	// print common key info
	Log.Infof("RegisterServer is done. Metadata is %q\n", response)

	return nil
}

// update server hosts
func (mgr *ServerManager) UpdateServerHosts() error {
	if serverInstance == nil {
		return errorf(errServerNotInit)
	}

	cfg := client.Config{
		Endpoints: mgr.etcdHosts,
		Transport: client.DefaultTransport,
		// set timeout per request to fail fast when the target endpoint is unavailable
		HeaderTimeoutPerRequest: time.Second,
	}
	c, err := client.New(cfg)
	if err != nil {
		return err
	}

	kapi := client.NewKeysAPI(c)
	prefix := EtcdServersPrefix
	opt := &client.GetOptions{Recursive: true}
	response, err := kapi.Get(context.Background(), prefix, opt)
	if err != nil {
		return err
	}

	servers := make(map[string](map[string][]string))

	root := response.Node
	if root.Dir != true {
		return errorf(errWrongEtcdPath, root.Key)
	}
	for _, server := range root.Nodes {
		if server.Dir != true {
			return errorf(errWrongEtcdPath, server.Key)
		}
		name := strings.Split(server.Key, "/")[EtcdServersPrefixCnt+1]
		servers[name] = make(map[string][]string)
		for _, hosttype := range server.Nodes {
			if hosttype.Dir != true {
				return errorf(errWrongEtcdPath, hosttype.Key)
			}
			host := strings.Split(hosttype.Key, "/")[EtcdServersPrefixCnt+2]
			servers[name][host] = []string{}
			for _, hostaddr := range hosttype.Nodes {
				addr := strings.Split(hostaddr.Key, "/")[EtcdServersPrefixCnt+3]
				servers[name][host] = append(servers[name][host], addr)
			}
		}
	}

	mgr.mapServers = servers

	Log.Infof("UpdateServerHosts is done: %v", mgr.mapServers)
	return nil

}

// get host ips for the server, now return all hosts
func (mgr *ServerManager) GetServerHosts(serverName string, hostType string) ([]string, error) {
	server, ok := mgr.mapServers[serverName]
	if !ok {
		// try update server hosts mannually
		mgr.UpdateServerHosts()
	}
	server, ok = mgr.mapServers[serverName]
	if !ok {
		return nil, errorf("no server for %s", serverName)
	}
	hosts, ok := server[hostType]
	if !ok || len(hosts) == 0 {
		return nil, errorf("no hosts for %s:%s", serverName, hostType)
	}

	return hosts, nil
}
<|endoftext|>


<commit_before>package donut

import (
	"bytes"
	"io"
	"strconv"
	"time"

	"crypto/md5"
	"encoding/hex"
	"errors"
	"github.com/minio-io/minio/pkg/encoding/erasure"
	"github.com/minio-io/minio/pkg/utils/split"
	"strings"
)

// erasureReader - returns aligned streaming reads over a PipeWriter
func erasureReader(readers []io.ReadCloser, donutMetadata map[string]string, writer *io.PipeWriter) {
	// TODO handle errors
	totalChunks, _ := strconv.Atoi(donutMetadata["chunkCount"])
	totalLeft, _ := strconv.Atoi(donutMetadata["size"])
	blockSize, _ := strconv.Atoi(donutMetadata["blockSize"])
	k, _ := strconv.Atoi(donutMetadata["erasureK"])
	m, _ := strconv.Atoi(donutMetadata["erasureM"])
	expectedMd5sum, _ := hex.DecodeString(donutMetadata["md5"])
	summer := md5.New()
	// TODO select technique properly
	params, _ := erasure.ParseEncoderParams(uint8(k), uint8(m), erasure.Cauchy)
	encoder := erasure.NewEncoder(params)
	for _, reader := range readers {
		defer reader.Close()
	}
	for i := 0; i < totalChunks; i++ {
		curBlockSize := totalLeft
		if blockSize < totalLeft {
			curBlockSize = blockSize
		}
		curChunkSize := erasure.GetEncodedChunkLen(curBlockSize, uint8(k))

		encodedBytes := make([][]byte, 16)
		for i, reader := range readers {
			var bytesBuffer bytes.Buffer
			// TODO watch for errors
			io.CopyN(&bytesBuffer, reader, int64(curChunkSize))
			encodedBytes[i] = bytesBuffer.Bytes()
		}
		decodedData, err := encoder.Decode(encodedBytes, curBlockSize)
		if err != nil {
			writer.CloseWithError(err)
			return
		}
		summer.Write(decodedData)
		io.Copy(writer, bytes.NewBuffer(decodedData))
		totalLeft = totalLeft - blockSize
	}
	actualMd5sum := summer.Sum(nil)
	if bytes.Compare(expectedMd5sum, actualMd5sum) != 0 {
		writer.CloseWithError(errors.New("decoded md5sum did not match"))
		return
	}
	writer.Close()
}

// erasure writer

type erasureWriter struct {
	writers       []Writer
	metadata      map[string]string
	donutMetadata map[string]string // not exposed
	erasureWriter *io.PipeWriter
	isClosed      <-chan bool
}

// newErasureWriter - get a new writer
func newErasureWriter(writers []Writer) ObjectWriter {
	r, w := io.Pipe()
	isClosed := make(chan bool)
	writer := erasureWriter{
		writers:       writers,
		metadata:      make(map[string]string),
		erasureWriter: w,
		isClosed:      isClosed,
	}
	go erasureGoroutine(r, writer, isClosed)
	return writer
}

func erasureGoroutine(r *io.PipeReader, eWriter erasureWriter, isClosed chan<- bool) {
	chunks := split.Stream(r, 10*1024*1024)
	params, _ := erasure.ParseEncoderParams(8, 8, erasure.Cauchy)
	encoder := erasure.NewEncoder(params)
	chunkCount := 0
	totalLength := 0
	summer := md5.New()
	for chunk := range chunks {
		if chunk.Err == nil {
			totalLength = totalLength + len(chunk.Data)
			encodedBlocks, _ := encoder.Encode(chunk.Data)
			summer.Write(chunk.Data)
			for blockIndex, block := range encodedBlocks {
				io.Copy(eWriter.writers[blockIndex], bytes.NewBuffer(block))
			}
		}
		chunkCount = chunkCount + 1
	}
	dataMd5sum := summer.Sum(nil)
	metadata := make(map[string]string)
	metadata["blockSize"] = strconv.Itoa(10 * 1024 * 1024)
	metadata["chunkCount"] = strconv.Itoa(chunkCount)
	metadata["created"] = time.Now().Format(time.RFC3339Nano)
	metadata["erasureK"] = "8"
	metadata["erasureM"] = "8"
	metadata["erasureTechnique"] = "Cauchy"
	metadata["md5"] = hex.EncodeToString(dataMd5sum)
	metadata["size"] = strconv.Itoa(totalLength)
	for _, nodeWriter := range eWriter.writers {
		if nodeWriter != nil {
			nodeWriter.SetMetadata(eWriter.metadata)
			nodeWriter.SetDonutMetadata(metadata)
			nodeWriter.Close()
		}
	}
	isClosed <- true
}

func (eWriter erasureWriter) Write(data []byte) (int, error) {
	io.Copy(eWriter.erasureWriter, bytes.NewBuffer(data))
	return len(data), nil
}

func (eWriter erasureWriter) Close() error {
	eWriter.erasureWriter.Close()
	<-eWriter.isClosed
	return nil
}

func (eWriter erasureWriter) CloseWithError(err error) error {
	for _, writer := range eWriter.writers {
		if writer != nil {
			writer.CloseWithError(err)
		}
	}
	return nil
}

func (eWriter erasureWriter) SetMetadata(metadata map[string]string) error {
	for k := range metadata {
		if strings.HasPrefix(k, "sys.") {
			return errors.New("Invalid key '" + k + "', cannot start with sys.'")
		}
	}
	for k := range eWriter.metadata {
		delete(eWriter.metadata, k)
	}
	for k, v := range metadata {
		eWriter.metadata[k] = v
	}
	return nil
}

func (eWriter erasureWriter) GetMetadata() (map[string]string, error) {
	metadata := make(map[string]string)
	for k, v := range eWriter.metadata {
		metadata[k] = v
	}
	return metadata, nil
}
<commit_msg>Handle errors properly during erasure Decoding, also populate technique and verify<commit_after>package donut

import (
	"bytes"
	"errors"
	"io"
	"strconv"
	"strings"
	"time"

	"crypto/md5"
	"encoding/hex"

	"github.com/minio-io/minio/pkg/encoding/erasure"
	"github.com/minio-io/minio/pkg/utils/split"
)

// getErasureTechnique - convert technique string into Technique type
func getErasureTechnique(technique string) (erasure.Technique, error) {
	switch true {
	case technique == "Cauchy":
		return erasure.Cauchy, nil
	case technique == "Vandermonde":
		return erasure.Cauchy, nil
	default:
		return -1, errors.New("Invalid erasure technique")
	}
}

// erasureReader - returns aligned streaming reads over a PipeWriter
func erasureReader(readers []io.ReadCloser, donutMetadata map[string]string, writer *io.PipeWriter) {
	totalChunks, err := strconv.Atoi(donutMetadata["chunkCount"])
	if err != nil {
		writer.CloseWithError(err)
		return
	}
	totalLeft, err := strconv.Atoi(donutMetadata["size"])
	if err != nil {
		writer.CloseWithError(err)
		return
	}
	blockSize, err := strconv.Atoi(donutMetadata["blockSize"])
	if err != nil {
		writer.CloseWithError(err)
		return
	}
	k, err := strconv.Atoi(donutMetadata["erasureK"])
	if err != nil {
		writer.CloseWithError(err)
		return
	}
	m, err := strconv.Atoi(donutMetadata["erasureM"])
	if err != nil {
		writer.CloseWithError(err)
		return
	}
	expectedMd5sum, err := hex.DecodeString(donutMetadata["md5"])
	if err != nil {
		writer.CloseWithError(err)
		return
	}
	technique, err := getErasureTechnique(donutMetadata["erasureTechnique"])
	if err != nil {
		writer.CloseWithError(err)
		return
	}
	summer := md5.New()
	params, _ := erasure.ParseEncoderParams(uint8(k), uint8(m), technique)
	encoder := erasure.NewEncoder(params)
	for _, reader := range readers {
		defer reader.Close()
	}
	for i := 0; i < totalChunks; i++ {
		curBlockSize := totalLeft
		if blockSize < totalLeft {
			curBlockSize = blockSize
		}
		curChunkSize := erasure.GetEncodedChunkLen(curBlockSize, uint8(k))
		encodedBytes := make([][]byte, 16)
		for i, reader := range readers {
			var bytesBuffer bytes.Buffer
			_, err := io.CopyN(&bytesBuffer, reader, int64(curChunkSize))
			if err != nil {
				writer.CloseWithError(err)
				return
			}
			encodedBytes[i] = bytesBuffer.Bytes()
		}
		decodedData, err := encoder.Decode(encodedBytes, curBlockSize)
		if err != nil {
			writer.CloseWithError(err)
			return
		}
		summer.Write(decodedData)
		_, err = io.Copy(writer, bytes.NewBuffer(decodedData))
		if err != nil {
			writer.CloseWithError(err)
			return
		}
		totalLeft = totalLeft - blockSize
	}
	actualMd5sum := summer.Sum(nil)
	if bytes.Compare(expectedMd5sum, actualMd5sum) != 0 {
		writer.CloseWithError(errors.New("decoded md5sum did not match"))
		return
	}
	writer.Close()
	return
}

// erasure writer

type erasureWriter struct {
	writers       []Writer
	metadata      map[string]string
	donutMetadata map[string]string // not exposed
	erasureWriter *io.PipeWriter
	isClosed      <-chan bool
}

// newErasureWriter - get a new writer
func newErasureWriter(writers []Writer) ObjectWriter {
	r, w := io.Pipe()
	isClosed := make(chan bool)
	writer := erasureWriter{
		writers:       writers,
		metadata:      make(map[string]string),
		erasureWriter: w,
		isClosed:      isClosed,
	}
	go erasureGoroutine(r, writer, isClosed)
	return writer
}

func erasureGoroutine(r *io.PipeReader, eWriter erasureWriter, isClosed chan<- bool) {
	chunks := split.Stream(r, 10*1024*1024)
	params, _ := erasure.ParseEncoderParams(8, 8, erasure.Cauchy)
	encoder := erasure.NewEncoder(params)
	chunkCount := 0
	totalLength := 0
	summer := md5.New()
	for chunk := range chunks {
		if chunk.Err == nil {
			totalLength = totalLength + len(chunk.Data)
			encodedBlocks, _ := encoder.Encode(chunk.Data)
			summer.Write(chunk.Data)
			for blockIndex, block := range encodedBlocks {
				io.Copy(eWriter.writers[blockIndex], bytes.NewBuffer(block))
			}
		}
		chunkCount = chunkCount + 1
	}
	dataMd5sum := summer.Sum(nil)
	metadata := make(map[string]string)
	metadata["blockSize"] = strconv.Itoa(10 * 1024 * 1024)
	metadata["chunkCount"] = strconv.Itoa(chunkCount)
	metadata["created"] = time.Now().Format(time.RFC3339Nano)
	metadata["erasureK"] = "8"
	metadata["erasureM"] = "8"
	metadata["erasureTechnique"] = "Cauchy"
	metadata["md5"] = hex.EncodeToString(dataMd5sum)
	metadata["size"] = strconv.Itoa(totalLength)
	for _, nodeWriter := range eWriter.writers {
		if nodeWriter != nil {
			nodeWriter.SetMetadata(eWriter.metadata)
			nodeWriter.SetDonutMetadata(metadata)
			nodeWriter.Close()
		}
	}
	isClosed <- true
}

func (eWriter erasureWriter) Write(data []byte) (int, error) {
	io.Copy(eWriter.erasureWriter, bytes.NewBuffer(data))
	return len(data), nil
}

func (eWriter erasureWriter) Close() error {
	eWriter.erasureWriter.Close()
	<-eWriter.isClosed
	return nil
}

func (eWriter erasureWriter) CloseWithError(err error) error {
	for _, writer := range eWriter.writers {
		if writer != nil {
			writer.CloseWithError(err)
		}
	}
	return nil
}

func (eWriter erasureWriter) SetMetadata(metadata map[string]string) error {
	for k := range metadata {
		if strings.HasPrefix(k, "sys.") {
			return errors.New("Invalid key '" + k + "', cannot start with sys.'")
		}
	}
	for k := range eWriter.metadata {
		delete(eWriter.metadata, k)
	}
	for k, v := range metadata {
		eWriter.metadata[k] = v
	}
	return nil
}

func (eWriter erasureWriter) GetMetadata() (map[string]string, error) {
	metadata := make(map[string]string)
	for k, v := range eWriter.metadata {
		metadata[k] = v
	}
	return metadata, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package slice

import (
	"reflect"
	"testing"
)

func TestCopyStrings(t *testing.T) {
	var src1 []string
	dest1 := CopyStrings(src1)

	if !reflect.DeepEqual(src1, dest1) {
		t.Errorf("%v and %v are not equal", src1, dest1)
	}

	src2 := []string{}
	dest2 := CopyStrings(src2)

	if !reflect.DeepEqual(src2, dest2) {
		t.Errorf("%v and %v are not equal", src2, dest2)
	}

	src3 := []string{"a", "c", "b"}
	dest3 := CopyStrings(src3)

	if !reflect.DeepEqual(src3, dest3) {
		t.Errorf("%v and %v are not equal", src3, dest3)
	}

	src3[0] = "A"
	if reflect.DeepEqual(src3, dest3) {
		t.Errorf("CopyStrings didn't make a copy")
	}
}

func TestSortStrings(t *testing.T) {
	src := []string{"a", "c", "b"}
	dest := SortStrings(src)
	expected := []string{"a", "b", "c"}

	if !reflect.DeepEqual(dest, expected) {
		t.Errorf("SortString didn't sort the strings")
	}

	if !reflect.DeepEqual(src, expected) {
		t.Errorf("SortString didn't sort in place")
	}
}

func TestShuffleStrings(t *testing.T) {
	var src []string
	dest := ShuffleStrings(src)

	if dest != nil {
		t.Errorf("ShuffleStrings for a nil slice got a non-nil slice")
	}

	src = []string{"a", "b", "c", "d", "e", "f"}
	dest = ShuffleStrings(src)

	if len(src) != len(dest) {
		t.Errorf("Shuffled slice is wrong length, expected %v got %v", len(src), len(dest))
	}

	m := make(map[string]bool, len(dest))
	for _, s := range dest {
		m[s] = true
	}

	for _, k := range src {
		if _, exists := m[k]; !exists {
			t.Errorf("Element %v missing from shuffled slice", k)
		}
	}
}
<commit_msg>add test for slice<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package slice

import (
	"reflect"
	"testing"
)

func TestCopyStrings(t *testing.T) {
	var src1 []string
	dest1 := CopyStrings(src1)

	if !reflect.DeepEqual(src1, dest1) {
		t.Errorf("%v and %v are not equal", src1, dest1)
	}

	src2 := []string{}
	dest2 := CopyStrings(src2)

	if !reflect.DeepEqual(src2, dest2) {
		t.Errorf("%v and %v are not equal", src2, dest2)
	}

	src3 := []string{"a", "c", "b"}
	dest3 := CopyStrings(src3)

	if !reflect.DeepEqual(src3, dest3) {
		t.Errorf("%v and %v are not equal", src3, dest3)
	}

	src3[0] = "A"
	if reflect.DeepEqual(src3, dest3) {
		t.Errorf("CopyStrings didn't make a copy")
	}
}

func TestSortStrings(t *testing.T) {
	src := []string{"a", "c", "b"}
	dest := SortStrings(src)
	expected := []string{"a", "b", "c"}

	if !reflect.DeepEqual(dest, expected) {
		t.Errorf("SortString didn't sort the strings")
	}

	if !reflect.DeepEqual(src, expected) {
		t.Errorf("SortString didn't sort in place")
	}
}

func TestShuffleStrings(t *testing.T) {
	var src []string
	dest := ShuffleStrings(src)

	if dest != nil {
		t.Errorf("ShuffleStrings for a nil slice got a non-nil slice")
	}

	src = []string{"a", "b", "c", "d", "e", "f"}
	dest = ShuffleStrings(src)

	if len(src) != len(dest) {
		t.Errorf("Shuffled slice is wrong length, expected %v got %v", len(src), len(dest))
	}

	m := make(map[string]bool, len(dest))
	for _, s := range dest {
		m[s] = true
	}

	for _, k := range src {
		if _, exists := m[k]; !exists {
			t.Errorf("Element %v missing from shuffled slice", k)
		}
	}
}

func TestContainsString(t *testing.T) {
	src := []string{"aa", "bb", "cc"}
	if !ContainsString(src, "bb", nil) {
		t.Errorf("ContainsString didn't find the string as expected")
	}

	modifier := func(s string) string {
		if s == "cc" {
			return "ee"
		}
		return s
	}
	if !ContainsString(src, "ee", modifier) {
		t.Errorf("ContainsString didn't find the string by modifier")
	}
}
<|endoftext|>


<commit_before>package reads

import (
	"github.com/influxdata/influxdb/storage"
	"github.com/influxdata/influxdb/storage/reads/datatypes"
)

type StringIteratorStream interface {
	Send(*datatypes.StringValuesResponse) error
}

type StringIteratorWriter struct {
	stream StringIteratorStream
	res    *datatypes.StringValuesResponse
	err    error

	sz int // estimated size in bytes for pending write
}

func NewStringIteratorWriter(stream StringIteratorStream) *StringIteratorWriter {
	siw := &StringIteratorWriter{
		stream: stream,
		res: &datatypes.StringValuesResponse{
			Values: nil,
		},
	}

	return siw
}

func (w *StringIteratorWriter) Err() error {
	return w.err
}

func (w *StringIteratorWriter) WriteStringIterator(si storage.StringIterator) error {
	for si.Next() {
		v := si.Value()
		if v == "" {
			// no value, no biggie
			continue
		}

		w.res.Values = append(w.res.Values, []byte(v))
		w.sz += len(v)
	}

	return nil
}

func (w *StringIteratorWriter) Flush() {
	if w.err != nil || w.sz == 0 {
		return
	}

	w.sz = 0

	if w.err = w.stream.Send(w.res); w.err != nil {
		return
	}

	for i := range w.res.Values {
		w.res.Values[i] = nil
	}
	w.res.Values = w.res.Values[:0]
}
<commit_msg>feat(storage): Add StringIteratorWriter.WrittenN() (#13456)<commit_after>package reads

import (
	"github.com/influxdata/influxdb/storage"
	"github.com/influxdata/influxdb/storage/reads/datatypes"
)

type StringIteratorStream interface {
	Send(*datatypes.StringValuesResponse) error
}

type StringIteratorWriter struct {
	stream StringIteratorStream
	res    *datatypes.StringValuesResponse
	err    error

	sz int // estimated size in bytes for pending write
	vc int // total value count
}

func NewStringIteratorWriter(stream StringIteratorStream) *StringIteratorWriter {
	siw := &StringIteratorWriter{
		stream: stream,
		res: &datatypes.StringValuesResponse{
			Values: nil,
		},
	}

	return siw
}

func (w *StringIteratorWriter) Err() error {
	return w.err
}

func (w *StringIteratorWriter) WrittenN() int {
	return w.vc
}

func (w *StringIteratorWriter) WriteStringIterator(si storage.StringIterator) error {
	for si.Next() {
		v := si.Value()
		if v == "" {
			// no value, no biggie
			continue
		}

		w.res.Values = append(w.res.Values, []byte(v))
		w.sz += len(v)
		w.vc++
	}

	return nil
}

func (w *StringIteratorWriter) Flush() {
	if w.err != nil || w.sz == 0 {
		return
	}

	w.sz, w.vc = 0, 0

	if w.err = w.stream.Send(w.res); w.err != nil {
		return
	}

	for i := range w.res.Values {
		w.res.Values[i] = nil
	}
	w.res.Values = w.res.Values[:0]
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package slice

import (
	"reflect"
	"testing"
)

func TestCopyStrings(t *testing.T) {
	var src1 []string
	dest1 := CopyStrings(src1)

	if !reflect.DeepEqual(src1, dest1) {
		t.Errorf("%v and %v are not equal", src1, dest1)
	}

	src2 := []string{}
	dest2 := CopyStrings(src2)

	if !reflect.DeepEqual(src2, dest2) {
		t.Errorf("%v and %v are not equal", src2, dest2)
	}

	src3 := []string{"a", "c", "b"}
	dest3 := CopyStrings(src3)

	if !reflect.DeepEqual(src3, dest3) {
		t.Errorf("%v and %v are not equal", src3, dest3)
	}

	src3[0] = "A"
	if reflect.DeepEqual(src3, dest3) {
		t.Errorf("CopyStrings didn't make a copy")
	}
}

func TestSortStrings(t *testing.T) {
	src := []string{"a", "c", "b"}
	dest := SortStrings(src)
	expected := []string{"a", "b", "c"}

	if !reflect.DeepEqual(dest, expected) {
		t.Errorf("SortString didn't sort the strings")
	}

	if !reflect.DeepEqual(src, expected) {
		t.Errorf("SortString didn't sort in place")
	}
}

func TestContainsString(t *testing.T) {
	src := []string{"aa", "bb", "cc"}
	if !ContainsString(src, "bb", nil) {
		t.Errorf("ContainsString didn't find the string as expected")
	}

	modifier := func(s string) string {
		if s == "cc" {
			return "ee"
		}
		return s
	}
	if !ContainsString(src, "ee", modifier) {
		t.Errorf("ContainsString didn't find the string by modifier")
	}
}

func TestRemoveString(t *testing.T) {
	modifier := func(s string) string {
		if s == "ab" {
			return "ee"
		}
		return s
	}
	tests := []struct {
		testName string
		input    []string
		remove   string
		modifier func(s string) string
		want     []string
	}{
		{
			testName: "Nil input slice",
			input:    nil,
			remove:   "",
			modifier: nil,
			want:     nil,
		},
		{
			testName: "Slice doesn't contain the string",
			input:    []string{"a", "ab", "cdef"},
			remove:   "NotPresentInSlice",
			modifier: nil,
			want:     []string{"a", "ab", "cdef"},
		},
		{
			testName: "All strings removed, result is nil",
			input:    []string{"a"},
			remove:   "a",
			modifier: nil,
			want:     nil,
		},
		{
			testName: "No modifier func, one string removed",
			input:    []string{"a", "ab", "cdef"},
			remove:   "ab",
			modifier: nil,
			want:     []string{"a", "cdef"},
		},
		{
			testName: "No modifier func, all(three) strings removed",
			input:    []string{"ab", "a", "ab", "cdef", "ab"},
			remove:   "ab",
			modifier: nil,
			want:     []string{"a", "cdef"},
		},
		{
			testName: "Removed both the string and the modifier func result",
			input:    []string{"a", "cd", "ab", "ee"},
			remove:   "ee",
			modifier: modifier,
			want:     []string{"a", "cd"},
		},
	}
	for _, tt := range tests {
		if got := RemoveString(tt.input, tt.remove, tt.modifier); !reflect.DeepEqual(got, tt.want) {
			t.Errorf("%v: RemoveString(%v, %q, %T) = %v WANT %v", tt.testName, tt.input, tt.remove, tt.modifier, got, tt.want)
		}
	}
}
<commit_msg>add unit test coverage for pkg/util/slice<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package slice

import (
	"reflect"
	"testing"
)

func TestCopyStrings(t *testing.T) {
	var src1 []string
	dest1 := CopyStrings(src1)

	if !reflect.DeepEqual(src1, dest1) {
		t.Errorf("%v and %v are not equal", src1, dest1)
	}

	src2 := []string{}
	dest2 := CopyStrings(src2)

	if !reflect.DeepEqual(src2, dest2) {
		t.Errorf("%v and %v are not equal", src2, dest2)
	}

	src3 := []string{"a", "c", "b"}
	dest3 := CopyStrings(src3)

	if !reflect.DeepEqual(src3, dest3) {
		t.Errorf("%v and %v are not equal", src3, dest3)
	}

	src3[0] = "A"
	if reflect.DeepEqual(src3, dest3) {
		t.Errorf("CopyStrings didn't make a copy")
	}
}

func TestSortStrings(t *testing.T) {
	src := []string{"a", "c", "b"}
	dest := SortStrings(src)
	expected := []string{"a", "b", "c"}

	if !reflect.DeepEqual(dest, expected) {
		t.Errorf("SortString didn't sort the strings")
	}

	if !reflect.DeepEqual(src, expected) {
		t.Errorf("SortString didn't sort in place")
	}
}

func TestContainsString(t *testing.T) {
	src := []string{"aa", "bb", "cc"}
	if !ContainsString(src, "bb", nil) {
		t.Errorf("ContainsString didn't find the string as expected")
	}

	modifier := func(s string) string {
		if s == "cc" {
			return "ee"
		}
		return s
	}
	if !ContainsString(src, "ee", modifier) {
		t.Errorf("ContainsString didn't find the string by modifier")
	}

	src = make([]string, 0)
	if ContainsString(src, "", nil) {
		t.Errorf("The result returned is not the expected result")
	}
}

func TestRemoveString(t *testing.T) {
	modifier := func(s string) string {
		if s == "ab" {
			return "ee"
		}
		return s
	}
	tests := []struct {
		testName string
		input    []string
		remove   string
		modifier func(s string) string
		want     []string
	}{
		{
			testName: "Nil input slice",
			input:    nil,
			remove:   "",
			modifier: nil,
			want:     nil,
		},
		{
			testName: "Slice doesn't contain the string",
			input:    []string{"a", "ab", "cdef"},
			remove:   "NotPresentInSlice",
			modifier: nil,
			want:     []string{"a", "ab", "cdef"},
		},
		{
			testName: "All strings removed, result is nil",
			input:    []string{"a"},
			remove:   "a",
			modifier: nil,
			want:     nil,
		},
		{
			testName: "No modifier func, one string removed",
			input:    []string{"a", "ab", "cdef"},
			remove:   "ab",
			modifier: nil,
			want:     []string{"a", "cdef"},
		},
		{
			testName: "No modifier func, all(three) strings removed",
			input:    []string{"ab", "a", "ab", "cdef", "ab"},
			remove:   "ab",
			modifier: nil,
			want:     []string{"a", "cdef"},
		},
		{
			testName: "Removed both the string and the modifier func result",
			input:    []string{"a", "cd", "ab", "ee"},
			remove:   "ee",
			modifier: modifier,
			want:     []string{"a", "cd"},
		},
	}
	for _, tt := range tests {
		if got := RemoveString(tt.input, tt.remove, tt.modifier); !reflect.DeepEqual(got, tt.want) {
			t.Errorf("%v: RemoveString(%v, %q, %T) = %v WANT %v", tt.testName, tt.input, tt.remove, tt.modifier, got, tt.want)
		}
	}
}
<|endoftext|>


<commit_before>package shufn_test

import (
	"fmt"
	"math/rand"
	"sort"

	"github.com/icio/shufn"
)

func ExampleNew() {
	// it produces values in the range 0-100 inclusive. The start parameter is
	// passed through to New verbatim. When New receives start=0 it replaces it
	// with a random value.
	rand.Seed(0)
	it := shufn.New(shufn.Calc(1, 3, 0))
	for {
		i, more := it.Next()
		if !more {
			break
		}

		fmt.Println(i)
	}
	// Output: 1
	// 3
	// 2
}

func ExampleNew_parallel() {
	work := make(chan int)
	done := make(chan int)

	// Distribute the random values via a channel.
	go func(it shufn.Iter) {
		for {
			i, more := it.Next()
			if !more {
				break
			}
			work <- int(i)
		}
		close(work)
	}(shufn.New(shufn.Calc(0, 10, 0)))

	// Create two workers to square each integer on the work queue and write to done.
	W, workers := 2, make(chan bool)
	for w := 0; w < W; w++ {
		go func(work chan int, done chan int) {
			for n := range work {
				done <- n * n
			}
			workers <- true
		}(work, done)
	}

	// Close the done channel after the workers have completed.
	go func() {
		for w := 0; w < W; w++ {
			<-workers
		}
		close(done)
	}()

	// Collect and sort the results.
	squares := make(chan []int)
	go func() {
		sqs := make([]int, 0, 10)
		for sq := range done {
			sqs = append(sqs, sq)
		}
		sort.Ints(sqs)
		squares <- sqs
	}()

	fmt.Println(<-squares)
	// Output: [0 1 4 9 16 25 36 49 64 81 100]
}
<commit_msg>Remove parallelism example<commit_after>package shufn_test

import (
	"fmt"
	"math/rand"
	"sort"

	"github.com/icio/shufn"
)

func ExampleNew() {
	// it produces values in the range 0-100 inclusive. The start parameter is
	// passed through to New verbatim. When New receives start=0 it replaces it
	// with a random value.
	rand.Seed(0)
	it := shufn.New(shufn.Calc(1, 3, 0))
	for {
		i, more := it.Next()
		if !more {
			break
		}

		fmt.Println(i)
	}
	// Output: 1
	// 3
	// 2
}
<|endoftext|>


<commit_before>package libcoap

/*
#cgo LDFLAGS: -lcoap-2-openssl
#include <coap2/coap.h>
*/
import "C"
import "encoding/binary"
import "bytes"

type OptionKey uint16

type Option struct {
    Key   OptionKey
    Value []byte
}

const (
    OptionIfMatch       OptionKey = C.COAP_OPTION_IF_MATCH
    OptionUriHost       OptionKey = C.COAP_OPTION_URI_HOST
    OptionEtag          OptionKey = C.COAP_OPTION_ETAG
    OptionIfNoneMatch   OptionKey = C.COAP_OPTION_IF_NONE_MATCH
    OptionObserve       OptionKey = C.COAP_OPTION_OBSERVE
    OptionUriPort       OptionKey = C.COAP_OPTION_URI_PORT
    OptionLocationPath  OptionKey = C.COAP_OPTION_LOCATION_PATH
    OptionUriPath       OptionKey = C.COAP_OPTION_URI_PATH
    OptionContentFormat OptionKey = C.COAP_OPTION_CONTENT_FORMAT
    OptionContentType   OptionKey = C.COAP_OPTION_CONTENT_TYPE
    OptionMaxage        OptionKey = C.COAP_OPTION_MAXAGE
    OptionUriQuery      OptionKey = C.COAP_OPTION_URI_QUERY
    OptionAccept        OptionKey = C.COAP_OPTION_ACCEPT
    OptionLocationQuery OptionKey = C.COAP_OPTION_LOCATION_QUERY
    OptionProxyUri      OptionKey = C.COAP_OPTION_PROXY_URI
    OptionProxyScheme   OptionKey = C.COAP_OPTION_PROXY_SCHEME
    OptionSize1         OptionKey = C.COAP_OPTION_SIZE1
    OptionBlock2        OptionKey = C.COAP_OPTION_BLOCK2
    OptionSize2         OptionKey = C.COAP_OPTION_SIZE2
)

func (key OptionKey) String(value string) Option {
    return Option{ key, []byte(value) }
}

func (opt Option) String() string {
    return string(opt.Value)
}

func (key OptionKey) Uint(value interface{}) (Option, error) {
	buf := new(bytes.Buffer)
	err := binary.Write(buf, binary.BigEndian, value)
	if err != nil {
		return Option{}, err
    }
	return Option{ key, buf.Bytes() }, nil
}

func (opt Option) Uint() (res uint32, err error) {
    buf := bytes.NewReader(opt.Value)

    switch len(opt.Value) {
        case 0:
            // The number 0 is represented with an empty option value.
            res = uint32(0)
        case 1:
            var temp uint8
            err = binary.Read(buf, binary.BigEndian, &temp)
            res = uint32(temp)
        case 2, 3:
            var temp uint16
            err = binary.Read(buf, binary.BigEndian, &temp)
            res = uint32(temp)
        case 4:
            var temp uint32
            err = binary.Read(buf, binary.BigEndian, &temp)
            res = uint32(temp)
        default:
            var temp uint32
            err = binary.Read(buf, binary.BigEndian, &temp)
            res = uint32(temp)
    }
	if err != nil {
		return 0, err
	}
	return
}<commit_msg>Fix bit length for observe option issue #329<commit_after>package libcoap

/*
#cgo LDFLAGS: -lcoap-2-openssl
#include <coap2/coap.h>
*/
import "C"
import "encoding/binary"
import "bytes"

type OptionKey uint16

type Option struct {
    Key   OptionKey
    Value []byte
}

const (
    OptionIfMatch       OptionKey = C.COAP_OPTION_IF_MATCH
    OptionUriHost       OptionKey = C.COAP_OPTION_URI_HOST
    OptionEtag          OptionKey = C.COAP_OPTION_ETAG
    OptionIfNoneMatch   OptionKey = C.COAP_OPTION_IF_NONE_MATCH
    OptionObserve       OptionKey = C.COAP_OPTION_OBSERVE
    OptionUriPort       OptionKey = C.COAP_OPTION_URI_PORT
    OptionLocationPath  OptionKey = C.COAP_OPTION_LOCATION_PATH
    OptionUriPath       OptionKey = C.COAP_OPTION_URI_PATH
    OptionContentFormat OptionKey = C.COAP_OPTION_CONTENT_FORMAT
    OptionContentType   OptionKey = C.COAP_OPTION_CONTENT_TYPE
    OptionMaxage        OptionKey = C.COAP_OPTION_MAXAGE
    OptionUriQuery      OptionKey = C.COAP_OPTION_URI_QUERY
    OptionAccept        OptionKey = C.COAP_OPTION_ACCEPT
    OptionLocationQuery OptionKey = C.COAP_OPTION_LOCATION_QUERY
    OptionProxyUri      OptionKey = C.COAP_OPTION_PROXY_URI
    OptionProxyScheme   OptionKey = C.COAP_OPTION_PROXY_SCHEME
    OptionSize1         OptionKey = C.COAP_OPTION_SIZE1
    OptionBlock2        OptionKey = C.COAP_OPTION_BLOCK2
    OptionSize2         OptionKey = C.COAP_OPTION_SIZE2
)

func (key OptionKey) String(value string) Option {
    return Option{ key, []byte(value) }
}

func (opt Option) String() string {
    return string(opt.Value)
}

func (key OptionKey) Uint(value interface{}) (Option, error) {
	buf := new(bytes.Buffer)
	err := binary.Write(buf, binary.BigEndian, value)
	if err != nil {
		return Option{}, err
    }
	return Option{ key, buf.Bytes() }, nil
}

func (opt Option) Uint() (res uint32, err error) {
    buf := bytes.NewReader(opt.Value)

    switch len(opt.Value) {
        case 0:
            // The number 0 is represented with an empty option value.
            res = uint32(0)
        case 1:
            var temp uint8
            err = binary.Read(buf, binary.BigEndian, &temp)
            res = uint32(temp)
        case 2:
            var temp uint16
            err = binary.Read(buf, binary.BigEndian, &temp)
            res = uint32(temp)
        case 3:
            res = uint32(uint(opt.Value[0]) | uint(opt.Value[1])<<8 | uint(opt.Value[2])<<16)
        case 4:
            var temp uint32
            err = binary.Read(buf, binary.BigEndian, &temp)
            res = uint32(temp)
        default:
            var temp uint32
            err = binary.Read(buf, binary.BigEndian, &temp)
            res = uint32(temp)
    }
	if err != nil {
		return 0, err
	}
	return
}<|endoftext|>


<commit_before>package byteslice

import "fmt"

func ExampleReverse() {
	data := []byte{0x55, 0xDA, 0xBA}

	fmt.Printf("%x\n", Reverse(data))
	// Output: bada55
}

func ExampleLPad() {
	data := []byte{0x55, 0xDA, 0xBA}

	fmt.Printf("%x\n", LPad(data, 5, 0x22))
	// Output: 222255daba
}
<commit_msg>[#46] Added examples for Toggle<commit_after>package byteslice

import "fmt"

func ExampleReverse() {
	data := []byte{0x55, 0xDA, 0xBA}

	fmt.Printf("%x\n", Reverse(data))
	// Output: bada55
}

func ExampleLPad() {
	data := []byte{0x55, 0xDA, 0xBA}

	fmt.Printf("%x\n", LPad(data, 5, 0x22))
	// Output: 222255daba
}

func ExampleToggle() {
	data := []byte{0xbb, 0xdb, 0x54}
	toggle := []byte{0x01, 0x01, 0x01}

	output, _ := Toggle(data, toggle)
	fmt.Printf("%x\n", output)
	// Output: bada55
}

func ExampleToggle_simple() {
	data := []byte{0x00, 0x01, 0x00}
	toggle := []byte{0x01, 0x00, 0x01}

	output, _ := Toggle(data, toggle)
	fmt.Printf("%x\n", output)
	// Output: 010101
}
<|endoftext|>


<commit_before>package helium

import (
	"bytes"
	"errors"
	"io"
	"strings"

	"github.com/lestrrat/helium/internal/debug"
)

func CreateDocument() *Document {
	return NewDocument("1.0", "", StandaloneImplicitNo)
}

func NewDocument(version, encoding string, standalone DocumentStandaloneType) *Document {
	doc := &Document{
		encoding:   encoding,
		standalone: standalone,
		version:    version,
	}

	doc.etype = DocumentNode
	doc.name = "(document)"
	return doc
}

func (d Document) XMLString() (string, error) {
	out := bytes.Buffer{}
	if err := d.XML(&out); err != nil {
		return "", err
	}
	return out.String(), nil
}

func (d *Document) XML(out io.Writer) error {
	return (&Dumper{}).DumpDoc(out, d)
}

func (d *Document) AddChild(cur Node) error {
	return addChild(d, cur)
}

func (d *Document) AddContent(b []byte) error {
	return addContent(d, b)
}

func (d *Document) AddSibling(n Node) error {
	return errors.New("can't add sibling to a document")
}

func (d *Document) SetTreeDoc(doc *Document) {
	setTreeDoc(d, doc)
}

func (d *Document) Encoding() string {
	// In order to differentiate between a document with explicit
	// encoding in the XML declaration and one without, the XML dump
	// routine must check for d.encoding == "", and not Encoding()
	if enc := d.encoding; enc != "" {
		return d.encoding
	}
	return "utf8"
}

func (d *Document) Standalone() DocumentStandaloneType {
	return d.standalone
}

func (d *Document) Version() string {
	return d.version
}

func (d *Document) IntSubset() *DTD {
	return d.intSubset
}

func (d *Document) ExtSubset() *DTD {
	return d.extSubset
}

func (d *Document) Replace(n Node) {
	panic("d.Replace does not make sense")
}

func (d *Document) SetDocumentElement(root Node) error {
	if d == nil {
		// what are you trying to do?
		return nil
	}

	if root == nil || root.Type() == NamespaceDeclNode {
		return nil
	}

	root.SetParent(d)
	var old Node
	for old = d.firstChild; old != nil; old = old.NextSibling() {
		if old.Type() == ElementNode {
			break
		}
	}

	if old == nil {
		d.AddChild(root)
	} else {
		old.Replace(root)
	}
	return nil
}

func (d *Document) CreateReference(name string) (*EntityRef, error) {
	if debug.Enabled {
		g := debug.IPrintf("START document.CreateReference '%s'", name)
		defer g.IRelease("END document.CreateReference")
	}
	n, err := d.CreateCharRef(name)
	if err != nil {
		return nil, err
	}

	ent, ok := d.GetEntity(n.name)
	if ok {
		n.content = []byte(ent.content)
		// Original code says:
		// The parent pointer in entity is a DTD pointer and thus is NOT
		// updated.  Not sure if this is 100% correct.
		n.setFirstChild(ent)
		n.setLastChild(ent)
	}

	return n, nil
}

func (d *Document) CreateAttribute(name, value string, ns *Namespace) (*Attribute, error) {
	attr := newAttribute(name, ns)
	if value != "" {
		n, err := d.stringToNodeList(value)
		if err != nil {
			return nil, err
		}
		attr.setFirstChild(n)
		for n != nil {
			n.SetParent(attr)
			x := n.NextSibling()
			if x == nil {
				n.setLastChild(x)
			}
			n = x
		}
	}
	return attr, nil
}

func (d *Document) CreateNamespace(prefix, uri string) (*Namespace, error) {
	ns := newNamespace(prefix, uri)
	ns.context = d
	return ns, nil
}

func (d *Document) CreatePI(target, data string) (*ProcessingInstruction, error) {
	return &ProcessingInstruction{
		target: target,
		data:   data,
	}, nil
}

func (d *Document) CreateDTD() (*DTD, error) {
	dtd := newDTD()
	dtd.doc = d
	return dtd, nil
}

func (d *Document) CreateElement(name string) (*Element, error) {
	e := newElement(name)
	e.doc = d
	return e, nil
}

func (d *Document) CreateText(value []byte) (*Text, error) {
	e := newText(value)
	e.doc = d
	return e, nil
}

func (d *Document) CreateComment(value []byte) (*Comment, error) {
	e := newComment(value)
	e.doc = d
	return e, nil
}

func (d *Document) CreateElementContent(name string, etype ElementContentType) (*ElementContent, error) {
	e, err := newElementContent(name, etype)
	if err != nil {
		return nil, err
	}
	return e, nil
}

func (d *Document) GetEntity(name string) (*Entity, bool) {
	if ints := d.intSubset; ints != nil {
		return ints.LookupEntity(name)
	}

	if exts := d.extSubset; exts != nil {
		return exts.LookupEntity(name)
	}

	return nil, false
}

func (d *Document) GetParameterEntity(name string) (*Entity, bool) {
	if ints := d.intSubset; ints != nil {
		return ints.LookupParameterEntity(name)
	}

	if exts := d.extSubset; exts != nil {
		return exts.LookupParameterEntity(name)
	}

	return nil, false
}

func (d *Document) IsMixedElement(name string) (bool, error) {
	if d.intSubset == nil {
		return false, errors.New("element declaration not found")
	}

	edecl, ok := d.intSubset.GetElementDesc(name)
	if !ok {
		return false, errors.New("element declaration not found")
	}

	switch edecl.decltype {
	case UndefinedElementType:
		return false, errors.New("element declaration not found")
	case ElementElementType:
		return false, nil
	case EmptyElementType, AnyElementType, MixedElementType:
		/*
		 * return 1 for EMPTY since we want VC error to pop up
		 * on <empty>     </empty> for example
		 */
		return true, nil
	}
	return true, nil
}

/*
 * @doc:  the document
 * @value:  the value of the attribute
 *
 * Parse the value string and build the node list associated. Should
 * produce a flat tree with only TEXTs and ENTITY_REFs.
 * Returns a pointer to the first child
 */
func (d *Document) stringToNodeList(value string) (Node, error) {
	rdr := strings.NewReader(value)
	buf := bytes.Buffer{}
	var ret Node
	var last Node
	var charval int32
	for rdr.Len() > 0 {
		r, _, err := rdr.ReadRune()
		if err != nil {
			return nil, err
		}

		// if this is not any sort of an entity , just go go go
		if r != '&' {
			buf.WriteRune(r)
			continue
		}

		// well, at least the first rune sure looks like an entity, see what
		// else we have.
		r, _, err = rdr.ReadRune()
		if err != nil {
			return nil, err
		}

		if r == '#' {
			r2, _, err := rdr.ReadRune()
			if err != nil {
				return nil, err
			}

			var accumulator func(int32, rune) (int32, error)
			if r2 == 'x' {
				accumulator = accumulateHexCharRef
			} else {
				rdr.UnreadRune()
				accumulator = accumulateDecimalCharRef
			}
			for {
				r, _, err = rdr.ReadRune()
				if err != nil {
					return nil, err
				}
				if r == ';' {
					break
				}
				charval, err = accumulator(charval, r)
				if err != nil {
					return nil, err
				}
			}
		} else {
			rdr.UnreadRune()
			entbuf := bytes.Buffer{}
			for rdr.Len() > 0 {
				r, _, err = rdr.ReadRune()
				if err != nil {
					return nil, err
				}
				if r == ';' {
					break
				}
				entbuf.WriteRune(r)
			}

			if rdr.Len() == 0 {
				return nil, errors.New("entity was unterminated (could not find terminating semicolon)")
			}

			val := entbuf.String()
			ent, ok := d.GetEntity(val)

			// XXX I *believe* libxml2 SKIPS entities that it can't resolve
			// at this point?
			if ok && ent.EntityType() == int(InternalPredefinedEntity) {
				buf.Write(ent.Content())
			} else {
				// flush the buffer so far
				if buf.Len() > 0 {
					node, err := d.CreateText(buf.Bytes())
					if err != nil {
						return nil, err
					}

					if last == nil {
						last = node
						ret = node
					} else {
						last.AddSibling(node)
						last = node
					}
				}

				// create a new REFERENCE_REF node
				node, err := d.CreateReference(val)
				if err != nil {
					return nil, err
				}

				// no children
				if ok && ent.FirstChild() == nil {
					// XXX WTF am I doing here...?
					refchildren, err := d.stringToNodeList(string(node.Content()))
					if err != nil {
						return nil, err
					}
					ent.setFirstChild(refchildren)
					for n := refchildren; n != nil; {
						n.SetParent(ent)
						x := n.NextSibling()
						if x == nil {
							ent.setLastChild(n)
						}
						n = x
					}
				}

				if last == nil {
					last = node
					ret = node
				} else {
					last.AddSibling(node)
					last = node
				}
			}
		}

		if charval != 0 {
			buf.WriteRune(rune(charval))
			charval = 0
		}
	}

	if buf.Len() > 0 {
		n, err := d.CreateText(buf.Bytes())
		if err != nil {
			return nil, err
		}

		if last == nil {
			ret = n
		} else {
			last.AddSibling(n)
		}
	}

	return ret, nil
}

func (d *Document) CreateCharRef(name string) (*EntityRef, error) {
	if debug.Enabled {
		g := debug.IPrintf("START document.CreateCharRef '%s'", name)
		defer g.IRelease("END document.CreateCharRef")
	}

	if name == "" {
		return nil, errors.New("empty name")
	}

	n := newEntityRef()
	n.doc = d
	if name[0] != '&' {
		n.name = name
	} else {
		// the name should be everything but '&' and ';'
		if name[len(name)-1] == ';' {
			n.name = name[1 : len(name)-1]
		} else {
			n.name = name[1:]
		}
	}
	return n, nil
}

<commit_msg>correctly parse the end of entity (';')<commit_after>package helium

import (
	"bytes"
	"errors"
	"io"
	"strings"

	"github.com/lestrrat/helium/internal/debug"
)

func CreateDocument() *Document {
	return NewDocument("1.0", "", StandaloneImplicitNo)
}

func NewDocument(version, encoding string, standalone DocumentStandaloneType) *Document {
	doc := &Document{
		encoding:   encoding,
		standalone: standalone,
		version:    version,
	}

	doc.etype = DocumentNode
	doc.name = "(document)"
	return doc
}

func (d Document) XMLString() (string, error) {
	out := bytes.Buffer{}
	if err := d.XML(&out); err != nil {
		return "", err
	}
	return out.String(), nil
}

func (d *Document) XML(out io.Writer) error {
	return (&Dumper{}).DumpDoc(out, d)
}

func (d *Document) AddChild(cur Node) error {
	return addChild(d, cur)
}

func (d *Document) AddContent(b []byte) error {
	return addContent(d, b)
}

func (d *Document) AddSibling(n Node) error {
	return errors.New("can't add sibling to a document")
}

func (d *Document) SetTreeDoc(doc *Document) {
	setTreeDoc(d, doc)
}

func (d *Document) Encoding() string {
	// In order to differentiate between a document with explicit
	// encoding in the XML declaration and one without, the XML dump
	// routine must check for d.encoding == "", and not Encoding()
	if enc := d.encoding; enc != "" {
		return d.encoding
	}
	return "utf8"
}

func (d *Document) Standalone() DocumentStandaloneType {
	return d.standalone
}

func (d *Document) Version() string {
	return d.version
}

func (d *Document) IntSubset() *DTD {
	return d.intSubset
}

func (d *Document) ExtSubset() *DTD {
	return d.extSubset
}

func (d *Document) Replace(n Node) {
	panic("d.Replace does not make sense")
}

func (d *Document) SetDocumentElement(root Node) error {
	if d == nil {
		// what are you trying to do?
		return nil
	}

	if root == nil || root.Type() == NamespaceDeclNode {
		return nil
	}

	root.SetParent(d)
	var old Node
	for old = d.firstChild; old != nil; old = old.NextSibling() {
		if old.Type() == ElementNode {
			break
		}
	}

	if old == nil {
		d.AddChild(root)
	} else {
		old.Replace(root)
	}
	return nil
}

func (d *Document) CreateReference(name string) (*EntityRef, error) {
	if debug.Enabled {
		g := debug.IPrintf("START document.CreateReference '%s'", name)
		defer g.IRelease("END document.CreateReference")
	}
	n, err := d.CreateCharRef(name)
	if err != nil {
		return nil, err
	}

	ent, ok := d.GetEntity(n.name)
	if ok {
		n.content = []byte(ent.content)
		// Original code says:
		// The parent pointer in entity is a DTD pointer and thus is NOT
		// updated.  Not sure if this is 100% correct.
		n.setFirstChild(ent)
		n.setLastChild(ent)
	}

	return n, nil
}

func (d *Document) CreateAttribute(name, value string, ns *Namespace) (*Attribute, error) {
	attr := newAttribute(name, ns)
	if value != "" {
		n, err := d.stringToNodeList(value)
		if err != nil {
			return nil, err
		}
		attr.setFirstChild(n)
		for n != nil {
			n.SetParent(attr)
			x := n.NextSibling()
			if x == nil {
				n.setLastChild(x)
			}
			n = x
		}
	}
	return attr, nil
}

func (d *Document) CreateNamespace(prefix, uri string) (*Namespace, error) {
	ns := newNamespace(prefix, uri)
	ns.context = d
	return ns, nil
}

func (d *Document) CreatePI(target, data string) (*ProcessingInstruction, error) {
	return &ProcessingInstruction{
		target: target,
		data:   data,
	}, nil
}

func (d *Document) CreateDTD() (*DTD, error) {
	dtd := newDTD()
	dtd.doc = d
	return dtd, nil
}

func (d *Document) CreateElement(name string) (*Element, error) {
	e := newElement(name)
	e.doc = d
	return e, nil
}

func (d *Document) CreateText(value []byte) (*Text, error) {
	e := newText(value)
	e.doc = d
	return e, nil
}

func (d *Document) CreateComment(value []byte) (*Comment, error) {
	e := newComment(value)
	e.doc = d
	return e, nil
}

func (d *Document) CreateElementContent(name string, etype ElementContentType) (*ElementContent, error) {
	e, err := newElementContent(name, etype)
	if err != nil {
		return nil, err
	}
	return e, nil
}

func (d *Document) GetEntity(name string) (*Entity, bool) {
	if ints := d.intSubset; ints != nil {
		return ints.LookupEntity(name)
	}

	if exts := d.extSubset; exts != nil {
		return exts.LookupEntity(name)
	}

	return nil, false
}

func (d *Document) GetParameterEntity(name string) (*Entity, bool) {
	if ints := d.intSubset; ints != nil {
		return ints.LookupParameterEntity(name)
	}

	if exts := d.extSubset; exts != nil {
		return exts.LookupParameterEntity(name)
	}

	return nil, false
}

func (d *Document) IsMixedElement(name string) (bool, error) {
	if d.intSubset == nil {
		return false, errors.New("element declaration not found")
	}

	edecl, ok := d.intSubset.GetElementDesc(name)
	if !ok {
		return false, errors.New("element declaration not found")
	}

	switch edecl.decltype {
	case UndefinedElementType:
		return false, errors.New("element declaration not found")
	case ElementElementType:
		return false, nil
	case EmptyElementType, AnyElementType, MixedElementType:
		/*
		 * return 1 for EMPTY since we want VC error to pop up
		 * on <empty>     </empty> for example
		 */
		return true, nil
	}
	return true, nil
}

/*
 * @doc:  the document
 * @value:  the value of the attribute
 *
 * Parse the value string and build the node list associated. Should
 * produce a flat tree with only TEXTs and ENTITY_REFs.
 * Returns a pointer to the first child
 */
func (d *Document) stringToNodeList(value string) (Node, error) {
	if debug.Enabled {
		g := debug.IPrintf("START document.stringToNodeList '%s'", value)
		defer g.IRelease("END document.stringToNodeList")
	}
	rdr := strings.NewReader(value)
	buf := bytes.Buffer{}
	var ret Node
	var last Node
	var charval int32
	for rdr.Len() > 0 {
		r, _, err := rdr.ReadRune()
		if err != nil {
			return nil, err
		}

		// if this is not any sort of an entity , just go go go
		if r != '&' {
			buf.WriteRune(r)
			continue
		}

		// well, at least the first rune sure looks like an entity, see what
		// else we have.
		r, _, err = rdr.ReadRune()
		if err != nil {
			return nil, err
		}

		if r == '#' {
			r2, _, err := rdr.ReadRune()
			if err != nil {
				return nil, err
			}

			var accumulator func(int32, rune) (int32, error)
			if r2 == 'x' {
				accumulator = accumulateHexCharRef
			} else {
				rdr.UnreadRune()
				accumulator = accumulateDecimalCharRef
			}
			for {
				r, _, err = rdr.ReadRune()
				if err != nil {
					return nil, err
				}
				if r == ';' {
					break
				}
				charval, err = accumulator(charval, r)
				if err != nil {
					return nil, err
				}
			}
		} else {
			rdr.UnreadRune()
			entbuf := bytes.Buffer{}
			for rdr.Len() > 0 {
				r, _, err = rdr.ReadRune()
				if err != nil {
					return nil, err
				}
				if r == ';' {
					break
				}
				entbuf.WriteRune(r)
			}

			if r != ';' {
				return nil, errors.New("entity was unterminated (could not find terminating semicolon)")
			}

			val := entbuf.String()
			ent, ok := d.GetEntity(val)

			// XXX I *believe* libxml2 SKIPS entities that it can't resolve
			// at this point?
			if ok && ent.EntityType() == int(InternalPredefinedEntity) {
				buf.Write(ent.Content())
			} else {
				// flush the buffer so far
				if buf.Len() > 0 {
					node, err := d.CreateText(buf.Bytes())
					if err != nil {
						return nil, err
					}

					if last == nil {
						last = node
						ret = node
					} else {
						last.AddSibling(node)
						last = node
					}
				}

				// create a new REFERENCE_REF node
				node, err := d.CreateReference(val)
				if err != nil {
					return nil, err
				}

				// no children
				if ok && ent.FirstChild() == nil {
					// XXX WTF am I doing here...?
					refchildren, err := d.stringToNodeList(string(node.Content()))
					if err != nil {
						return nil, err
					}
					ent.setFirstChild(refchildren)
					for n := refchildren; n != nil; {
						ent.AddChild(n)
						if x := n.NextSibling(); x != nil {
							n = x
						} else {
							n = nil
						}
					}
				}

				if last == nil {
					last = node
					ret = node
				} else {
					last.AddSibling(node)
					last = node
				}
			}
		}

		if charval != 0 {
			buf.WriteRune(rune(charval))
			charval = 0
		}
	}

	if buf.Len() > 0 {
		n, err := d.CreateText(buf.Bytes())
		if err != nil {
			return nil, err
		}

		if last == nil {
			ret = n
		} else {
			last.AddSibling(n)
		}
	}

	if debug.Enabled {
		for n := last; n != nil; n = n.PrevSibling() {
			debug.Printf("---> %s (%#v)", n.Name(), n)
		}
	}

	return ret, nil
}

func (d *Document) CreateCharRef(name string) (*EntityRef, error) {
	if debug.Enabled {
		g := debug.IPrintf("START document.CreateCharRef '%s'", name)
		defer g.IRelease("END document.CreateCharRef")
	}

	if name == "" {
		return nil, errors.New("empty name")
	}

	n := newEntityRef()
	n.doc = d
	if name[0] != '&' {
		n.name = name
	} else {
		// the name should be everything but '&' and ';'
		if name[len(name)-1] == ';' {
			n.name = name[1 : len(name)-1]
		} else {
			n.name = name[1:]
		}
	}
	return n, nil
}
<|endoftext|>


<commit_before>package box

import (
	"errors"
	"fmt"
	"time"
)

type DocumentService struct {
	client *Client
}

type Document struct {
	Type      string    `json:"type"`
	ID        string    `json:"id"`
	Status    string    `json:"status"`
	Name      string    `json:"name"`
	CreatedAt time.Time `json:"created_at"`
}

type DocumentInput struct {
	URL        string `json:"url"`
	Name       string `json:"name"`
	Thumbnails string `json:"thumbnails"`
	NonSVG     bool   `json:"non_svg"`
}

func (s *DocumentService) NewURL(doc DocumentInput) (*Document, error) {
	if doc.URL == "" {
		return nil, errors.New("URL is required")
	}

	req, err := s.client.NewRequest("POST", "documents", doc)
	if err != nil {
		return nil, err
	}

	uResp := new(Document)
	_, err = s.client.Do(req, uResp)
	return uResp, err
}

func (s *DocumentService) FindOne(id string, fields string) (*Document, error) {
	u := fmt.Sprintf("/documents/%s", id)
	if fields != "" {
		u = fmt.Sprintf("%s?fields=%s", u, fields)
	}
	req, err := s.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, err
	}

	uResp := new(Document)
	_, err = s.client.Do(req, uResp)
	return uResp, err
}
<commit_msg>Content and Thumbnail for document<commit_after>package box

import (
	"errors"
	"fmt"
	"net/http"
	"time"
)

type DocumentService struct {
	client *Client
}

type Document struct {
	Type      string    `json:"type"`
	ID        string    `json:"id"`
	Status    string    `json:"status"`
	Name      string    `json:"name"`
	CreatedAt time.Time `json:"created_at"`
}

type DocumentInput struct {
	URL        string `json:"url"`
	Name       string `json:"name"`
	Thumbnails string `json:"thumbnails"`
	NonSVG     bool   `json:"non_svg"`
}

func (s *DocumentService) NewURL(doc DocumentInput) (*Document, error) {
	if doc.URL == "" {
		return nil, errors.New("URL is required")
	}

	req, err := s.client.NewRequest("POST", "documents", doc)
	if err != nil {
		return nil, err
	}

	uResp := new(Document)
	_, err = s.client.Do(req, uResp)
	return uResp, err
}

func (s *DocumentService) FindOne(id string, fields string) (*Document, error) {
	u := fmt.Sprintf("/documents/%s", id)
	if fields != "" {
		u = fmt.Sprintf("%s?fields=%s", u, fields)
	}
	req, err := s.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, err
	}

	uResp := new(Document)
	_, err = s.client.Do(req, uResp)
	return uResp, err
}

func (s *DocumentService) GetThumbnail(id string, width, height int) (*http.Response, error) {
	u := fmt.Sprintf("/documents/%s/thumbnail?width=%d&height=%d", id, width, height)
	req, err := s.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, err
	}

	resp, err := s.client.Do(req, nil)
	return resp, err
}

func (s *DocumentService) GetContent(id, extension string) (*http.Response, error) {
	u := fmt.Sprintf("/documents/%s/content.%s", id, extension)
	req, err := s.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, err
	}

	resp, err := s.client.Do(req, nil)
	return resp, err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/cubicdaiya/nginx-build/builder"
	"github.com/cubicdaiya/nginx-build/command"
	"github.com/cubicdaiya/nginx-build/util"
)

const DefaultDownloadTimeout = time.Duration(900) * time.Second

func extractArchive(path string) error {
	return command.Run([]string{"tar", "zxvf", path})
}

func download(b *builder.Builder) error {
	c := &http.Client{
		Timeout: DefaultDownloadTimeout,
	}

	res, err := c.Get(b.DownloadURL())
	if err != nil {
		return err
	}
	defer res.Body.Close()

	tmpFileName := b.ArchivePath() + ".download"
	f, err := os.Create(tmpFileName)
	if err != nil {
		return err
	}
	defer f.Close()

	_, err = io.Copy(f, res.Body)
	if err != nil && err != io.EOF {
		return err
	}

	err = os.Rename(tmpFileName, b.ArchivePath())
	if err != nil {
		return err
	}

	return nil
}

func downloadAndExtract(b *builder.Builder) error {
	if !util.FileExists(b.SourcePath()) {
		if !util.FileExists(b.ArchivePath()) {

			log.Printf("Download %s.....", b.SourcePath())

			if err := download(b); err != nil {
				return fmt.Errorf("Failed to download %s. %s", b.SourcePath(), err.Error())
			}
		}

		log.Printf("Extract %s.....", b.ArchivePath())

		if err := extractArchive(b.ArchivePath()); err != nil {
			return fmt.Errorf("Failed to extract %s. %s", b.ArchivePath(), err.Error())
		}
	} else {
		log.Printf("%s already exists.", b.SourcePath())
	}
	return nil
}

func downloadAndExtractParallel(b *builder.Builder) {
	if err := downloadAndExtract(b); err != nil {
		util.PrintFatalMsg(err, b.LogPath())
	}
}
<commit_msg>style: sharrowed variable scope.<commit_after>package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/cubicdaiya/nginx-build/builder"
	"github.com/cubicdaiya/nginx-build/command"
	"github.com/cubicdaiya/nginx-build/util"
)

const DefaultDownloadTimeout = time.Duration(900) * time.Second

func extractArchive(path string) error {
	return command.Run([]string{"tar", "zxvf", path})
}

func download(b *builder.Builder) error {
	c := &http.Client{
		Timeout: DefaultDownloadTimeout,
	}
	res, err := c.Get(b.DownloadURL())
	if err != nil {
		return err
	}
	defer res.Body.Close()

	tmpFileName := b.ArchivePath() + ".download"
	f, err := os.Create(tmpFileName)
	if err != nil {
		return err
	}
	defer f.Close()

	if _, err := io.Copy(f, res.Body); err != nil && err != io.EOF {
		return err
	}

	if err := os.Rename(tmpFileName, b.ArchivePath()); err != nil {
		return err
	}

	return nil
}

func downloadAndExtract(b *builder.Builder) error {
	if !util.FileExists(b.SourcePath()) {
		if !util.FileExists(b.ArchivePath()) {

			log.Printf("Download %s.....", b.SourcePath())

			if err := download(b); err != nil {
				return fmt.Errorf("Failed to download %s. %s", b.SourcePath(), err.Error())
			}
		}

		log.Printf("Extract %s.....", b.ArchivePath())

		if err := extractArchive(b.ArchivePath()); err != nil {
			return fmt.Errorf("Failed to extract %s. %s", b.ArchivePath(), err.Error())
		}
	} else {
		log.Printf("%s already exists.", b.SourcePath())
	}
	return nil
}

func downloadAndExtractParallel(b *builder.Builder) {
	if err := downloadAndExtract(b); err != nil {
		util.PrintFatalMsg(err, b.LogPath())
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"gopkg.in/mgo.v2"
	"log"
	"net/http"
	"runtime"
)

const (
	connectionString = "localhost"
	helloWorldString = "Hello, world!"
)

type Message struct {
	Message string `json:"message"`
}

type World struct {
	Id           uint16 `json:"id"`
	RandomNumber uint16 `json:"randomNumber"`
}

type Fortune struct {
	Id      uint16 `json:"id"`
	Message string `json:"message"`
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	session, err := mgo.Dial(connectionString)
	if err != nil {
		log.Fatalf("Error opening database: %v", err)
	}
	fmt.Println("Test")
	defer session.Close()
	session.SetPoolLimit(5)
	http.HandleFunc("/json", jsonHandler)
	http.ListenAndServe(":8228", nil)
}

// Test 1: JSON serialization
func jsonHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/javascript")
	json.NewEncoder(w).Encode(&Message{helloWorldString})
}
<commit_msg>Initial work on DB handler<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"gopkg.in/mgo.v2"
	"log"
	"net/http"
	"runtime"
)

const (
	connectionString = "localhost"
	helloWorldString = "Hello, world!"
	worldRowCount    = 10000
)

var (
	collection *mgo.Collection
)

type Message struct {
	Message string `json:"message"`
}

type World struct {
	Id           uint16 `json:"id"`
	RandomNumber uint16 `json:"randomNumber"`
}

type Fortune struct {
	Id      uint16 `json:"id"`
	Message string `json:"message"`
}

func main() {
	port := ":8228"
	runtime.GOMAXPROCS(runtime.NumCPU())
	session, err := mgo.Dial(connectionString)
	if err != nil {
		log.Fatalf("Error opening database: %v", err)
	}
	defer session.Close()
	session.SetPoolLimit(5)
	collection = session.DB("hello_world").C("world")
	http.HandleFunc("/db", dbHandler)
	http.HandleFunc("/json", jsonHandler)
	fmt.Println("Serving on http://localhost" + port)
	http.ListenAndServe(port, nil)
}

// Test 1: JSON serialization
func jsonHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/javascript")
	json.NewEncoder(w).Encode(&Message{helloWorldString})
}

func dbHandler(w http.ResponseWriter, r *http.Request) {
	var world World
	var randomNumber = rand.Intn(worldRowCount) + 1
	query := bson.M{
		"id": randomNumber,
	}
	if collection != nil {
		if err := collection.Find(query).One(&World); err != nil {
			log.Fatalf("Error finding world with id: %s", err.Error())
			return
		} else {
			w.Header().Set("Content-Type", "application/json")
			json.NewEncoder(w).Encode(&world)
			return
		}
	} else {
		log.Fatal("Collection not initialized properly")
	}
}
<|endoftext|>


<commit_before>package expreduce

import (
	"math/big"
	"strings"
	"go/token"
	"os"
	//"fmt"
	"log"
	"bytes"
	"github.com/cznic/wl"
)

func fullyAssoc(op string, lhs Ex, rhs Ex) Ex {
	opExpr, isOp := HeadAssertion(lhs, op)
	if isOp {
		opExpr.Parts = append(opExpr.Parts, rhs)
		return opExpr
	}
	return NewExpression([]Ex{&Symbol{op}, lhs, rhs})
}

func rightFullyAssoc(op string, lhs Ex, rhs Ex) Ex {
	opExpr, isOp := HeadAssertion(rhs, op)
	if isOp {
		opExpr.Parts = append([]Ex{opExpr.Parts[0], lhs}, opExpr.Parts[1:]...)
		return opExpr
	}
	return NewExpression([]Ex{&Symbol{op}, lhs, rhs})
}

func addContextAndDefine(e Ex, context string, contextPath []string, es *EvalState) {
	if sym, isSym := e.(*Symbol); isSym {
		if !strings.Contains(sym.Name, "`") {
			for _, toTry := range contextPath {
				if es.IsDef(toTry + sym.Name) {
					sym.Name = toTry + sym.Name
					return
				}
			}
			sym.Name = context + sym.Name
		}
		es.MarkSeen(sym.Name)
	}
	expr, isExpr := e.(*Expression)
	if isExpr {
		for _, part := range expr.Parts {
			addContextAndDefine(part, context, contextPath, es)
		}
	}
}

func parsePattern(buf string) Ex {
	delim := "_"
	blankType := &Symbol{"System`Blank"}
	if strings.Contains(buf, "___") {
		delim = "___"
		blankType = &Symbol{"System`BlankNullSequence"}
	} else if strings.Contains(buf, "__") {
		delim = "__"
		blankType = &Symbol{"System`BlankSequence"}
	}
	parts := strings.Split(buf, delim)
	if len(parts) == 1 {
		return NewExpression([]Ex{&Symbol{"System`Pattern"}, &Symbol{parts[0]}, NewExpression([]Ex{blankType})})
	}
	if len(parts) == 2 {
		if parts[0] == "" {
			if parts[1] == "" {
				return NewExpression([]Ex{blankType})
			} else if delim == "_" && parts[1] == "." {
				return NewExpression([]Ex{&Symbol{"System`Optional"}, NewExpression([]Ex{blankType})})
			}
			return NewExpression([]Ex{blankType, &Symbol{parts[1]}})
		} else {
			if parts[1] == "" {
				return NewExpression([]Ex{&Symbol{"System`Pattern"}, &Symbol{parts[0]}, NewExpression([]Ex{blankType})})
			} else if delim == "_" && parts[1] == "." {
				return NewExpression([]Ex{&Symbol{"System`Optional"}, NewExpression([]Ex{&Symbol{"System`Pattern"}, &Symbol{parts[0]}, NewExpression([]Ex{blankType})})})
			}
			return NewExpression([]Ex{&Symbol{"System`Pattern"}, &Symbol{parts[0]}, NewExpression([]Ex{blankType, &Symbol{parts[1]}})})
		}
	}
	return NewExpression([]Ex{&Symbol{"System`Error"}, &String{"Pattern parse error."}})
}

func ParserTokenConv(tk wl.Token) Ex {
	switch tk.Rune {
	case wl.IDENT:
		return &Symbol{tk.Val}
	case wl.INT:
		base := 10
		tmpi := big.NewInt(0)
		_, ok := tmpi.SetString(tk.Val, base)
		if !ok {
			log.Fatal("Failed in integer parsing.")
		}
		return &Integer{tmpi}
	case wl.FLOAT:
		tmpf := big.NewFloat(0)
		_, ok := tmpf.SetString(tk.Val)
		if !ok {
			log.Fatal("Failed in float parsing.")
		}
		return &Flt{tmpf}
	case wl.STRING:
		return &String{tk.Val}
	case wl.PATTERN:
		return parsePattern(tk.Val)
	case wl.SLOT:
		tmpi := big.NewInt(1)
		if tk.Val != "#" {
			_, ok := tmpi.SetString(tk.Val[1:], 10)
			if !ok {
				log.Fatal("Failed in integer parsing.")
			}
		}
		return NewExpression([]Ex{
			&Symbol{"System`Slot"},
			&Integer{tmpi},
		})

	default:
		return &Symbol{"System`UnParsedToken"}
	}
	return &Symbol{"System`UnParsedToken"}
}

func ParserTagConv(tag *wl.Tag) Ex {
	return ParserTokenConv(tag.Token)
}

func ParserExprListConv(l *wl.ExprList) (res []Ex) {
	for l != nil {
		if l.Expression != nil {
			res = append(res, ParserExprConv(l.Expression))
		} else {
			res = append(res, ParserTokenConv(l.Token))
		}
		l = l.ExprList
	}
	return
}

func ParserTermConv(term *wl.Term) Ex {
	if term.Token2.Rune > 0 {
		switch term.Token2.Rune {
		case wl.MESSAGE_NAME:
			return NewExpression([]Ex{
				&Symbol{"System`MessageName"},
				ParserTokenConv(term.Token),
				&String{ParserTagConv(term.Tag).(*Symbol).Name},
			})
		case wl.RPART:
			e := NewExpression([]Ex{
				&Symbol{"System`Part"},
				ParserTermConv(term.Term),
			})
			e.appendExArray(ParserExprListConv(term.ExprList))
			return e
		case ']':
			e := NewExpression([]Ex{
				ParserTermConv(term.Term),
			})
			e.appendExArray(ParserExprListConv(term.ExprList))
			return e
		case '}':
			e := NewExpression([]Ex{
				&Symbol{"System`List"},
			})
			e.appendExArray(ParserExprListConv(term.ExprList))
			return e
		case ')':
			return ParserExprConv(term.Expression)
		default:
			return &Symbol{"System`UnParsedTermWithToken2"}
		}
	}
	if term.Token.Rune > 0 {
		switch term.Token.Rune {
		case '!':
			return NewExpression([]Ex{
				&Symbol{"System`Factorial"},
				ParserTermConv(term.Term),
			})
		case '&':
			return NewExpression([]Ex{
				&Symbol{"System`Function"},
				ParserTermConv(term.Term),
			})
		}
		return ParserTokenConv(term.Token)
	}
	return &Symbol{"System`UnParsedTerm"}
}

func ParserFactorConv(fact *wl.Factor) Ex {
	if fact.Term != nil {
		tv := ParserTermConv(fact.Term)
		if fact.Factor != nil {
			return NewExpression([]Ex{
				&Symbol{"System`Times"},
				tv,
				ParserFactorConv(fact.Factor),
			})
		}
		return tv
	}
	return &Symbol{"System`UnParsedFactor"}
}

var binaryOps = map[int]string{
	44: "Set",
	19: "SetDelayed",
	15: "ReplaceRepeated",
	13: "ReplaceAll",
	11: "Rule",
	20: "RuleDelayed",
	48: "Power",
	46: "PatternTest",
	40: "Optional",
	17: "Condition",
	28: "Apply",
	18: "Map",
}

var fullyAssocOps = map[int]string{
	36: "Plus",
	35: "Times",
	24: "Equal",
	25: "SameQ",
	23: "UnsameQ",
	22: "StringJoin",
	43: "Less",
	21: "LessEqual",
	45: "Greater",
	26: "GreaterEqual",
	34: "Or",
	8: "And",
	49: "Alternatives",
}

func ParserExprConv(expr *wl.Expression) Ex {
	if expr.Factor != nil {
		return ParserFactorConv(expr.Factor)
	}
	for c, head := range binaryOps {
		if expr.Case == c {
			return NewExpression([]Ex{
				&Symbol{"System`" + head},
				ParserExprConv(expr.Expression),
				ParserExprConv(expr.Expression2),
			})
		}
	}
	for c, head := range fullyAssocOps {
		if expr.Case == c {
			return fullyAssoc(
				"System`" + head,
				ParserExprConv(expr.Expression),
				ParserExprConv(expr.Expression2),
			)
		}
	}
	switch expr.Case {
	case 14:
		return NewExpression([]Ex{
			ParserExprConv(expr.Expression2),
			ParserExprConv(expr.Expression),
		})
	case 47:
		return NewExpression([]Ex{
			ParserExprConv(expr.Expression),
			ParserExprConv(expr.Expression2),
		})
	case 6:
		return NewExpression([]Ex{
			&Symbol{"System`Not"},
			ParserExprConv(expr.Expression),
		})
	case 41:
		return fullyAssoc(
			"System`CompoundExpression",
			ParserExprConv(expr.Expression),
			&Symbol{"System`Null"},
		)
	case 42:
		return fullyAssoc(
			"System`CompoundExpression",
			ParserExprConv(expr.Expression),
			ParserExprConv(expr.Expression2),
		)
	case 37:
		return fullyAssoc(
			"System`Plus",
			ParserExprConv(expr.Expression),
			NewExpression([]Ex{
				&Symbol{"System`Times"},
				&Integer{big.NewInt(-1)},
				ParserExprConv(expr.Expression2),
			}),
		)
	case 7:
		return NewExpression([]Ex{
			&Symbol{"System`Times"},
			&Integer{big.NewInt(-1)},
			ParserExprConv(expr.Expression),
		})
	case 39:
		return NewExpression([]Ex{
			&Symbol{"System`Times"},
			ParserExprConv(expr.Expression),
			NewExpression([]Ex{
				&Symbol{"System`Power"},
				ParserExprConv(expr.Expression2),
				&Integer{big.NewInt(-1)},
			}),
		})
	}
	return &Symbol{"System`UnParsed"}
}

func Interp(src string, es *EvalState) Ex {
	//fmt.Printf("Parsing %s\n", src)
	buf := bytes.NewBufferString(src)
	// TODO(corywalker): use the interactive mode for proper newline handling.
	in, err := wl.NewInput(buf, false)
	if err != nil {
		return &Symbol{"System`Null"}
	}
	expr, err := in.ParseExpression(token.NewFileSet().AddFile(os.Stdin.Name(), -1, 1e6))
	if err != nil {
		return &Symbol{"System`Null"}
	}
	parsed := ParserExprConv(expr)
	context := es.GetStringDef("System`$Context", "")
	contextPathEx := es.GetListDef("System`$ContextPath")
	contextPath := []string{}
	for _, pathPart := range contextPathEx.Parts[1:] {
		contextPath = append(contextPath, pathPart.(*String).Val)
	}
	addContextAndDefine(parsed, context, contextPath, es)
	//fmt.Printf("result %s\n", parsed)
	return parsed
}

func EvalInterp(src string, es *EvalState) Ex {
	return Interp(src, es).Eval(es)
}

func EvalInterpMany(doc string, es *EvalState) Ex {
	var last Ex
	for _, expr := range strings.Split(doc, "\n\n\n") {
		if len(strings.TrimSpace(expr)) == 0 {
			continue
		}
		last = EvalInterp(expr, es)
	}
	return last
}

func EasyRun(src string, es *EvalState) string {
	context, contextPath := ActualStringFormArgs(es)
	return EvalInterp(src, es).StringForm("InputForm", context, contextPath)
}
<commit_msg>A few improvements to the parser. Optional vs Pattern has some trouble.<commit_after>package expreduce

import (
	"math/big"
	"strings"
	"go/token"
	"os"
	//"fmt"
	"log"
	"bytes"
	"github.com/cznic/wl"
)

func fullyAssoc(op string, lhs Ex, rhs Ex) Ex {
	opExpr, isOp := HeadAssertion(lhs, op)
	if isOp {
		opExpr.Parts = append(opExpr.Parts, rhs)
		return opExpr
	}
	return NewExpression([]Ex{&Symbol{op}, lhs, rhs})
}

func rightFullyAssoc(op string, lhs Ex, rhs Ex) Ex {
	opExpr, isOp := HeadAssertion(rhs, op)
	if isOp {
		opExpr.Parts = append([]Ex{opExpr.Parts[0], lhs}, opExpr.Parts[1:]...)
		return opExpr
	}
	return NewExpression([]Ex{&Symbol{op}, lhs, rhs})
}

func addContextAndDefine(e Ex, context string, contextPath []string, es *EvalState) {
	if sym, isSym := e.(*Symbol); isSym {
		if !strings.Contains(sym.Name, "`") {
			for _, toTry := range contextPath {
				if es.IsDef(toTry + sym.Name) {
					sym.Name = toTry + sym.Name
					return
				}
			}
			sym.Name = context + sym.Name
		}
		es.MarkSeen(sym.Name)
	}
	expr, isExpr := e.(*Expression)
	if isExpr {
		for _, part := range expr.Parts {
			addContextAndDefine(part, context, contextPath, es)
		}
	}
}

func parsePattern(buf string) Ex {
	delim := "_"
	blankType := &Symbol{"System`Blank"}
	if strings.Contains(buf, "___") {
		delim = "___"
		blankType = &Symbol{"System`BlankNullSequence"}
	} else if strings.Contains(buf, "__") {
		delim = "__"
		blankType = &Symbol{"System`BlankSequence"}
	}
	parts := strings.Split(buf, delim)
	if len(parts) == 1 {
		return NewExpression([]Ex{&Symbol{"System`Pattern"}, &Symbol{parts[0]}, NewExpression([]Ex{blankType})})
	}
	if len(parts) == 2 {
		if parts[0] == "" {
			if parts[1] == "" {
				return NewExpression([]Ex{blankType})
			} else if delim == "_" && parts[1] == "." {
				return NewExpression([]Ex{&Symbol{"System`Optional"}, NewExpression([]Ex{blankType})})
			}
			return NewExpression([]Ex{blankType, &Symbol{parts[1]}})
		} else {
			if parts[1] == "" {
				return NewExpression([]Ex{&Symbol{"System`Pattern"}, &Symbol{parts[0]}, NewExpression([]Ex{blankType})})
			} else if delim == "_" && parts[1] == "." {
				return NewExpression([]Ex{&Symbol{"System`Optional"}, NewExpression([]Ex{&Symbol{"System`Pattern"}, &Symbol{parts[0]}, NewExpression([]Ex{blankType})})})
			}
			return NewExpression([]Ex{&Symbol{"System`Pattern"}, &Symbol{parts[0]}, NewExpression([]Ex{blankType, &Symbol{parts[1]}})})
		}
	}
	return NewExpression([]Ex{&Symbol{"System`Error"}, &String{"Pattern parse error."}})
}

func ParserTokenConv(tk wl.Token) Ex {
	switch tk.Rune {
	case wl.IDENT:
		return &Symbol{tk.Val}
	case wl.INT:
		base := 10
		tmpi := big.NewInt(0)
		_, ok := tmpi.SetString(tk.Val, base)
		if !ok {
			log.Fatal("Failed in integer parsing.")
		}
		return &Integer{tmpi}
	case wl.FLOAT:
		tmpf := big.NewFloat(0)
		_, ok := tmpf.SetString(tk.Val)
		if !ok {
			log.Fatal("Failed in float parsing.")
		}
		return &Flt{tmpf}
	case wl.STRING:
		return &String{tk.Val}
	case wl.PATTERN:
		return parsePattern(tk.Val)
	case wl.SLOT:
		tmpi := big.NewInt(1)
		if tk.Val != "#" {
			_, ok := tmpi.SetString(tk.Val[1:], 10)
			if !ok {
				log.Fatal("Failed in integer parsing.")
			}
		}
		return NewExpression([]Ex{
			&Symbol{"System`Slot"},
			&Integer{tmpi},
		})

	default:
		return &Symbol{"System`UnParsedToken"}
	}
	return &Symbol{"System`UnParsedToken"}
}

func ParserTagConv(tag *wl.Tag) Ex {
	return ParserTokenConv(tag.Token)
}

func ParserExprListConv(l *wl.ExprList) (res []Ex) {
	for l != nil {
		if l.Expression != nil {
			res = append(res, ParserExprConv(l.Expression))
		} else {
			res = append(res, ParserTokenConv(l.Token))
		}
		l = l.ExprList
	}
	return
}

func ParserTermConv(term *wl.Term) Ex {
	if term.Token2.Rune > 0 {
		switch term.Token2.Rune {
		case wl.MESSAGE_NAME:
			return NewExpression([]Ex{
				&Symbol{"System`MessageName"},
				ParserTokenConv(term.Token),
				&String{ParserTagConv(term.Tag).(*Symbol).Name},
			})
		case wl.RPART:
			e := NewExpression([]Ex{
				&Symbol{"System`Part"},
				ParserTermConv(term.Term),
			})
			e.appendExArray(ParserExprListConv(term.ExprList))
			return e
		case ']':
			e := NewExpression([]Ex{
				ParserTermConv(term.Term),
			})
			e.appendExArray(ParserExprListConv(term.ExprList))
			return e
		case '}':
			e := NewExpression([]Ex{
				&Symbol{"System`List"},
			})
			e.appendExArray(ParserExprListConv(term.ExprList))
			return e
		case ')':
			return ParserExprConv(term.Expression)
		default:
			return &Symbol{"System`UnParsedTermWithToken2"}
		}
	}
	if term.Token.Rune > 0 {
		switch term.Token.Rune {
		case '!':
			return NewExpression([]Ex{
				&Symbol{"System`Factorial"},
				ParserTermConv(term.Term),
			})
		case '&':
			return NewExpression([]Ex{
				&Symbol{"System`Function"},
				ParserTermConv(term.Term),
			})
		}
		return ParserTokenConv(term.Token)
	}
	return &Symbol{"System`UnParsedTerm"}
}

func ParserFactorConv(fact *wl.Factor) Ex {
	if fact.Term != nil {
		tv := ParserTermConv(fact.Term)
		if fact.Factor != nil {
			return NewExpression([]Ex{
				&Symbol{"System`Times"},
				tv,
				ParserFactorConv(fact.Factor),
			})
		}
		return tv
	}
	return &Symbol{"System`UnParsedFactor"}
}

var binaryOps = map[int]string{
	44: "Set",
	19: "SetDelayed",
	15: "ReplaceRepeated",
	13: "ReplaceAll",
	11: "Rule",
	20: "RuleDelayed",
	48: "Power",
	46: "PatternTest",
	40: "Optional",
	17: "Condition",
	28: "Apply",
	18: "Map",
}

var fullyAssocOps = map[int]string{
	36: "Plus",
	35: "Times",
	24: "Equal",
	70: "Unequal",
	25: "SameQ",
	23: "UnsameQ",
	22: "StringJoin",
	43: "Less",
	21: "LessEqual",
	45: "Greater",
	26: "GreaterEqual",
	34: "Or",
	8: "And",
	38: "Dot",
	49: "Alternatives",
}

func ParserExprConv(expr *wl.Expression) Ex {
	if expr.Factor != nil {
		return ParserFactorConv(expr.Factor)
	}
	for c, head := range binaryOps {
		if expr.Case == c {
			return NewExpression([]Ex{
				&Symbol{"System`" + head},
				ParserExprConv(expr.Expression),
				ParserExprConv(expr.Expression2),
			})
		}
	}
	for c, head := range fullyAssocOps {
		if expr.Case == c {
			return fullyAssoc(
				"System`" + head,
				ParserExprConv(expr.Expression),
				ParserExprConv(expr.Expression2),
			)
		}
	}
	switch expr.Case {
	case 14:
		return NewExpression([]Ex{
			ParserExprConv(expr.Expression2),
			ParserExprConv(expr.Expression),
		})
	case 47:
		return NewExpression([]Ex{
			ParserExprConv(expr.Expression),
			ParserExprConv(expr.Expression2),
		})
	case 6:
		return NewExpression([]Ex{
			&Symbol{"System`Not"},
			ParserExprConv(expr.Expression),
		})
	case 41:
		return fullyAssoc(
			"System`CompoundExpression",
			ParserExprConv(expr.Expression),
			&Symbol{"System`Null"},
		)
	case 42:
		return fullyAssoc(
			"System`CompoundExpression",
			ParserExprConv(expr.Expression),
			ParserExprConv(expr.Expression2),
		)
	case 37:
		return fullyAssoc(
			"System`Plus",
			ParserExprConv(expr.Expression),
			NewExpression([]Ex{
				&Symbol{"System`Times"},
				ParserExprConv(expr.Expression2),
				&Integer{big.NewInt(-1)},
			}),
		)
	case 7:
		e := ParserExprConv(expr.Expression)
		if integer, isInteger := e.(*Integer); isInteger {
			return &Integer{integer.Val.Neg(integer.Val)}
		} else if flt, isFlt := e.(*Flt); isFlt {
			return &Flt{flt.Val.Neg(flt.Val)}
		}
		return NewExpression([]Ex{
			&Symbol{"System`Times"},
			ParserExprConv(expr.Expression),
			&Integer{big.NewInt(-1)},
		})
	case 39:
		return NewExpression([]Ex{
			&Symbol{"System`Times"},
			ParserExprConv(expr.Expression),
			NewExpression([]Ex{
				&Symbol{"System`Power"},
				ParserExprConv(expr.Expression2),
				&Integer{big.NewInt(-1)},
			}),
		})
	}
	return &Symbol{"System`UnParsed"}
}

func Interp(src string, es *EvalState) Ex {
	//fmt.Printf("Parsing %s\n", src)
	buf := bytes.NewBufferString(src)
	// TODO(corywalker): use the interactive mode for proper newline handling.
	in, err := wl.NewInput(buf, false)
	if err != nil {
		return &Symbol{"System`Null"}
	}
	expr, err := in.ParseExpression(token.NewFileSet().AddFile(os.Stdin.Name(), -1, 1e6))
	if err != nil {
		return &Symbol{"System`Null"}
	}
	parsed := ParserExprConv(expr)
	context := es.GetStringDef("System`$Context", "")
	contextPathEx := es.GetListDef("System`$ContextPath")
	contextPath := []string{}
	for _, pathPart := range contextPathEx.Parts[1:] {
		contextPath = append(contextPath, pathPart.(*String).Val)
	}
	addContextAndDefine(parsed, context, contextPath, es)
	//fmt.Printf("result %s\n", parsed)
	return parsed
}

func EvalInterp(src string, es *EvalState) Ex {
	return Interp(src, es).Eval(es)
}

func EvalInterpMany(doc string, es *EvalState) Ex {
	var last Ex
	for _, expr := range strings.Split(doc, "\n\n\n") {
		if len(strings.TrimSpace(expr)) == 0 {
			continue
		}
		last = EvalInterp(expr, es)
	}
	return last
}

func EasyRun(src string, es *EvalState) string {
	context, contextPath := ActualStringFormArgs(es)
	return EvalInterp(src, es).StringForm("InputForm", context, contextPath)
}
<|endoftext|>


<commit_before>package publish

import (
	"fmt"
	"net/http"
	"os"
	"path"
	"reflect"
	"strings"

	"github.com/qor/qor/admin"
)

type PublishController struct {
	*Publish
}

func (db *PublishController) Preview(context *admin.Context) {
	drafts := make(map[*admin.Resource]interface{})
	for _, res := range context.Admin.GetResources() {
		results := res.NewSlice()
		if db.DraftDB().Unscoped().Where("publish_status = ?", DIRTY).Find(results).RowsAffected > 0 {
			drafts[res] = results
		}
	}
	context.Execute("publish/drafts", drafts)
}

func (db *PublishController) Diff(context *admin.Context) {
	resourceID := strings.Split(context.Request.URL.Path, "/")[4]
	params := strings.Split(resourceID, "__")
	name, id := params[0], params[1]
	res := context.Admin.GetResource(name)

	draft := res.NewStruct()
	db.DraftDB().Unscoped().First(draft, id)

	production := res.NewStruct()
	db.ProductionDB().Unscoped().First(production, id)

	results := map[string]interface{}{"Production": production, "Draft": draft, "Resource": res}

	fmt.Fprintf(context.Writer, context.Render("publish/diff", results))
}

func (db *PublishController) PublishOrDiscard(context *admin.Context) {
	var request = context.Request
	var ids = request.Form["checked_ids[]"]
	var records = []interface{}{}
	var values = map[string][]string{}

	for _, id := range ids {
		if keys := strings.Split(id, "__"); len(keys) == 2 {
			name, id := keys[0], keys[1]
			values[name] = append(values[name], id)
		}
	}

	for name, value := range values {
		res := context.Admin.GetResource(name)
		results := res.NewSlice()
		if db.DraftDB().Unscoped().Find(results, fmt.Sprintf("%v IN (?)", res.PrimaryDBName()), value).Error == nil {
			resultValues := reflect.Indirect(reflect.ValueOf(results))
			for i := 0; i < resultValues.Len(); i++ {
				records = append(records, resultValues.Index(i).Interface())
			}
		}
	}

	if request.Form.Get("publish_type") == "publish" {
		db.Publish.Publish(records...)
	} else if request.Form.Get("publish_type") == "discard" {
		db.Publish.Discard(records...)
	}
	http.Redirect(context.Writer, context.Request, context.Request.RequestURI, http.StatusFound)
}

func (publish *Publish) InjectQorAdmin(res *admin.Resource) {
	controller := PublishController{publish}
	router := res.GetAdmin().GetRouter()
	router.Get(fmt.Sprintf("^/%v/diff/", res.ToParam()), controller.Diff)
	router.Get(fmt.Sprintf("^/%v", res.ToParam()), controller.Preview)
	router.Post(fmt.Sprintf("^/%v", res.ToParam()), controller.PublishOrDiscard)

	for _, gopath := range strings.Split(os.Getenv("GOPATH"), ":") {
		admin.RegisterViewPath(path.Join(gopath, "src/github.com/qor/qor/publish/views"))
	}
}
<commit_msg>Only show publishable models in controller<commit_after>package publish

import (
	"fmt"
	"net/http"
	"os"
	"path"
	"reflect"
	"strings"

	"github.com/qor/qor/admin"
)

type PublishController struct {
	*Publish
}

func (db *PublishController) Preview(context *admin.Context) {
	drafts := make(map[*admin.Resource]interface{})
	for _, res := range context.Admin.GetResources() {
		results := res.NewSlice()
		if IsPublishableModel(res.Value) {
			if db.DraftDB().Unscoped().Where("publish_status = ?", DIRTY).Find(results).RowsAffected > 0 {
				drafts[res] = results
			}
		}
	}
	context.Execute("publish/drafts", drafts)
}

func (db *PublishController) Diff(context *admin.Context) {
	resourceID := strings.Split(context.Request.URL.Path, "/")[4]
	params := strings.Split(resourceID, "__")
	name, id := params[0], params[1]
	res := context.Admin.GetResource(name)

	draft := res.NewStruct()
	db.DraftDB().Unscoped().First(draft, id)

	production := res.NewStruct()
	db.ProductionDB().Unscoped().First(production, id)

	results := map[string]interface{}{"Production": production, "Draft": draft, "Resource": res}

	fmt.Fprintf(context.Writer, context.Render("publish/diff", results))
}

func (db *PublishController) PublishOrDiscard(context *admin.Context) {
	var request = context.Request
	var ids = request.Form["checked_ids[]"]
	var records = []interface{}{}
	var values = map[string][]string{}

	for _, id := range ids {
		if keys := strings.Split(id, "__"); len(keys) == 2 {
			name, id := keys[0], keys[1]
			values[name] = append(values[name], id)
		}
	}

	for name, value := range values {
		res := context.Admin.GetResource(name)
		results := res.NewSlice()
		if db.DraftDB().Unscoped().Find(results, fmt.Sprintf("%v IN (?)", res.PrimaryDBName()), value).Error == nil {
			resultValues := reflect.Indirect(reflect.ValueOf(results))
			for i := 0; i < resultValues.Len(); i++ {
				records = append(records, resultValues.Index(i).Interface())
			}
		}
	}

	if request.Form.Get("publish_type") == "publish" {
		db.Publish.Publish(records...)
	} else if request.Form.Get("publish_type") == "discard" {
		db.Publish.Discard(records...)
	}
	http.Redirect(context.Writer, context.Request, context.Request.RequestURI, http.StatusFound)
}

func (publish *Publish) InjectQorAdmin(res *admin.Resource) {
	controller := PublishController{publish}
	router := res.GetAdmin().GetRouter()
	router.Get(fmt.Sprintf("^/%v/diff/", res.ToParam()), controller.Diff)
	router.Get(fmt.Sprintf("^/%v", res.ToParam()), controller.Preview)
	router.Post(fmt.Sprintf("^/%v", res.ToParam()), controller.PublishOrDiscard)

	for _, gopath := range strings.Split(os.Getenv("GOPATH"), ":") {
		admin.RegisterViewPath(path.Join(gopath, "src/github.com/qor/qor/publish/views"))
	}
}
<|endoftext|>


<commit_before>package generator

import (
	"bytes"
	"fmt"
	"io"
	"log"

	"github.com/alvaroloes/ocgen/parser"
)

func GenerateMethods(classFile *parser.ObjCClassFile) {
	createBackup(classFile.MName)
	for _, class := range classFile.Classes {
		//TODO: Make a backup of the file and override original file
		// implSrcFile, err := ioutil.ReadFile(class.ImplFileName)
		// if err != nil {
		// 	log.Printf("Class: %v. Unable to open implementation file: %v", class.Name, class.ImplFileName)
		// 	continue
		// }

		// implDstFile, err := os.Create(class.ImplFileName + ".ocgen")
		// if err != nil {
		// 	log.Printf("Class: %v. Unable to create implementation file: %v", class.Name, class.ImplFileName)
		// 	continue
		// }

		//TODO: Get the methods sorted by appearance
		//TODO: Write all before the method, write method, write all after it and before the following method

		codingInitMethod, err := getNSCodingInit(&class)
		if err == nil {
			fmt.Println("* NSCoding.init:", string(codingInitMethod))
			//writeMethod(codingInitMethod, class.NSCodingInfo.InitWithCoder, implFile)
		} else {
			log.Printf("Class: %v. Error when generating NSCoding.initWithCoder method: %v\n", class.Name, err)
		}

		codingEncodeMethod, err := getNSCodingEncode(&class)
		if err == nil {
			fmt.Println("* NSCoding.encode:", string(codingEncodeMethod))
		} else {
			log.Printf("Class: %v. Error when generating NSCoding.encodeWithCoder method: %v\n", class.Name, err)
		}

		copyingMethod, err := getNSCopying(&class)
		if err == nil {
			fmt.Println("* NSCopying.copy:", string(copyingMethod))
		} else {
			log.Printf("Class: %v. Error when generating NSCopying.copyWithZone method: %v\n", class.Name, err)
		}
	}
}

func getNSCopying(class *parser.ObjCClass) ([]byte, error) {
	var res bytes.Buffer
	err := NSCopyingTpl.Execute(&res, class)
	return res.Bytes(), err
}

func getNSCodingInit(class *parser.ObjCClass) ([]byte, error) {
	var res bytes.Buffer
	err := NSCodingInitTpl.Execute(&res, class)
	return res.Bytes(), err
}

func getNSCodingEncode(class *parser.ObjCClass) ([]byte, error) {
	var res bytes.Buffer
	err := NSCodingEncodeTpl.Execute(&res, class)
	return res.Bytes(), err
}

func writeMethod(methodText []byte, methodInfo parser.MethodInfo, writer io.Writer) {

}

func createBackup(fileName string) {
	// s, err := os.Open(src)
	// if err != nil {
	// 	return err
	// }
	// // no need to check errors on read only file, we already got everything
	// // we need from the filesystem, so nothing can go wrong now.
	// defer s.Close()
	// d, err := os.Create(dst)
	// if err != nil {
	// 	return err
	// }
	// if _, err := io.Copy(d, s); err != nil {
	// 	d.Close()
	// 	return err
	// }
	// return d.Close()
}
<commit_msg>Backup file is now created before writing the original file<commit_after>package generator

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"

	"github.com/alvaroloes/ocgen/parser"
)

var BackupFileExt = ".backup"

func GenerateMethods(classFile *parser.ObjCClassFile) error {
	if err := createBackup(classFile.MName); err != nil {
		return err
	}

	fileBytes, err := ioutil.ReadFile(class.ImplFileName)
	if err != nil {
		log.Printf("Class: %v. Unable to open implementation file: %v", class.Name, class.ImplFileName)
		return err
	}

	// TODO Open the MFile for writing (os.Create)

	for _, class := range classFile.Classes {

		//TODO: insert the methods bytes in the fileBytes slice in the corresponding location
		//TODO: Write the fileBytes into the MFile

		codingInitMethod, err := getNSCodingInit(&class)
		if err == nil {
			fmt.Println("* NSCoding.init:", string(codingInitMethod))
			//writeMethod(codingInitMethod, class.NSCodingInfo.InitWithCoder, implFile)
		} else {
			log.Printf("Class: %v. Error when generating NSCoding.initWithCoder method: %v\n", class.Name, err)
		}

		codingEncodeMethod, err := getNSCodingEncode(&class)
		if err == nil {
			fmt.Println("* NSCoding.encode:", string(codingEncodeMethod))
		} else {
			log.Printf("Class: %v. Error when generating NSCoding.encodeWithCoder method: %v\n", class.Name, err)
		}

		copyingMethod, err := getNSCopying(&class)
		if err == nil {
			fmt.Println("* NSCopying.copy:", string(copyingMethod))
		} else {
			log.Printf("Class: %v. Error when generating NSCopying.copyWithZone method: %v\n", class.Name, err)
		}
	}
}

func getNSCopying(class *parser.ObjCClass) ([]byte, error) {
	var res bytes.Buffer
	err := NSCopyingTpl.Execute(&res, class)
	return res.Bytes(), err
}

func getNSCodingInit(class *parser.ObjCClass) ([]byte, error) {
	var res bytes.Buffer
	err := NSCodingInitTpl.Execute(&res, class)
	return res.Bytes(), err
}

func getNSCodingEncode(class *parser.ObjCClass) ([]byte, error) {
	var res bytes.Buffer
	err := NSCodingEncodeTpl.Execute(&res, class)
	return res.Bytes(), err
}

func writeMethod(methodText []byte, methodInfo parser.MethodInfo, writer io.Writer) {

}

func createBackup(fileName string) (err error) {
	backupFileName := fileName + BackupFileExt
	file, err := os.Open(fileName)
	if err != nil {
		return
	}
	defer file.Close()

	backupFile, err := os.Create(backupFileName)
	if err != nil {
		return
	}
	defer func() {
		err = backupFile.Close()
	}()

	_, err = io.Copy(backupFile, file)
	return
}
<|endoftext|>