<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package testing

import (
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httptest"
	"os"
	"path"
	"testing"
	"time"

	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/apiserver/pkg/storage/etcd/etcdtest"
	"k8s.io/apiserver/pkg/storage/etcd/testing/testingcert"
	"k8s.io/apiserver/pkg/storage/storagebackend"

	etcd "github.com/coreos/etcd/client"
	"github.com/coreos/etcd/clientv3"
	"github.com/coreos/etcd/etcdserver"
	"github.com/coreos/etcd/etcdserver/api/v2http"
	"github.com/coreos/etcd/integration"
	"github.com/coreos/etcd/pkg/testutil"
	"github.com/coreos/etcd/pkg/transport"
	"github.com/coreos/etcd/pkg/types"
	"github.com/golang/glog"
	"golang.org/x/net/context"
)

// EtcdTestServer encapsulates the datastructures needed to start local instance for testing
type EtcdTestServer struct {
	// The following are lumped etcd2 test server params
	// TODO: Deprecate in a post 1.5 release
	etcdserver.ServerConfig
	PeerListeners, ClientListeners []net.Listener
	Client                         etcd.Client

	CertificatesDir string
	CertFile        string
	KeyFile         string
	CAFile          string

	raftHandler http.Handler
	s           *etcdserver.EtcdServer
	hss         []*httptest.Server

	// The following are lumped etcd3 test server params
	v3Cluster *integration.ClusterV3
	V3Client  *clientv3.Client
}

// newLocalListener opens a port localhost using any port
func newLocalListener(t *testing.T) net.Listener {
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	return l
}

// newSecuredLocalListener opens a port localhost using any port
// with SSL enable
func newSecuredLocalListener(t *testing.T, certFile, keyFile, caFile string) net.Listener {
	var l net.Listener
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	tlsInfo := transport.TLSInfo{
		CertFile: certFile,
		KeyFile:  keyFile,
		CAFile:   caFile,
	}
	tlscfg, err := tlsInfo.ServerConfig()
	if err != nil {
		t.Fatalf("unexpected serverConfig error: %v", err)
	}
	l, err = transport.NewKeepAliveListener(l, "https", tlscfg)
	if err != nil {
		t.Fatal(err)
	}
	return l
}

func newHttpTransport(t *testing.T, certFile, keyFile, caFile string) etcd.CancelableTransport {
	tlsInfo := transport.TLSInfo{
		CertFile: certFile,
		KeyFile:  keyFile,
		CAFile:   caFile,
	}
	tr, err := transport.NewTransport(tlsInfo, time.Second)
	if err != nil {
		t.Fatal(err)
	}
	return tr
}

// configureTestCluster will set the params to start an etcd server
func configureTestCluster(t *testing.T, name string, https bool) *EtcdTestServer {
	var err error
	m := &EtcdTestServer{}

	pln := newLocalListener(t)
	m.PeerListeners = []net.Listener{pln}
	m.PeerURLs, err = types.NewURLs([]string{"http://" + pln.Addr().String()})
	if err != nil {
		t.Fatal(err)
	}

	// Allow test launches to control where etcd data goes, for space or performance reasons
	baseDir := os.Getenv("TEST_ETCD_DIR")
	if len(baseDir) == 0 {
		baseDir = os.TempDir()
	}

	if https {
		m.CertificatesDir, err = ioutil.TempDir(baseDir, "etcd_certificates")
		if err != nil {
			t.Fatal(err)
		}
		m.CertFile = path.Join(m.CertificatesDir, "etcdcert.pem")
		if err = ioutil.WriteFile(m.CertFile, []byte(testingcert.CertFileContent), 0644); err != nil {
			t.Fatal(err)
		}
		m.KeyFile = path.Join(m.CertificatesDir, "etcdkey.pem")
		if err = ioutil.WriteFile(m.KeyFile, []byte(testingcert.KeyFileContent), 0644); err != nil {
			t.Fatal(err)
		}
		m.CAFile = path.Join(m.CertificatesDir, "ca.pem")
		if err = ioutil.WriteFile(m.CAFile, []byte(testingcert.CAFileContent), 0644); err != nil {
			t.Fatal(err)
		}

		cln := newSecuredLocalListener(t, m.CertFile, m.KeyFile, m.CAFile)
		m.ClientListeners = []net.Listener{cln}
		m.ClientURLs, err = types.NewURLs([]string{"https://" + cln.Addr().String()})
		if err != nil {
			t.Fatal(err)
		}
	} else {
		cln := newLocalListener(t)
		m.ClientListeners = []net.Listener{cln}
		m.ClientURLs, err = types.NewURLs([]string{"http://" + cln.Addr().String()})
		if err != nil {
			t.Fatal(err)
		}
	}

	m.Name = name
	m.DataDir, err = ioutil.TempDir(baseDir, "etcd")
	if err != nil {
		t.Fatal(err)
	}

	clusterStr := fmt.Sprintf("%s=http://%s", name, pln.Addr().String())
	m.InitialPeerURLsMap, err = types.NewURLsMap(clusterStr)
	if err != nil {
		t.Fatal(err)
	}
	m.InitialClusterToken = "TestEtcd"
	m.NewCluster = true
	m.ForceNewCluster = false
	m.ElectionTicks = 10
	m.TickMs = uint(10)

	return m
}

// launch will attempt to start the etcd server
func (m *EtcdTestServer) launch(t *testing.T) error {
	var err error
	if m.s, err = etcdserver.NewServer(&m.ServerConfig); err != nil {
		return fmt.Errorf("failed to initialize the etcd server: %v", err)
	}
	m.s.SyncTicker = time.Tick(500 * time.Millisecond)
	m.s.Start()
	m.raftHandler = &testutil.PauseableHandler{Next: v2http.NewPeerHandler(m.s)}
	for _, ln := range m.PeerListeners {
		hs := &httptest.Server{
			Listener: ln,
			Config:   &http.Server{Handler: m.raftHandler},
		}
		hs.Start()
		m.hss = append(m.hss, hs)
	}
	for _, ln := range m.ClientListeners {
		hs := &httptest.Server{
			Listener: ln,
			Config:   &http.Server{Handler: v2http.NewClientHandler(m.s, m.ServerConfig.ReqTimeout())},
		}
		hs.Start()
		m.hss = append(m.hss, hs)
	}
	return nil
}

// waitForEtcd wait until etcd is propagated correctly
func (m *EtcdTestServer) waitUntilUp() error {
	membersAPI := etcd.NewMembersAPI(m.Client)
	for start := time.Now(); time.Since(start) < wait.ForeverTestTimeout; time.Sleep(10 * time.Millisecond) {
		members, err := membersAPI.List(context.TODO())
		if err != nil {
			glog.Errorf("Error when getting etcd cluster members")
			continue
		}
		if len(members) == 1 && len(members[0].ClientURLs) > 0 {
			return nil
		}
	}
	return fmt.Errorf("timeout on waiting for etcd cluster")
}

// Terminate will shutdown the running etcd server
func (m *EtcdTestServer) Terminate(t *testing.T) {
	if m.v3Cluster != nil {
		m.v3Cluster.Terminate(t)
	} else {
		m.Client = nil
		m.s.Stop()
		// TODO: This is a pretty ugly hack to workaround races during closing
		// in-memory etcd server in unit tests - see #18928 for more details.
		// We should get rid of it as soon as we have a proper fix - etcd clients
		// have overwritten transport counting opened connections (probably by
		// overwriting Dial function) and termination function waiting for all
		// connections to be closed and stopping accepting new ones.
		time.Sleep(250 * time.Millisecond)
		for _, hs := range m.hss {
			hs.CloseClientConnections()
			hs.Close()
		}
		if err := os.RemoveAll(m.ServerConfig.DataDir); err != nil {
			t.Fatal(err)
		}
		if len(m.CertificatesDir) > 0 {
			if err := os.RemoveAll(m.CertificatesDir); err != nil {
				t.Fatal(err)
			}
		}
	}
}

// NewEtcdTestClientServer DEPRECATED creates a new client and server for testing
func NewEtcdTestClientServer(t *testing.T) *EtcdTestServer {
	server := configureTestCluster(t, "foo", true)
	err := server.launch(t)
	if err != nil {
		t.Fatalf("Failed to start etcd server error=%v", err)
		return nil
	}

	cfg := etcd.Config{
		Endpoints: server.ClientURLs.StringSlice(),
		Transport: newHttpTransport(t, server.CertFile, server.KeyFile, server.CAFile),
	}
	server.Client, err = etcd.New(cfg)
	if err != nil {
		server.Terminate(t)
		t.Fatalf("Unexpected error in NewEtcdTestClientServer (%v)", err)
		return nil
	}
	if err := server.waitUntilUp(); err != nil {
		server.Terminate(t)
		t.Fatalf("Unexpected error in waitUntilUp (%v)", err)
		return nil
	}
	return server
}

// NewUnsecuredEtcdTestClientServer DEPRECATED creates a new client and server for testing
func NewUnsecuredEtcdTestClientServer(t *testing.T) *EtcdTestServer {
	server := configureTestCluster(t, "foo", false)
	err := server.launch(t)
	if err != nil {
		t.Fatalf("Failed to start etcd server error=%v", err)
		return nil
	}
	cfg := etcd.Config{
		Endpoints: server.ClientURLs.StringSlice(),
		Transport: newHttpTransport(t, server.CertFile, server.KeyFile, server.CAFile),
	}
	server.Client, err = etcd.New(cfg)
	if err != nil {
		t.Errorf("Unexpected error in NewUnsecuredEtcdTestClientServer (%v)", err)
		server.Terminate(t)
		return nil
	}
	if err := server.waitUntilUp(); err != nil {
		t.Errorf("Unexpected error in waitUntilUp (%v)", err)
		server.Terminate(t)
		return nil
	}
	return server
}

// NewEtcd3TestClientServer creates a new client and server for testing
func NewUnsecuredEtcd3TestClientServer(t *testing.T, scheme *runtime.Scheme) (*EtcdTestServer, *storagebackend.Config) {
	server := &EtcdTestServer{
		v3Cluster: integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}),
	}
	server.V3Client = server.v3Cluster.RandClient()
	config := &storagebackend.Config{
		Type:                     "etcd3",
		Prefix:                   etcdtest.PathPrefix(),
		ServerList:               server.V3Client.Endpoints(),
		DeserializationCacheSize: etcdtest.DeserializationCacheSize,
		Copier: scheme,
	}
	return server, config
}
<commit_msg>UPSTREAM: <drop>: Adapt etcd testing util to v3.2.1<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package testing

import (
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httptest"
	"os"
	"path"
	"testing"
	"time"

	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/apiserver/pkg/storage/etcd/etcdtest"
	"k8s.io/apiserver/pkg/storage/etcd/testing/testingcert"
	"k8s.io/apiserver/pkg/storage/storagebackend"

	etcd "github.com/coreos/etcd/client"
	"github.com/coreos/etcd/clientv3"
	"github.com/coreos/etcd/etcdserver"
	"github.com/coreos/etcd/etcdserver/api/v2http"
	"github.com/coreos/etcd/integration"
	"github.com/coreos/etcd/pkg/testutil"
	"github.com/coreos/etcd/pkg/transport"
	"github.com/coreos/etcd/pkg/types"
	"github.com/golang/glog"
	"golang.org/x/net/context"
)

// EtcdTestServer encapsulates the datastructures needed to start local instance for testing
type EtcdTestServer struct {
	// The following are lumped etcd2 test server params
	// TODO: Deprecate in a post 1.5 release
	etcdserver.ServerConfig
	PeerListeners, ClientListeners []net.Listener
	Client                         etcd.Client

	CertificatesDir string
	CertFile        string
	KeyFile         string
	CAFile          string

	raftHandler http.Handler
	s           *etcdserver.EtcdServer
	hss         []*httptest.Server

	// The following are lumped etcd3 test server params
	v3Cluster *integration.ClusterV3
	V3Client  *clientv3.Client
}

// newLocalListener opens a port localhost using any port
func newLocalListener(t *testing.T) net.Listener {
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	return l
}

// newSecuredLocalListener opens a port localhost using any port
// with SSL enable
func newSecuredLocalListener(t *testing.T, certFile, keyFile, caFile string) net.Listener {
	var l net.Listener
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	tlsInfo := transport.TLSInfo{
		CertFile: certFile,
		KeyFile:  keyFile,
		CAFile:   caFile,
	}
	tlscfg, err := tlsInfo.ServerConfig()
	if err != nil {
		t.Fatalf("unexpected serverConfig error: %v", err)
	}
	l, err = transport.NewKeepAliveListener(l, "https", tlscfg)
	if err != nil {
		t.Fatal(err)
	}
	return l
}

func newHttpTransport(t *testing.T, certFile, keyFile, caFile string) etcd.CancelableTransport {
	tlsInfo := transport.TLSInfo{
		CertFile: certFile,
		KeyFile:  keyFile,
		CAFile:   caFile,
	}
	tr, err := transport.NewTransport(tlsInfo, time.Second)
	if err != nil {
		t.Fatal(err)
	}
	return tr
}

// configureTestCluster will set the params to start an etcd server
func configureTestCluster(t *testing.T, name string, https bool) *EtcdTestServer {
	var err error
	m := &EtcdTestServer{}

	pln := newLocalListener(t)
	m.PeerListeners = []net.Listener{pln}
	m.PeerURLs, err = types.NewURLs([]string{"http://" + pln.Addr().String()})
	if err != nil {
		t.Fatal(err)
	}

	// Allow test launches to control where etcd data goes, for space or performance reasons
	baseDir := os.Getenv("TEST_ETCD_DIR")
	if len(baseDir) == 0 {
		baseDir = os.TempDir()
	}

	if https {
		m.CertificatesDir, err = ioutil.TempDir(baseDir, "etcd_certificates")
		if err != nil {
			t.Fatal(err)
		}
		m.CertFile = path.Join(m.CertificatesDir, "etcdcert.pem")
		if err = ioutil.WriteFile(m.CertFile, []byte(testingcert.CertFileContent), 0644); err != nil {
			t.Fatal(err)
		}
		m.KeyFile = path.Join(m.CertificatesDir, "etcdkey.pem")
		if err = ioutil.WriteFile(m.KeyFile, []byte(testingcert.KeyFileContent), 0644); err != nil {
			t.Fatal(err)
		}
		m.CAFile = path.Join(m.CertificatesDir, "ca.pem")
		if err = ioutil.WriteFile(m.CAFile, []byte(testingcert.CAFileContent), 0644); err != nil {
			t.Fatal(err)
		}

		cln := newSecuredLocalListener(t, m.CertFile, m.KeyFile, m.CAFile)
		m.ClientListeners = []net.Listener{cln}
		m.ClientURLs, err = types.NewURLs([]string{"https://" + cln.Addr().String()})
		if err != nil {
			t.Fatal(err)
		}
	} else {
		cln := newLocalListener(t)
		m.ClientListeners = []net.Listener{cln}
		m.ClientURLs, err = types.NewURLs([]string{"http://" + cln.Addr().String()})
		if err != nil {
			t.Fatal(err)
		}
	}

	m.Name = name
	m.DataDir, err = ioutil.TempDir(baseDir, "etcd")
	if err != nil {
		t.Fatal(err)
	}
	m.AuthToken = "simple"

	clusterStr := fmt.Sprintf("%s=http://%s", name, pln.Addr().String())
	m.InitialPeerURLsMap, err = types.NewURLsMap(clusterStr)
	if err != nil {
		t.Fatal(err)
	}
	m.InitialClusterToken = "TestEtcd"
	m.NewCluster = true
	m.ForceNewCluster = false
	m.ElectionTicks = 10
	m.TickMs = uint(10)

	return m
}

// launch will attempt to start the etcd server
func (m *EtcdTestServer) launch(t *testing.T) error {
	var err error
	if m.s, err = etcdserver.NewServer(&m.ServerConfig); err != nil {
		return fmt.Errorf("failed to initialize the etcd server: %v", err)
	}
	m.s.SyncTicker = time.NewTicker(500 * time.Millisecond)
	m.s.Start()
	m.raftHandler = &testutil.PauseableHandler{Next: v2http.NewPeerHandler(m.s)}
	for _, ln := range m.PeerListeners {
		hs := &httptest.Server{
			Listener: ln,
			Config:   &http.Server{Handler: m.raftHandler},
		}
		hs.Start()
		m.hss = append(m.hss, hs)
	}
	for _, ln := range m.ClientListeners {
		hs := &httptest.Server{
			Listener: ln,
			Config:   &http.Server{Handler: v2http.NewClientHandler(m.s, m.ServerConfig.ReqTimeout())},
		}
		hs.Start()
		m.hss = append(m.hss, hs)
	}
	return nil
}

// waitForEtcd wait until etcd is propagated correctly
func (m *EtcdTestServer) waitUntilUp() error {
	membersAPI := etcd.NewMembersAPI(m.Client)
	for start := time.Now(); time.Since(start) < wait.ForeverTestTimeout; time.Sleep(10 * time.Millisecond) {
		members, err := membersAPI.List(context.TODO())
		if err != nil {
			glog.Errorf("Error when getting etcd cluster members")
			continue
		}
		if len(members) == 1 && len(members[0].ClientURLs) > 0 {
			return nil
		}
	}
	return fmt.Errorf("timeout on waiting for etcd cluster")
}

// Terminate will shutdown the running etcd server
func (m *EtcdTestServer) Terminate(t *testing.T) {
	if m.v3Cluster != nil {
		m.v3Cluster.Terminate(t)
	} else {
		m.Client = nil
		m.s.Stop()
		// TODO: This is a pretty ugly hack to workaround races during closing
		// in-memory etcd server in unit tests - see #18928 for more details.
		// We should get rid of it as soon as we have a proper fix - etcd clients
		// have overwritten transport counting opened connections (probably by
		// overwriting Dial function) and termination function waiting for all
		// connections to be closed and stopping accepting new ones.
		time.Sleep(250 * time.Millisecond)
		for _, hs := range m.hss {
			hs.CloseClientConnections()
			hs.Close()
		}
		if err := os.RemoveAll(m.ServerConfig.DataDir); err != nil {
			t.Fatal(err)
		}
		if len(m.CertificatesDir) > 0 {
			if err := os.RemoveAll(m.CertificatesDir); err != nil {
				t.Fatal(err)
			}
		}
	}
}

// NewEtcdTestClientServer DEPRECATED creates a new client and server for testing
func NewEtcdTestClientServer(t *testing.T) *EtcdTestServer {
	server := configureTestCluster(t, "foo", true)
	err := server.launch(t)
	if err != nil {
		t.Fatalf("Failed to start etcd server error=%v", err)
		return nil
	}

	cfg := etcd.Config{
		Endpoints: server.ClientURLs.StringSlice(),
		Transport: newHttpTransport(t, server.CertFile, server.KeyFile, server.CAFile),
	}
	server.Client, err = etcd.New(cfg)
	if err != nil {
		server.Terminate(t)
		t.Fatalf("Unexpected error in NewEtcdTestClientServer (%v)", err)
		return nil
	}
	if err := server.waitUntilUp(); err != nil {
		server.Terminate(t)
		t.Fatalf("Unexpected error in waitUntilUp (%v)", err)
		return nil
	}
	return server
}

// NewUnsecuredEtcdTestClientServer DEPRECATED creates a new client and server for testing
func NewUnsecuredEtcdTestClientServer(t *testing.T) *EtcdTestServer {
	server := configureTestCluster(t, "foo", false)
	err := server.launch(t)
	if err != nil {
		t.Fatalf("Failed to start etcd server error=%v", err)
		return nil
	}
	cfg := etcd.Config{
		Endpoints: server.ClientURLs.StringSlice(),
		Transport: newHttpTransport(t, server.CertFile, server.KeyFile, server.CAFile),
	}
	server.Client, err = etcd.New(cfg)
	if err != nil {
		t.Errorf("Unexpected error in NewUnsecuredEtcdTestClientServer (%v)", err)
		server.Terminate(t)
		return nil
	}
	if err := server.waitUntilUp(); err != nil {
		t.Errorf("Unexpected error in waitUntilUp (%v)", err)
		server.Terminate(t)
		return nil
	}
	return server
}

// NewEtcd3TestClientServer creates a new client and server for testing
func NewUnsecuredEtcd3TestClientServer(t *testing.T, scheme *runtime.Scheme) (*EtcdTestServer, *storagebackend.Config) {
	server := &EtcdTestServer{
		v3Cluster: integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}),
	}
	server.V3Client = server.v3Cluster.RandClient()
	config := &storagebackend.Config{
		Type:                     "etcd3",
		Prefix:                   etcdtest.PathPrefix(),
		ServerList:               server.V3Client.Endpoints(),
		DeserializationCacheSize: etcdtest.DeserializationCacheSize,
		Copier: scheme,
	}
	return server, config
}
<|endoftext|>


<commit_before>package unpacker

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"time"

	domlib "github.com/Symantec/Dominator/dom/lib"
	imageclient "github.com/Symantec/Dominator/imageserver/client"
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/filesystem/util"
	"github.com/Symantec/Dominator/lib/filter"
	"github.com/Symantec/Dominator/lib/format"
	"github.com/Symantec/Dominator/lib/fsutil"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/image"
	"github.com/Symantec/Dominator/lib/log"
	"github.com/Symantec/Dominator/lib/objectcache"
	"github.com/Symantec/Dominator/lib/objectserver"
	objectclient "github.com/Symantec/Dominator/lib/objectserver/client"
	"github.com/Symantec/Dominator/lib/srpc"
	unpackproto "github.com/Symantec/Dominator/proto/imageunpacker"
	subproto "github.com/Symantec/Dominator/proto/sub"
	sublib "github.com/Symantec/Dominator/sub/lib"
)

func (u *Unpacker) unpackImage(streamName string, imageLeafName string) error {
	u.updateUsageTime()
	defer u.updateUsageTime()
	streamInfo := u.getStream(streamName)
	if streamInfo == nil {
		return errors.New("unknown stream")
	}
	fs := u.getImage(filepath.Join(streamName, imageLeafName)).FileSystem
	if err := fs.RebuildInodePointers(); err != nil {
		return err
	}
	fs.InodeToFilenamesTable()
	fs.FilenameToInodeTable()
	fs.HashToInodesTable()
	fs.ComputeTotalDataBytes()
	fs.BuildEntryMap()
	errorChannel := make(chan error)
	request := requestType{
		request:      requestUnpack,
		desiredFS:    fs,
		imageName:    filepath.Join(streamName, imageLeafName),
		errorChannel: errorChannel,
	}
	streamInfo.requestChannel <- request
	return <-errorChannel
}

func (u *Unpacker) getImage(imageName string) *image.Image {
	u.logger.Printf("Getting image: %s\n", imageName)
	interval := time.Second
	for ; true; time.Sleep(interval) {
		srpcClient, err := srpc.DialHTTP("tcp", u.imageServerAddress,
			time.Second*15)
		if err != nil {
			u.logger.Printf("Error connecting to image server: %s\n", err)
			continue
		}
		image, err := imageclient.GetImageWithTimeout(srpcClient, imageName,
			time.Minute)
		srpcClient.Close()
		if err != nil {
			u.logger.Printf("Error getting image: %s\n", err)
			continue
		}
		if image != nil {
			return image
		}
		u.logger.Printf("Image: %s not ready yet\n", imageName)
		if interval < time.Second*10 {
			interval += time.Second
		}
	}
	return nil
}

func (stream *streamManagerState) unpack(imageName string,
	desiredFS *filesystem.FileSystem) error {
	srpcClient, err := srpc.DialHTTP("tcp", stream.unpacker.imageServerAddress,
		time.Second*15)
	if err != nil {
		return err
	}
	defer srpcClient.Close()
	objectServer := objectclient.AttachObjectClient(srpcClient)
	defer objectServer.Close()
	mountPoint := filepath.Join(stream.unpacker.baseDir, "mnt")
	streamInfo := stream.streamInfo
	switch streamInfo.status {
	case unpackproto.StatusStreamScanned:
		// Everything is set up. Ready to unpack.
	case unpackproto.StatusStreamNoFileSystem:
		err := stream.mkfs(desiredFS, objectServer, stream.unpacker.logger)
		if err != nil {
			return err
		}
		if err := stream.scan(false); err != nil {
			return err
		}
	default:
		return errors.New("not yet scanned")
	}
	err = stream.deleteUnneededFiles(imageName, stream.fileSystem, desiredFS,
		mountPoint)
	if err != nil {
		return err
	}
	subObj := domlib.Sub{
		FileSystem:  stream.fileSystem,
		ObjectCache: stream.objectCache,
	}
	stream.fileSystem = nil
	emptyFilter, _ := filter.New(nil)
	desiredImage := &image.Image{FileSystem: desiredFS, Filter: emptyFilter}
	fetchMap, _ := domlib.BuildMissingLists(subObj, desiredImage, false,
		true, stream.unpacker.logger)
	objectsToFetch := objectcache.ObjectMapToCache(fetchMap)
	objectsDir := filepath.Join(mountPoint, ".subd", "objects")
	err = stream.fetch(imageName, objectsToFetch, objectsDir, objectServer)
	if err != nil {
		streamInfo.status = unpackproto.StatusStreamMounted
		return err
	}
	subObj.ObjectCache = append(subObj.ObjectCache, objectsToFetch...)
	streamInfo.status = unpackproto.StatusStreamUpdating
	stream.unpacker.logger.Printf("Update(%s) starting\n", imageName)
	startTime := time.Now()
	var request subproto.UpdateRequest
	domlib.BuildUpdateRequest(subObj, desiredImage, &request, true, false,
		stream.unpacker.logger)
	_, _, err = sublib.Update(request, mountPoint, objectsDir, nil, nil, nil,
		stream.unpacker.logger)
	streamInfo.status = unpackproto.StatusStreamMounted
	stream.unpacker.logger.Printf("Update(%s) completed in %s\n",
		imageName, format.Duration(time.Since(startTime)))
	return err
}

func (stream *streamManagerState) deleteUnneededFiles(imageName string,
	subFS, imgFS *filesystem.FileSystem, mountPoint string) error {
	pathsToDelete := make([]string, 0)
	imgHashToInodesTable := imgFS.HashToInodesTable()
	imgFilenameToInodeTable := imgFS.FilenameToInodeTable()
	for pathname, inum := range subFS.FilenameToInodeTable() {
		if inode, ok := subFS.InodeTable[inum].(*filesystem.RegularInode); ok {
			if inode.Size > 0 {
				if _, ok := imgHashToInodesTable[inode.Hash]; !ok {
					pathsToDelete = append(pathsToDelete, pathname)
				}
			} else {
				if _, ok := imgFilenameToInodeTable[pathname]; !ok {
					pathsToDelete = append(pathsToDelete, pathname)
				}
			}
		}
	}
	if len(pathsToDelete) < 1 {
		return nil
	}
	stream.unpacker.logger.Printf("Deleting(%s): %d unneeded files\n",
		imageName, len(pathsToDelete))
	for _, pathname := range pathsToDelete {
		stream.unpacker.logger.Printf("Delete(%s): %s\n", imageName, pathname)
		os.Remove(filepath.Join(mountPoint, pathname))
	}
	return nil
}

func (stream *streamManagerState) fetch(imageName string,
	objectsToFetch []hash.Hash, destDirname string,
	objectsGetter objectserver.ObjectsGetter) error {
	startTime := time.Now()
	stream.streamInfo.status = unpackproto.StatusStreamFetching
	objectsReader, err := objectsGetter.GetObjects(objectsToFetch)
	if err != nil {
		stream.streamInfo.status = unpackproto.StatusStreamMounted
		return err
	}
	defer objectsReader.Close()
	stream.unpacker.logger.Printf("Fetching(%s) %d objects\n",
		imageName, len(objectsToFetch))
	for _, hashVal := range objectsToFetch {
		length, reader, err := objectsReader.NextObject()
		if err != nil {
			stream.unpacker.logger.Println(err)
			stream.streamInfo.status = unpackproto.StatusStreamMounted
			return err
		}
		err = readOne(destDirname, hashVal, length, reader)
		reader.Close()
		if err != nil {
			stream.unpacker.logger.Println(err)
			stream.streamInfo.status = unpackproto.StatusStreamMounted
			return err
		}
	}
	stream.unpacker.logger.Printf("Fetched(%s) %d objects in %s\n",
		imageName, len(objectsToFetch), format.Duration(time.Since(startTime)))
	return nil
}

func (stream *streamManagerState) mkfs(fs *filesystem.FileSystem,
	objectsGetter objectserver.ObjectsGetter, logger log.Logger) error {
	unsupportedOptions, err := util.GetUnsupportedExt4fsOptions(fs,
		objectsGetter)
	if err != nil {
		return err
	}
	stream.unpacker.rwMutex.RLock()
	device := stream.unpacker.pState.Devices[stream.streamInfo.DeviceId]
	stream.unpacker.rwMutex.RUnlock()
	// udev has a bug where the partition device node is created and sometimes
	// is removed and then created again. Based on experiments the device node
	// is gone for ~15 milliseconds. Wait long enough since the partition was
	// created to hopefully never encounter this race again.
	if !device.partitionTimestamp.IsZero() {
		timeSincePartition := time.Since(device.partitionTimestamp)
		if timeSincePartition < time.Second {
			sleepTime := time.Second - timeSincePartition
			logger.Printf("sleeping %s to work around udev race\n",
				format.Duration(sleepTime))
			time.Sleep(sleepTime)
		}
	}
	partitionPath, err := getPartition(filepath.Join("/dev", device.DeviceName))
	if err != nil {
		return err
	}
	rootLabel := fmt.Sprintf("rootfs@%x", time.Now().Unix())
	err = util.MakeExt4fs(partitionPath, rootLabel, unsupportedOptions, 8192,
		logger)
	if err != nil {
		return err
	}
	// Make sure it's still a block device. If not it means udev still had not
	// settled down after waiting, so remove the inode and return an error.
	if err := checkIfBlockDevice(partitionPath); err != nil {
		os.Remove(partitionPath)
		return err
	}
	stream.streamInfo.status = unpackproto.StatusStreamNotMounted
	stream.rootLabel = rootLabel
	return nil
}

func checkIfBlockDevice(path string) error {
	if fi, err := os.Lstat(path); err != nil {
		return err
	} else if fi.Mode()&os.ModeType != os.ModeDevice {
		return fmt.Errorf("%s is not a device, mode: %s", path, fi.Mode())
	}
	return nil
}

func getPartition(devicePath string) (string, error) {
	partitionPaths := []string{devicePath + "1", devicePath + "p1"}
	for _, partitionPath := range partitionPaths {
		if err := checkIfBlockDevice(partitionPath); err != nil {
			if os.IsNotExist(err) {
				continue
			}
			return "", err
		}
		if file, err := os.Open(partitionPath); err == nil {
			file.Close()
			return partitionPath, nil
		}
	}
	return "", fmt.Errorf("no partitions found for: %s", devicePath)
}

func readOne(objectsDir string, hashVal hash.Hash, length uint64,
	reader io.Reader) error {
	filename := filepath.Join(objectsDir, objectcache.HashToFilename(hashVal))
	dirname := filepath.Dir(filename)
	if err := os.MkdirAll(dirname, dirPerms); err != nil {
		return err
	}
	return fsutil.CopyToFile(filename, filePerms, reader, length)
}
<commit_msg>Write /var/log/unpacked-image in image-unpacker.<commit_after>package unpacker

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"time"

	domlib "github.com/Symantec/Dominator/dom/lib"
	imageclient "github.com/Symantec/Dominator/imageserver/client"
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/filesystem/util"
	"github.com/Symantec/Dominator/lib/filter"
	"github.com/Symantec/Dominator/lib/format"
	"github.com/Symantec/Dominator/lib/fsutil"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/image"
	"github.com/Symantec/Dominator/lib/log"
	"github.com/Symantec/Dominator/lib/objectcache"
	"github.com/Symantec/Dominator/lib/objectserver"
	objectclient "github.com/Symantec/Dominator/lib/objectserver/client"
	"github.com/Symantec/Dominator/lib/srpc"
	unpackproto "github.com/Symantec/Dominator/proto/imageunpacker"
	subproto "github.com/Symantec/Dominator/proto/sub"
	sublib "github.com/Symantec/Dominator/sub/lib"
)

func (u *Unpacker) unpackImage(streamName string, imageLeafName string) error {
	u.updateUsageTime()
	defer u.updateUsageTime()
	streamInfo := u.getStream(streamName)
	if streamInfo == nil {
		return errors.New("unknown stream")
	}
	fs := u.getImage(filepath.Join(streamName, imageLeafName)).FileSystem
	if err := fs.RebuildInodePointers(); err != nil {
		return err
	}
	fs.InodeToFilenamesTable()
	fs.FilenameToInodeTable()
	fs.HashToInodesTable()
	fs.ComputeTotalDataBytes()
	fs.BuildEntryMap()
	errorChannel := make(chan error)
	request := requestType{
		request:      requestUnpack,
		desiredFS:    fs,
		imageName:    filepath.Join(streamName, imageLeafName),
		errorChannel: errorChannel,
	}
	streamInfo.requestChannel <- request
	return <-errorChannel
}

func (u *Unpacker) getImage(imageName string) *image.Image {
	u.logger.Printf("Getting image: %s\n", imageName)
	interval := time.Second
	for ; true; time.Sleep(interval) {
		srpcClient, err := srpc.DialHTTP("tcp", u.imageServerAddress,
			time.Second*15)
		if err != nil {
			u.logger.Printf("Error connecting to image server: %s\n", err)
			continue
		}
		image, err := imageclient.GetImageWithTimeout(srpcClient, imageName,
			time.Minute)
		srpcClient.Close()
		if err != nil {
			u.logger.Printf("Error getting image: %s\n", err)
			continue
		}
		if image != nil {
			return image
		}
		u.logger.Printf("Image: %s not ready yet\n", imageName)
		if interval < time.Second*10 {
			interval += time.Second
		}
	}
	return nil
}

func (stream *streamManagerState) unpack(imageName string,
	desiredFS *filesystem.FileSystem) error {
	srpcClient, err := srpc.DialHTTP("tcp", stream.unpacker.imageServerAddress,
		time.Second*15)
	if err != nil {
		return err
	}
	defer srpcClient.Close()
	objectServer := objectclient.AttachObjectClient(srpcClient)
	defer objectServer.Close()
	mountPoint := filepath.Join(stream.unpacker.baseDir, "mnt")
	streamInfo := stream.streamInfo
	switch streamInfo.status {
	case unpackproto.StatusStreamScanned:
		// Everything is set up. Ready to unpack.
	case unpackproto.StatusStreamNoFileSystem:
		err := stream.mkfs(desiredFS, objectServer, stream.unpacker.logger)
		if err != nil {
			return err
		}
		if err := stream.scan(false); err != nil {
			return err
		}
	default:
		return errors.New("not yet scanned")
	}
	err = stream.deleteUnneededFiles(imageName, stream.fileSystem, desiredFS,
		mountPoint)
	if err != nil {
		return err
	}
	subObj := domlib.Sub{
		FileSystem:  stream.fileSystem,
		ObjectCache: stream.objectCache,
	}
	stream.fileSystem = nil
	emptyFilter, _ := filter.New(nil)
	desiredImage := &image.Image{FileSystem: desiredFS, Filter: emptyFilter}
	fetchMap, _ := domlib.BuildMissingLists(subObj, desiredImage, false,
		true, stream.unpacker.logger)
	objectsToFetch := objectcache.ObjectMapToCache(fetchMap)
	objectsDir := filepath.Join(mountPoint, ".subd", "objects")
	err = stream.fetch(imageName, objectsToFetch, objectsDir, objectServer)
	if err != nil {
		streamInfo.status = unpackproto.StatusStreamMounted
		return err
	}
	subObj.ObjectCache = append(subObj.ObjectCache, objectsToFetch...)
	streamInfo.status = unpackproto.StatusStreamUpdating
	stream.unpacker.logger.Printf("Update(%s) starting\n", imageName)
	startTime := time.Now()
	var request subproto.UpdateRequest
	domlib.BuildUpdateRequest(subObj, desiredImage, &request, true, false,
		stream.unpacker.logger)
	_, _, err = sublib.Update(request, mountPoint, objectsDir, nil, nil, nil,
		stream.unpacker.logger)
	writeImageName(imageName, mountPoint)
	streamInfo.status = unpackproto.StatusStreamMounted
	stream.unpacker.logger.Printf("Update(%s) completed in %s\n",
		imageName, format.Duration(time.Since(startTime)))
	return err
}

func (stream *streamManagerState) deleteUnneededFiles(imageName string,
	subFS, imgFS *filesystem.FileSystem, mountPoint string) error {
	pathsToDelete := make([]string, 0)
	imgHashToInodesTable := imgFS.HashToInodesTable()
	imgFilenameToInodeTable := imgFS.FilenameToInodeTable()
	for pathname, inum := range subFS.FilenameToInodeTable() {
		if inode, ok := subFS.InodeTable[inum].(*filesystem.RegularInode); ok {
			if inode.Size > 0 {
				if _, ok := imgHashToInodesTable[inode.Hash]; !ok {
					pathsToDelete = append(pathsToDelete, pathname)
				}
			} else {
				if _, ok := imgFilenameToInodeTable[pathname]; !ok {
					pathsToDelete = append(pathsToDelete, pathname)
				}
			}
		}
	}
	if len(pathsToDelete) < 1 {
		return nil
	}
	stream.unpacker.logger.Printf("Deleting(%s): %d unneeded files\n",
		imageName, len(pathsToDelete))
	for _, pathname := range pathsToDelete {
		stream.unpacker.logger.Printf("Delete(%s): %s\n", imageName, pathname)
		os.Remove(filepath.Join(mountPoint, pathname))
	}
	return nil
}

func (stream *streamManagerState) fetch(imageName string,
	objectsToFetch []hash.Hash, destDirname string,
	objectsGetter objectserver.ObjectsGetter) error {
	startTime := time.Now()
	stream.streamInfo.status = unpackproto.StatusStreamFetching
	objectsReader, err := objectsGetter.GetObjects(objectsToFetch)
	if err != nil {
		stream.streamInfo.status = unpackproto.StatusStreamMounted
		return err
	}
	defer objectsReader.Close()
	stream.unpacker.logger.Printf("Fetching(%s) %d objects\n",
		imageName, len(objectsToFetch))
	for _, hashVal := range objectsToFetch {
		length, reader, err := objectsReader.NextObject()
		if err != nil {
			stream.unpacker.logger.Println(err)
			stream.streamInfo.status = unpackproto.StatusStreamMounted
			return err
		}
		err = readOne(destDirname, hashVal, length, reader)
		reader.Close()
		if err != nil {
			stream.unpacker.logger.Println(err)
			stream.streamInfo.status = unpackproto.StatusStreamMounted
			return err
		}
	}
	stream.unpacker.logger.Printf("Fetched(%s) %d objects in %s\n",
		imageName, len(objectsToFetch), format.Duration(time.Since(startTime)))
	return nil
}

func (stream *streamManagerState) mkfs(fs *filesystem.FileSystem,
	objectsGetter objectserver.ObjectsGetter, logger log.Logger) error {
	unsupportedOptions, err := util.GetUnsupportedExt4fsOptions(fs,
		objectsGetter)
	if err != nil {
		return err
	}
	stream.unpacker.rwMutex.RLock()
	device := stream.unpacker.pState.Devices[stream.streamInfo.DeviceId]
	stream.unpacker.rwMutex.RUnlock()
	// udev has a bug where the partition device node is created and sometimes
	// is removed and then created again. Based on experiments the device node
	// is gone for ~15 milliseconds. Wait long enough since the partition was
	// created to hopefully never encounter this race again.
	if !device.partitionTimestamp.IsZero() {
		timeSincePartition := time.Since(device.partitionTimestamp)
		if timeSincePartition < time.Second {
			sleepTime := time.Second - timeSincePartition
			logger.Printf("sleeping %s to work around udev race\n",
				format.Duration(sleepTime))
			time.Sleep(sleepTime)
		}
	}
	partitionPath, err := getPartition(filepath.Join("/dev", device.DeviceName))
	if err != nil {
		return err
	}
	rootLabel := fmt.Sprintf("rootfs@%x", time.Now().Unix())
	err = util.MakeExt4fs(partitionPath, rootLabel, unsupportedOptions, 8192,
		logger)
	if err != nil {
		return err
	}
	// Make sure it's still a block device. If not it means udev still had not
	// settled down after waiting, so remove the inode and return an error.
	if err := checkIfBlockDevice(partitionPath); err != nil {
		os.Remove(partitionPath)
		return err
	}
	stream.streamInfo.status = unpackproto.StatusStreamNotMounted
	stream.rootLabel = rootLabel
	return nil
}

func checkIfBlockDevice(path string) error {
	if fi, err := os.Lstat(path); err != nil {
		return err
	} else if fi.Mode()&os.ModeType != os.ModeDevice {
		return fmt.Errorf("%s is not a device, mode: %s", path, fi.Mode())
	}
	return nil
}

func getPartition(devicePath string) (string, error) {
	partitionPaths := []string{devicePath + "1", devicePath + "p1"}
	for _, partitionPath := range partitionPaths {
		if err := checkIfBlockDevice(partitionPath); err != nil {
			if os.IsNotExist(err) {
				continue
			}
			return "", err
		}
		if file, err := os.Open(partitionPath); err == nil {
			file.Close()
			return partitionPath, nil
		}
	}
	return "", fmt.Errorf("no partitions found for: %s", devicePath)
}

func readOne(objectsDir string, hashVal hash.Hash, length uint64,
	reader io.Reader) error {
	filename := filepath.Join(objectsDir, objectcache.HashToFilename(hashVal))
	dirname := filepath.Dir(filename)
	if err := os.MkdirAll(dirname, dirPerms); err != nil {
		return err
	}
	return fsutil.CopyToFile(filename, filePerms, reader, length)
}

func writeImageName(imageName, mountPoint string) {
	dirname := filepath.Join(mountPoint, "var", "log")
	if err := os.MkdirAll(dirname, fsutil.DirPerms); err != nil {
		return
	}
	buffer := &bytes.Buffer{}
	fmt.Fprintln(buffer, imageName)
	fsutil.CopyToFile(filepath.Join(dirname, "unpacked-image"),
		fsutil.PublicFilePerms, buffer, 0)
}
<|endoftext|>


<commit_before>package gencorpus

import (
	"context"
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"runtime"
	"sync/atomic"

	"github.com/google/subcommands"
	"github.com/nelhage/taktician/ai"
	"github.com/nelhage/taktician/ptn"
	"github.com/nelhage/taktician/tak"
	"golang.org/x/sync/errgroup"
)

type Command struct {
	seed int64
	size int

	games int

	epsilon float64
	depth   int
	threads int

	stats  bool
	output string
}

func (*Command) Name() string     { return "gencorpus" }
func (*Command) Synopsis() string { return "Generate a corpus of 3x3 positions" }
func (*Command) Usage() string {
	return `gencorpus [flags]
`
}

func (c *Command) SetFlags(flags *flag.FlagSet) {
	flags.IntVar(&c.size, "size", 3, "what size to analyze")
	flags.IntVar(&c.games, "games", 100, "games to generate")
	flags.Int64Var(&c.seed, "seed", 0, "Random seed")
	flags.IntVar(&c.threads, "threads", runtime.NumCPU(), "Number of threads")

	flags.BoolVar(&c.stats, "stats", false, "compute and print stats")
	flags.IntVar(&c.depth, "depth", 2, "minimax depth")
	flags.Float64Var(&c.epsilon, "epsilon", 0.95, "epsilon for epsilon-greedy generation")

	flags.StringVar(&c.output, "output", "positions.txt", "output file")

}

type game struct {
	positions []*tak.Position
	moves     []tak.Move
}

func growslice[T any](sl []T, newlen int) []T {
	if len(sl) >= newlen {
		return sl
	}
	newsl := make([]T, newlen)
	copy(newsl, sl)
	return newsl
}

func (c *Command) Execute(ctx context.Context, flag *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
	var byLength []int
	var posCount []map[uint64]int

	games := make(chan *game)
	go c.generateGames(ctx, games)

	var gameList []*game

	for g := range games {
		gameList = append(gameList, g)
	}
	if c.stats {
		for _, g := range gameList {
			moves := len(g.positions)
			byLength = growslice(byLength, moves)
			byLength[moves-1] += 1
			posCount = growslice(posCount, moves+1)
			for i, p := range g.positions {
				if posCount[i] == nil {
					posCount[i] = make(map[uint64]int)
				}
				posCount[i][p.Hash()] += 1
			}
		}
		for i := range byLength {
			log.Printf("ply=%3d games=%3d uniq=%4d", i, byLength[i], len(posCount[i]))
		}
	}

	rng := rand.New(rand.NewSource(c.seed))

	positions := make(map[uint64]*tak.Position)

	for _, g := range gameList {
		// select position
		var idx int
		r := rng.Float64()
		if r < 0.01 {
			idx = int(rng.Int31n(4))
		} else if r < 0.25 {
			idx = 4 + int(rng.Int31n(5))
		} else if r < 0.95 {
			npos := len(g.positions)
			if npos <= 9 {
				continue
			}
			idx = 9 + int(rng.Int31n(int32(npos)-9))
		}
		if idx >= len(g.positions) {
			continue
		}
		pos := g.positions[idx]
		positions[pos.Hash()] = pos
	}

	fh, err := os.Create(c.output)
	if err != nil {
		log.Printf("open %q: %s", c.output, err.Error())
		return subcommands.ExitFailure
	}
	defer fh.Close()
	for _, p := range positions {
		fmt.Fprintf(fh, "%s\n", ptn.FormatTPS(p))
	}

	return subcommands.ExitSuccess
}

func (c *Command) generateGames(ctx context.Context, games chan<- *game) {
	defer close(games)
	todo := int64(c.games)

	grp, ctx := errgroup.WithContext(ctx)
	for i := 0; i < c.threads; i++ {
		grp.Go(func() error {
			for {
				id := atomic.AddInt64(&todo, -1)
				if id <= 0 {
					break
				}
				games <- c.generateOne(ctx, id)
			}
			return nil
		})
	}
	grp.Wait()
}

const prime = 1099511628211

func (c *Command) generateOne(ctx context.Context, id int64) *game {
	rng := rand.New(rand.NewSource(prime*c.seed + id))
	mm := ai.NewMinimax(ai.MinimaxConfig{
		Size:  c.size,
		Seed:  rng.Int63(),
		Depth: c.depth,
	})
	rnd := ai.NewRandom(rng.Int63())

	pos := tak.New(tak.Config{Size: c.size})
	g := game{positions: []*tak.Position{pos}}
	for {
		if done, _ := pos.GameOver(); done {
			break
		}
		var player ai.TakPlayer
		if rng.Float64() < c.epsilon {
			player = rnd
		} else {
			player = mm
		}

		for {
			m := player.GetMove(ctx, pos)
			child, err := pos.Move(m)
			if err != nil {
				continue
			}
			g.positions = append(g.positions, child)
			g.moves = append(g.moves, m)
			pos = child
			break
		}
	}
	return &g
}
<commit_msg>don't build the ai in the inner loop<commit_after>package gencorpus

import (
	"context"
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"runtime"
	"sync/atomic"

	"github.com/google/subcommands"
	"github.com/nelhage/taktician/ai"
	"github.com/nelhage/taktician/ptn"
	"github.com/nelhage/taktician/tak"
	"golang.org/x/sync/errgroup"
)

type Command struct {
	seed int64
	size int

	games int

	epsilon float64
	depth   int
	threads int

	stats  bool
	output string
}

func (*Command) Name() string     { return "gencorpus" }
func (*Command) Synopsis() string { return "Generate a corpus of 3x3 positions" }
func (*Command) Usage() string {
	return `gencorpus [flags]
`
}

func (c *Command) SetFlags(flags *flag.FlagSet) {
	flags.IntVar(&c.size, "size", 3, "what size to analyze")
	flags.IntVar(&c.games, "games", 100, "games to generate")
	flags.Int64Var(&c.seed, "seed", 0, "Random seed")
	flags.IntVar(&c.threads, "threads", runtime.NumCPU(), "Number of threads")

	flags.BoolVar(&c.stats, "stats", false, "compute and print stats")
	flags.IntVar(&c.depth, "depth", 2, "minimax depth")
	flags.Float64Var(&c.epsilon, "epsilon", 0.95, "epsilon for epsilon-greedy generation")

	flags.StringVar(&c.output, "output", "positions.txt", "output file")

}

type game struct {
	positions []*tak.Position
	moves     []tak.Move
}

func growslice[T any](sl []T, newlen int) []T {
	if len(sl) >= newlen {
		return sl
	}
	newsl := make([]T, newlen)
	copy(newsl, sl)
	return newsl
}

func (c *Command) Execute(ctx context.Context, flag *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
	var byLength []int
	var posCount []map[uint64]int

	games := make(chan *game)
	go c.generateGames(ctx, games)

	var gameList []*game

	for g := range games {
		gameList = append(gameList, g)
	}
	if c.stats {
		for _, g := range gameList {
			moves := len(g.positions)
			byLength = growslice(byLength, moves)
			byLength[moves-1] += 1
			posCount = growslice(posCount, moves+1)
			for i, p := range g.positions {
				if posCount[i] == nil {
					posCount[i] = make(map[uint64]int)
				}
				posCount[i][p.Hash()] += 1
			}
		}
		for i := range byLength {
			log.Printf("ply=%3d games=%3d uniq=%4d", i, byLength[i], len(posCount[i]))
		}
	}

	rng := rand.New(rand.NewSource(c.seed))

	positions := make(map[uint64]*tak.Position)

	for _, g := range gameList {
		// select position
		var idx int
		r := rng.Float64()
		if r < 0.01 {
			idx = int(rng.Int31n(4))
		} else if r < 0.25 {
			idx = 4 + int(rng.Int31n(5))
		} else if r < 0.95 {
			npos := len(g.positions)
			if npos <= 9 {
				continue
			}
			idx = 9 + int(rng.Int31n(int32(npos)-9))
		}
		if idx >= len(g.positions) {
			continue
		}
		pos := g.positions[idx]
		positions[pos.Hash()] = pos
	}

	fh, err := os.Create(c.output)
	if err != nil {
		log.Printf("open %q: %s", c.output, err.Error())
		return subcommands.ExitFailure
	}
	defer fh.Close()
	for _, p := range positions {
		fmt.Fprintf(fh, "%s\n", ptn.FormatTPS(p))
	}

	return subcommands.ExitSuccess
}

func (c *Command) generateGames(ctx context.Context, games chan<- *game) {
	defer close(games)
	todo := int64(c.games)

	grp, ctx := errgroup.WithContext(ctx)
	for i := 0; i < c.threads; i++ {
		grp.Go(func() error {
			c.generateWorker(ctx, games, &todo, i)
			return nil
		})
	}
	grp.Wait()
}

const prime = 1099511628211

func (c *Command) generateWorker(ctx context.Context, games chan<- *game, todo *int64, id int) {
	rng := rand.New(rand.NewSource(prime*c.seed + int64(id)))
	mm := ai.NewMinimax(ai.MinimaxConfig{
		Size:  c.size,
		Seed:  rng.Int63(),
		Depth: c.depth,
	})
	rnd := ai.NewRandom(rng.Int63())
	for {
		gid := atomic.AddInt64(todo, -1)
		if gid < 0 {
			return
		}
		pos := tak.New(tak.Config{Size: c.size})
		g := game{positions: []*tak.Position{pos}}
		for {
			if done, _ := pos.GameOver(); done {
				break
			}
			var player ai.TakPlayer
			if rng.Float64() < c.epsilon {
				player = rnd
			} else {
				player = mm
			}

			for {
				m := player.GetMove(ctx, pos)
				child, err := pos.Move(m)
				if err != nil {
					continue
				}
				g.positions = append(g.positions, child)
				g.moves = append(g.moves, m)
				pos = child
				break
			}
		}
		games <- &g
	}
}
<|endoftext|>


<commit_before>/*
   Copyright 2019 Artem Sidorenko <artem@posteo.de>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

// Package cli provides some helper functions for github.com/urfave/cli
package cli

import (
	"fmt"

	"github.com/urfave/cli"
)

// OnUsageError represents a workaround for https://github.com/urfave/cli/issues/610
// once there is a fix upstream, this can be removed
func OnUsageError(context *cli.Context, err error, isSubcommand bool) error {
	fmt.Fprintf( // nolint: errcheck
		cli.ErrWriter, "%s - %s %s\n\n",
		context.App.Name, "incorrect usage:", err.Error(),
	)
	cli.ShowAppHelp(context) // nolint: gosec, errcheck
	return cli.NewExitError("", 1)
}

// ExitErrHandler implements cli.ExitErrHandlerFunc
// we make it simple, we always return exit code 1
func ExitErrHandler(_ *cli.Context, err error) {
	fmt.Fprintf(cli.ErrWriter, "Error: %+v\n", err) // nolint: errcheck
	cli.OsExiter(1)
}
<commit_msg>Bugfix: don't print error if there is no text<commit_after>/*
   Copyright 2019 Artem Sidorenko <artem@posteo.de>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

// Package cli provides some helper functions for github.com/urfave/cli
package cli

import (
	"fmt"

	"github.com/urfave/cli"
)

// OnUsageError represents a workaround for https://github.com/urfave/cli/issues/610
// once there is a fix upstream, this can be removed
func OnUsageError(context *cli.Context, err error, isSubcommand bool) error {
	fmt.Fprintf( // nolint: errcheck
		cli.ErrWriter, "%s - %s %s\n\n",
		context.App.Name, "incorrect usage:", err.Error(),
	)
	cli.ShowAppHelp(context) // nolint: gosec, errcheck
	return cli.NewExitError("", 1)
}

// ExitErrHandler implements cli.ExitErrHandlerFunc
// we make it simple, we always return exit code 1
func ExitErrHandler(_ *cli.Context, err error) {
	if err.Error() != "" {
		fmt.Fprintf(cli.ErrWriter, "Error: %+v\n", err) // nolint: errcheck
	}
	cli.OsExiter(1)
}
<|endoftext|>


<commit_before>package pgghelpers

import (
	"encoding/json"
	"fmt"
	"regexp"
	"strings"
	"text/template"

	"github.com/Masterminds/sprig"
	"github.com/golang/protobuf/proto"
	"github.com/golang/protobuf/protoc-gen-go/descriptor"
	ggdescriptor "github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway/descriptor"
	"github.com/huandu/xstrings"
	options "google.golang.org/genproto/googleapis/api/annotations"
)

var jsReservedRe *regexp.Regexp = regexp.MustCompile(`(^|[^A-Za-z])(do|if|in|for|let|new|try|var|case|else|enum|eval|false|null|this|true|void|with|break|catch|class|const|super|throw|while|yield|delete|export|import|public|return|static|switch|typeof|default|extends|finally|package|private|continue|debugger|function|arguments|interface|protected|implements|instanceof)($|[^A-Za-z])`)

var (
	registry *ggdescriptor.Registry // some helpers need access to registry
)

func SetRegistry(reg *ggdescriptor.Registry) {
	registry = reg
}

var ProtoHelpersFuncMap = template.FuncMap{
	"string": func(i interface {
		String() string
	}) string {
		return i.String()
	},
	"json": func(v interface{}) string {
		a, _ := json.Marshal(v)
		return string(a)
	},
	"prettyjson": func(v interface{}) string {
		a, _ := json.MarshalIndent(v, "", "  ")
		return string(a)
	},
	"splitArray": func(sep string, s string) []string {
		return strings.Split(s, sep)
	},
	"first": func(a []string) string {
		return a[0]
	},
	"last": func(a []string) string {
		return a[len(a)-1]
	},
	"upperFirst": func(s string) string {
		return strings.ToUpper(s[:1]) + s[1:]
	},
	"lowerFirst": func(s string) string {
		return strings.ToLower(s[:1]) + s[1:]
	},
	"camelCase": func(s string) string {
		if len(s) > 1 {
			return xstrings.ToCamelCase(s)
		}

		return strings.ToUpper(s[:1])
	},
	"lowerCamelCase": func(s string) string {
		if len(s) > 1 {
			s = xstrings.ToCamelCase(s)
		}

		return strings.ToLower(s[:1]) + s[1:]
	},
	"kebabCase": func(s string) string {
		return strings.Replace(xstrings.ToSnakeCase(s), "_", "-", -1)
	},
	"snakeCase":             xstrings.ToSnakeCase,
	"getProtoFile":          getProtoFile,
	"getMessageType":        getMessageType,
	"getEnumValue":          getEnumValue,
	"isFieldMessage":        isFieldMessage,
	"isFieldRepeated":       isFieldRepeated,
	"goType":                goType,
	"goTypeWithPackage":     goTypeWithPackage,
	"jsType":                jsType,
	"jsSuffixReserved":      jsSuffixReservedKeyword,
	"namespacedFlowType":    namespacedFlowType,
	"httpVerb":              httpVerb,
	"httpPath":              httpPath,
	"shortType":             shortType,
	"urlHasVarsFromMessage": urlHasVarsFromMessage,
}

func init() {
	for k, v := range sprig.TxtFuncMap() {
		ProtoHelpersFuncMap[k] = v
	}
}

func getProtoFile(name string) *ggdescriptor.File {
	if registry == nil {
		return nil
	}
	file, err := registry.LookupFile(name)
	if err != nil {
		panic(err)
	}
	return file
}

func getMessageType(f *descriptor.FileDescriptorProto, name string) *ggdescriptor.Message {
	if registry != nil {
		msg, err := registry.LookupMsg(".", name)
		if err != nil {
			panic(err)
		}
		return msg
	}

	// name is in the form .packageName.MessageTypeName.InnerMessageTypeName...
	// e.g. .article.ProductTag
	splits := strings.Split(name, ".")
	target := splits[len(splits)-1]
	for _, m := range f.MessageType {
		if target == *m.Name {
			return &ggdescriptor.Message{
				DescriptorProto: m,
			}
		}
	}
	return nil
}

func getEnumValue(f []*descriptor.EnumDescriptorProto, name string) []*descriptor.EnumValueDescriptorProto {
	for _, item := range f {
		if strings.EqualFold(*item.Name, name) {
			return item.GetValue()
		}
	}

	return nil
}

func isFieldMessage(f *descriptor.FieldDescriptorProto) bool {
	if f.Type != nil && *f.Type == descriptor.FieldDescriptorProto_TYPE_MESSAGE {
		return true
	}

	return false
}

func isFieldRepeated(f *descriptor.FieldDescriptorProto) bool {
	if f.Type != nil && f.Label != nil && *f.Label == descriptor.FieldDescriptorProto_LABEL_REPEATED {
		return true
	}

	return false
}

func goTypeWithPackage(f *descriptor.FieldDescriptorProto) string {
	pkg := ""
	if *f.Type == descriptor.FieldDescriptorProto_TYPE_MESSAGE {
		pkg = getPackageTypeName(*f.TypeName)
	}
	return goType(pkg, f)
}

func goType(pkg string, f *descriptor.FieldDescriptorProto) string {
	switch *f.Type {
	case descriptor.FieldDescriptorProto_TYPE_DOUBLE:
		if *f.Label == descriptor.FieldDescriptorProto_LABEL_REPEATED {
			return "[]float64"
		}
		return "float64"
	case descriptor.FieldDescriptorProto_TYPE_FLOAT:
		if *f.Label == descriptor.FieldDescriptorProto_LABEL_REPEATED {
			return "[]float32"
		}
		return "float32"
	case descriptor.FieldDescriptorProto_TYPE_INT64:
		if *f.Label == descriptor.FieldDescriptorProto_LABEL_REPEATED {
			return "[]int64"
		}
		return "int64"
	case descriptor.FieldDescriptorProto_TYPE_UINT64:
		if *f.Label == descriptor.FieldDescriptorProto_LABEL_REPEATED {
			return "[]uint64"
		}
		return "uint64"
	case descriptor.FieldDescriptorProto_TYPE_INT32:
		if *f.Label == descriptor.FieldDescriptorProto_LABEL_REPEATED {
			return "[]int32"
		}
		return "int32"
	case descriptor.FieldDescriptorProto_TYPE_UINT32:
		if *f.Label == descriptor.FieldDescriptorProto_LABEL_REPEATED {
			return "[]uint32"
		}
		return "uint32"
	case descriptor.FieldDescriptorProto_TYPE_BOOL:
		if *f.Label == descriptor.FieldDescriptorProto_LABEL_REPEATED {
			return "[]bool"
		}
		return "bool"
	case descriptor.FieldDescriptorProto_TYPE_STRING:
		if *f.Label == descriptor.FieldDescriptorProto_LABEL_REPEATED {
			return "[]string"
		}
		return "string"
	case descriptor.FieldDescriptorProto_TYPE_MESSAGE:
		if pkg != "" {
			pkg = pkg + "."
		}
		if *f.Label == descriptor.FieldDescriptorProto_LABEL_REPEATED {
			return fmt.Sprintf("[]*%s%s", pkg, shortType(*f.TypeName))
		}
		return fmt.Sprintf("*%s%s", pkg, shortType(*f.TypeName))
	case descriptor.FieldDescriptorProto_TYPE_BYTES:
		if *f.Label == descriptor.FieldDescriptorProto_LABEL_REPEATED {
			return "[]byte"
		}
		return "byte"
	case descriptor.FieldDescriptorProto_TYPE_ENUM:
		return fmt.Sprintf("*%s.%s", pkg, shortType(*f.TypeName))
	default:
		return "interface{}"
	}
}

func jsType(f *descriptor.FieldDescriptorProto) string {
	template := "%s"
	if isFieldRepeated(f) == true {
		template = "Array<%s>"
	}

	switch *f.Type {
	case descriptor.FieldDescriptorProto_TYPE_MESSAGE,
		descriptor.FieldDescriptorProto_TYPE_ENUM:
		return fmt.Sprintf(template, namespacedFlowType(*f.TypeName))
	case descriptor.FieldDescriptorProto_TYPE_DOUBLE,
		descriptor.FieldDescriptorProto_TYPE_FLOAT,
		descriptor.FieldDescriptorProto_TYPE_INT64,
		descriptor.FieldDescriptorProto_TYPE_UINT64,
		descriptor.FieldDescriptorProto_TYPE_INT32,
		descriptor.FieldDescriptorProto_TYPE_FIXED64,
		descriptor.FieldDescriptorProto_TYPE_FIXED32,
		descriptor.FieldDescriptorProto_TYPE_UINT32,
		descriptor.FieldDescriptorProto_TYPE_SFIXED32,
		descriptor.FieldDescriptorProto_TYPE_SFIXED64,
		descriptor.FieldDescriptorProto_TYPE_SINT32,
		descriptor.FieldDescriptorProto_TYPE_SINT64:
		return fmt.Sprintf(template, "number")
	case descriptor.FieldDescriptorProto_TYPE_BOOL:
		return fmt.Sprintf(template, "boolean")
	case descriptor.FieldDescriptorProto_TYPE_BYTES:
		return fmt.Sprintf(template, "Uint8Array")
	case descriptor.FieldDescriptorProto_TYPE_STRING:
		return fmt.Sprintf(template, "string")
	default:
		return fmt.Sprintf(template, "any")
	}
}

func jsSuffixReservedKeyword(s string) string {
	return jsReservedRe.ReplaceAllString(s, "${1}${2}_${3}")
}

func getPackageTypeName(s string) string {
	if strings.Compare(s, ".google.protobuf.Timestamp") == 0 {
		return "timestamp"
	}
	if strings.Contains(s, ".") {
		return strings.Split(s, ".")[1]
	}
	return ""
}

func shortType(s string) string {
	t := strings.Split(s, ".")
	return t[len(t)-1]
}

func namespacedFlowType(s string) string {
	trimmed := strings.TrimLeft(s, ".")
	splitted := strings.Split(trimmed, ".")
	return strings.Join(splitted, "$")
}

func httpPath(m *descriptor.MethodDescriptorProto) string {

	ext, err := proto.GetExtension(m.Options, options.E_Http)
	if err != nil {
		return err.Error()
	}
	opts, ok := ext.(*options.HttpRule)
	if !ok {
		return fmt.Sprintf("extension is %T; want an HttpRule", ext)
	}

	switch t := opts.Pattern.(type) {
	default:
		return ""
	case *options.HttpRule_Get:
		return t.Get
	case *options.HttpRule_Post:
		return t.Post
	case *options.HttpRule_Put:
		return t.Put
	case *options.HttpRule_Delete:
		return t.Delete
	case *options.HttpRule_Patch:
		return t.Patch
	case *options.HttpRule_Custom:
		return t.Custom.Path
	}
}

func httpVerb(m *descriptor.MethodDescriptorProto) string {

	ext, err := proto.GetExtension(m.Options, options.E_Http)
	if err != nil {
		return err.Error()
	}
	opts, ok := ext.(*options.HttpRule)
	if !ok {
		return fmt.Sprintf("extension is %T; want an HttpRule", ext)
	}

	switch t := opts.Pattern.(type) {
	default:
		return ""
	case *options.HttpRule_Get:
		return "GET"
	case *options.HttpRule_Post:
		return "POST"
	case *options.HttpRule_Put:
		return "PUT"
	case *options.HttpRule_Delete:
		return "DELETE"
	case *options.HttpRule_Patch:
		return "PATCH"
	case *options.HttpRule_Custom:
		return t.Custom.Kind
	}
}

func urlHasVarsFromMessage(path string, d *ggdescriptor.Message) bool {
	for _, field := range d.Field {
		if !isFieldMessage(field) {
			if strings.Contains(path, fmt.Sprintf("{%s}", *field.Name)) {
				return true
			}
		}
	}

	return false
}
<commit_msg>fix (helper): splitArray helper does not return emtpy string<commit_after>package pgghelpers

import (
	"encoding/json"
	"fmt"
	"regexp"
	"strings"
	"text/template"

	"github.com/Masterminds/sprig"
	"github.com/golang/protobuf/proto"
	"github.com/golang/protobuf/protoc-gen-go/descriptor"
	ggdescriptor "github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway/descriptor"
	"github.com/huandu/xstrings"
	options "google.golang.org/genproto/googleapis/api/annotations"
)

var jsReservedRe *regexp.Regexp = regexp.MustCompile(`(^|[^A-Za-z])(do|if|in|for|let|new|try|var|case|else|enum|eval|false|null|this|true|void|with|break|catch|class|const|super|throw|while|yield|delete|export|import|public|return|static|switch|typeof|default|extends|finally|package|private|continue|debugger|function|arguments|interface|protected|implements|instanceof)($|[^A-Za-z])`)

var (
	registry *ggdescriptor.Registry // some helpers need access to registry
)

func SetRegistry(reg *ggdescriptor.Registry) {
	registry = reg
}

var ProtoHelpersFuncMap = template.FuncMap{
	"string": func(i interface {
		String() string
	}) string {
		return i.String()
	},
	"json": func(v interface{}) string {
		a, _ := json.Marshal(v)
		return string(a)
	},
	"prettyjson": func(v interface{}) string {
		a, _ := json.MarshalIndent(v, "", "  ")
		return string(a)
	},
	"splitArray": func(sep string, s string) []string {
		var r []string
		t := strings.Split(s, sep)
		for i := range t {
			if t[i] != "" {
				r = append(r, t[i])
			}
		}
		return r
	},
	"first": func(a []string) string {
		return a[0]
	},
	"last": func(a []string) string {
		return a[len(a)-1]
	},
	"upperFirst": func(s string) string {
		return strings.ToUpper(s[:1]) + s[1:]
	},
	"lowerFirst": func(s string) string {
		return strings.ToLower(s[:1]) + s[1:]
	},
	"camelCase": func(s string) string {
		if len(s) > 1 {
			return xstrings.ToCamelCase(s)
		}

		return strings.ToUpper(s[:1])
	},
	"lowerCamelCase": func(s string) string {
		if len(s) > 1 {
			s = xstrings.ToCamelCase(s)
		}

		return strings.ToLower(s[:1]) + s[1:]
	},
	"kebabCase": func(s string) string {
		return strings.Replace(xstrings.ToSnakeCase(s), "_", "-", -1)
	},
	"snakeCase":             xstrings.ToSnakeCase,
	"getProtoFile":          getProtoFile,
	"getMessageType":        getMessageType,
	"getEnumValue":          getEnumValue,
	"isFieldMessage":        isFieldMessage,
	"isFieldRepeated":       isFieldRepeated,
	"goType":                goType,
	"goTypeWithPackage":     goTypeWithPackage,
	"jsType":                jsType,
	"jsSuffixReserved":      jsSuffixReservedKeyword,
	"namespacedFlowType":    namespacedFlowType,
	"httpVerb":              httpVerb,
	"httpPath":              httpPath,
	"shortType":             shortType,
	"urlHasVarsFromMessage": urlHasVarsFromMessage,
}

func init() {
	for k, v := range sprig.TxtFuncMap() {
		ProtoHelpersFuncMap[k] = v
	}
}

func getProtoFile(name string) *ggdescriptor.File {
	if registry == nil {
		return nil
	}
	file, err := registry.LookupFile(name)
	if err != nil {
		panic(err)
	}
	return file
}

func getMessageType(f *descriptor.FileDescriptorProto, name string) *ggdescriptor.Message {
	if registry != nil {
		msg, err := registry.LookupMsg(".", name)
		if err != nil {
			panic(err)
		}
		return msg
	}

	// name is in the form .packageName.MessageTypeName.InnerMessageTypeName...
	// e.g. .article.ProductTag
	splits := strings.Split(name, ".")
	target := splits[len(splits)-1]
	for _, m := range f.MessageType {
		if target == *m.Name {
			return &ggdescriptor.Message{
				DescriptorProto: m,
			}
		}
	}
	return nil
}

func getEnumValue(f []*descriptor.EnumDescriptorProto, name string) []*descriptor.EnumValueDescriptorProto {
	for _, item := range f {
		if strings.EqualFold(*item.Name, name) {
			return item.GetValue()
		}
	}

	return nil
}

func isFieldMessage(f *descriptor.FieldDescriptorProto) bool {
	if f.Type != nil && *f.Type == descriptor.FieldDescriptorProto_TYPE_MESSAGE {
		return true
	}

	return false
}

func isFieldRepeated(f *descriptor.FieldDescriptorProto) bool {
	if f.Type != nil && f.Label != nil && *f.Label == descriptor.FieldDescriptorProto_LABEL_REPEATED {
		return true
	}

	return false
}

func goTypeWithPackage(f *descriptor.FieldDescriptorProto) string {
	pkg := ""
	if *f.Type == descriptor.FieldDescriptorProto_TYPE_MESSAGE {
		pkg = getPackageTypeName(*f.TypeName)
	}
	return goType(pkg, f)
}

func goType(pkg string, f *descriptor.FieldDescriptorProto) string {
	switch *f.Type {
	case descriptor.FieldDescriptorProto_TYPE_DOUBLE:
		if *f.Label == descriptor.FieldDescriptorProto_LABEL_REPEATED {
			return "[]float64"
		}
		return "float64"
	case descriptor.FieldDescriptorProto_TYPE_FLOAT:
		if *f.Label == descriptor.FieldDescriptorProto_LABEL_REPEATED {
			return "[]float32"
		}
		return "float32"
	case descriptor.FieldDescriptorProto_TYPE_INT64:
		if *f.Label == descriptor.FieldDescriptorProto_LABEL_REPEATED {
			return "[]int64"
		}
		return "int64"
	case descriptor.FieldDescriptorProto_TYPE_UINT64:
		if *f.Label == descriptor.FieldDescriptorProto_LABEL_REPEATED {
			return "[]uint64"
		}
		return "uint64"
	case descriptor.FieldDescriptorProto_TYPE_INT32:
		if *f.Label == descriptor.FieldDescriptorProto_LABEL_REPEATED {
			return "[]int32"
		}
		return "int32"
	case descriptor.FieldDescriptorProto_TYPE_UINT32:
		if *f.Label == descriptor.FieldDescriptorProto_LABEL_REPEATED {
			return "[]uint32"
		}
		return "uint32"
	case descriptor.FieldDescriptorProto_TYPE_BOOL:
		if *f.Label == descriptor.FieldDescriptorProto_LABEL_REPEATED {
			return "[]bool"
		}
		return "bool"
	case descriptor.FieldDescriptorProto_TYPE_STRING:
		if *f.Label == descriptor.FieldDescriptorProto_LABEL_REPEATED {
			return "[]string"
		}
		return "string"
	case descriptor.FieldDescriptorProto_TYPE_MESSAGE:
		if pkg != "" {
			pkg = pkg + "."
		}
		if *f.Label == descriptor.FieldDescriptorProto_LABEL_REPEATED {
			return fmt.Sprintf("[]*%s%s", pkg, shortType(*f.TypeName))
		}
		return fmt.Sprintf("*%s%s", pkg, shortType(*f.TypeName))
	case descriptor.FieldDescriptorProto_TYPE_BYTES:
		if *f.Label == descriptor.FieldDescriptorProto_LABEL_REPEATED {
			return "[]byte"
		}
		return "byte"
	case descriptor.FieldDescriptorProto_TYPE_ENUM:
		return fmt.Sprintf("*%s.%s", pkg, shortType(*f.TypeName))
	default:
		return "interface{}"
	}
}

func jsType(f *descriptor.FieldDescriptorProto) string {
	template := "%s"
	if isFieldRepeated(f) == true {
		template = "Array<%s>"
	}

	switch *f.Type {
	case descriptor.FieldDescriptorProto_TYPE_MESSAGE,
		descriptor.FieldDescriptorProto_TYPE_ENUM:
		return fmt.Sprintf(template, namespacedFlowType(*f.TypeName))
	case descriptor.FieldDescriptorProto_TYPE_DOUBLE,
		descriptor.FieldDescriptorProto_TYPE_FLOAT,
		descriptor.FieldDescriptorProto_TYPE_INT64,
		descriptor.FieldDescriptorProto_TYPE_UINT64,
		descriptor.FieldDescriptorProto_TYPE_INT32,
		descriptor.FieldDescriptorProto_TYPE_FIXED64,
		descriptor.FieldDescriptorProto_TYPE_FIXED32,
		descriptor.FieldDescriptorProto_TYPE_UINT32,
		descriptor.FieldDescriptorProto_TYPE_SFIXED32,
		descriptor.FieldDescriptorProto_TYPE_SFIXED64,
		descriptor.FieldDescriptorProto_TYPE_SINT32,
		descriptor.FieldDescriptorProto_TYPE_SINT64:
		return fmt.Sprintf(template, "number")
	case descriptor.FieldDescriptorProto_TYPE_BOOL:
		return fmt.Sprintf(template, "boolean")
	case descriptor.FieldDescriptorProto_TYPE_BYTES:
		return fmt.Sprintf(template, "Uint8Array")
	case descriptor.FieldDescriptorProto_TYPE_STRING:
		return fmt.Sprintf(template, "string")
	default:
		return fmt.Sprintf(template, "any")
	}
}

func jsSuffixReservedKeyword(s string) string {
	return jsReservedRe.ReplaceAllString(s, "${1}${2}_${3}")
}

func getPackageTypeName(s string) string {
	if strings.Compare(s, ".google.protobuf.Timestamp") == 0 {
		return "timestamp"
	}
	if strings.Contains(s, ".") {
		return strings.Split(s, ".")[1]
	}
	return ""
}

func shortType(s string) string {
	t := strings.Split(s, ".")
	return t[len(t)-1]
}

func namespacedFlowType(s string) string {
	trimmed := strings.TrimLeft(s, ".")
	splitted := strings.Split(trimmed, ".")
	return strings.Join(splitted, "$")
}

func httpPath(m *descriptor.MethodDescriptorProto) string {

	ext, err := proto.GetExtension(m.Options, options.E_Http)
	if err != nil {
		return err.Error()
	}
	opts, ok := ext.(*options.HttpRule)
	if !ok {
		return fmt.Sprintf("extension is %T; want an HttpRule", ext)
	}

	switch t := opts.Pattern.(type) {
	default:
		return ""
	case *options.HttpRule_Get:
		return t.Get
	case *options.HttpRule_Post:
		return t.Post
	case *options.HttpRule_Put:
		return t.Put
	case *options.HttpRule_Delete:
		return t.Delete
	case *options.HttpRule_Patch:
		return t.Patch
	case *options.HttpRule_Custom:
		return t.Custom.Path
	}
}

func httpVerb(m *descriptor.MethodDescriptorProto) string {

	ext, err := proto.GetExtension(m.Options, options.E_Http)
	if err != nil {
		return err.Error()
	}
	opts, ok := ext.(*options.HttpRule)
	if !ok {
		return fmt.Sprintf("extension is %T; want an HttpRule", ext)
	}

	switch t := opts.Pattern.(type) {
	default:
		return ""
	case *options.HttpRule_Get:
		return "GET"
	case *options.HttpRule_Post:
		return "POST"
	case *options.HttpRule_Put:
		return "PUT"
	case *options.HttpRule_Delete:
		return "DELETE"
	case *options.HttpRule_Patch:
		return "PATCH"
	case *options.HttpRule_Custom:
		return t.Custom.Kind
	}
}

func urlHasVarsFromMessage(path string, d *ggdescriptor.Message) bool {
	for _, field := range d.Field {
		if !isFieldMessage(field) {
			if strings.Contains(path, fmt.Sprintf("{%s}", *field.Name)) {
				return true
			}
		}
	}

	return false
}
<|endoftext|>


<commit_before>package helpers

import (
	"fmt"
	"os/exec"
	"strings"
)

func GetDpkgPackage(name string) (string, error) {
	out, err := exec.Command("dpkg", "-l", name).Output() // TODO: Security
	if err != nil {
		return "", err
	}
	lines := strings.Split(string(out), "\n")
	for _, line := range lines {
		split := strings.Fields(line)
		if len(split) >= 1 {
			f := string(split[0][0])
			if f == "h" || f == "i" || f == "p" || f == "r" || f == "u" {
				return split[0], nil
			}
		}
	}
	return "", nil
}

func GetPackageStatus(name string) (string, error) {
	distro, err := GetDistro()
	if err != nil {
		return "", err
	}

	switch distro.Family {
	case "debian":
		status, err := GetDpkgPackage(name)
		if err != nil {
			return "", err
		}
		switch status {
		case "ii":
			return "installed", nil
		case "rc":
			return "removed", nil
		case "":
			return "purged", nil
		default:
			return "", fmt.Errorf("Package %s is in an unknown state: %s", name, status)
		}
	default:
		return "", fmt.Errorf("Unsupported operating system: %s", distro.Family)
	}
}
<commit_msg>If the dpkg package lookup command fails the function will return "rc" indicating the package is removed.<commit_after>package helpers

import (
	"fmt"
	"os/exec"
	"strings"
)

func GetDpkgPackage(name string) (string, error) {
	out, err := exec.Command("dpkg", "-l", name).Output() // TODO: Security
	if err != nil { // dpkg doesn't know about the package
		return "rc", nil
	}
	lines := strings.Split(string(out), "\n")
	for _, line := range lines {
		split := strings.Fields(line)
		if len(split) >= 1 {
			f := string(split[0][0])
			if f == "h" || f == "i" || f == "p" || f == "r" || f == "u" {
				return split[0], nil
			}
		}
	}
	return "", nil
}

func GetPackageStatus(name string) (string, error) {
	distro, err := GetDistro()
	if err != nil {
		return "", err
	}

	switch distro.Family {
	case "debian":
		status, err := GetDpkgPackage(name)
		if err != nil {
			return "", err
		}
		switch status {
		case "ii":
			return "installed", nil
		case "rc":
			return "removed", nil
		case "":
			return "purged", nil
		default:
			return "", fmt.Errorf("Package %s is in an unknown state: %s", name, status)
		}
	default:
		return "", fmt.Errorf("Unsupported operating system: %s", distro.Family)
	}
}
<|endoftext|>


<commit_before>package helpers

import (
	"code.cloudfoundry.org/bbs"
	"code.cloudfoundry.org/bbs/models"
	"code.cloudfoundry.org/lager"

	. "github.com/onsi/gomega"
)

func filteredActualLRPs(logger lager.Logger, client bbs.InternalClient, processGuid string, filter func(lrp *models.ActualLRP) bool) []models.ActualLRP {
	lrps, err := client.ActualLRPs(logger, models.ActualLRPFilter{ProcessGuid: processGuid})
	Expect(err).NotTo(HaveOccurred())

	startedLRPs := make([]models.ActualLRP, 0, len(lrps))
	for _, lrp := range lrps {
		if filter(lrp) {
			startedLRPs = append(startedLRPs, *lrp)
		}
	}

	return startedLRPs
}

func ActiveActualLRPs(logger lager.Logger, client bbs.InternalClient, processGuid string) []models.ActualLRP {
	return filteredActualLRPs(logger, client, processGuid, func(lrp *models.ActualLRP) bool {
		return lrp.State != models.ActualLRPStateUnclaimed
	})
}

func RunningActualLRPs(logger lager.Logger, client bbs.InternalClient, processGuid string) []models.ActualLRP {
	return filteredActualLRPs(logger, client, processGuid, func(lrp *models.ActualLRP) bool {
		return lrp.State == models.ActualLRPStateRunning
	})
}

func TaskStatePoller(logger lager.Logger, client bbs.InternalClient, taskGuid string, task *models.Task) func() models.Task_State {
	return func() models.Task_State {
		rTask, err := client.TaskByGuid(logger, taskGuid)
		Expect(err).NotTo(HaveOccurred())

		if task != nil {
			*task = *rTask
		}

		return rTask.State
	}
}

func TaskFailedPoller(logger lager.Logger, client bbs.InternalClient, taskGuid string, task *models.Task) func() bool {
	return func() bool {
		rTask, err := client.TaskByGuid(logger, taskGuid)
		Expect(err).NotTo(HaveOccurred())

		if task != nil {
			*task = *rTask
		}

		return rTask.Failed
	}
}

func LRPStatePoller(logger lager.Logger, client bbs.InternalClient, processGuid string, lrp *models.ActualLRP) func() string {
	return func() string {
		lrps, err := client.ActualLRPs(logger, models.ActualLRPFilter{ProcessGuid: processGuid})
		Expect(err).NotTo(HaveOccurred())
		if len(lrps) == 0 {
			return ""
		}
		Expect(len(lrps)).To(BeNumerically(">", 0))
		if lrp != nil {
			*lrp = *lrps[0]
		}
		return lrps[0].State
	}
}

func LRPInstanceStatePoller(logger lager.Logger, client bbs.InternalClient, processGuid string, index int, lrp *models.ActualLRP) func() string {
	return func() string {
		i := int32(index)
		lrps, err := client.ActualLRPs(logger, models.ActualLRPFilter{ProcessGuid: processGuid, Index: &i})
		Expect(err).NotTo(HaveOccurred())
		Expect(len(lrps)).To(Equal(1))
		if lrp != nil {
			*lrp = *lrps[0]
		}
		return lrps[0].State
	}
}
<commit_msg>Ensure  LRPStatePoller is backward compatible for v1.0.0 and v1.25.0<commit_after>package helpers

import (
	"strings"

	"code.cloudfoundry.org/bbs"
	"code.cloudfoundry.org/bbs/models"
	"code.cloudfoundry.org/lager"

	. "github.com/onsi/gomega"
)

func filteredActualLRPs(logger lager.Logger, client bbs.InternalClient, processGuid string, filter func(lrp *models.ActualLRP) bool) []models.ActualLRP {
	lrps, err := client.ActualLRPs(logger, models.ActualLRPFilter{ProcessGuid: processGuid})
	Expect(err).NotTo(HaveOccurred())

	startedLRPs := make([]models.ActualLRP, 0, len(lrps))
	for _, lrp := range lrps {
		if filter(lrp) {
			startedLRPs = append(startedLRPs, *lrp)
		}
	}

	return startedLRPs
}

func ActiveActualLRPs(logger lager.Logger, client bbs.InternalClient, processGuid string) []models.ActualLRP {
	return filteredActualLRPs(logger, client, processGuid, func(lrp *models.ActualLRP) bool {
		return lrp.State != models.ActualLRPStateUnclaimed
	})
}

func RunningActualLRPs(logger lager.Logger, client bbs.InternalClient, processGuid string) []models.ActualLRP {
	return filteredActualLRPs(logger, client, processGuid, func(lrp *models.ActualLRP) bool {
		return lrp.State == models.ActualLRPStateRunning
	})
}

func TaskStatePoller(logger lager.Logger, client bbs.InternalClient, taskGuid string, task *models.Task) func() models.Task_State {
	return func() models.Task_State {
		rTask, err := client.TaskByGuid(logger, taskGuid)
		Expect(err).NotTo(HaveOccurred())

		if task != nil {
			*task = *rTask
		}

		return rTask.State
	}
}

func TaskFailedPoller(logger lager.Logger, client bbs.InternalClient, taskGuid string, task *models.Task) func() bool {
	return func() bool {
		rTask, err := client.TaskByGuid(logger, taskGuid)
		Expect(err).NotTo(HaveOccurred())

		if task != nil {
			*task = *rTask
		}

		return rTask.Failed
	}
}

func LRPStatePoller(logger lager.Logger, client bbs.InternalClient, processGuid string, lrp *models.ActualLRP) func() string {
	return func() string {
		var foundLRP *models.ActualLRP

		lrps, err := client.ActualLRPs(logger, models.ActualLRPFilter{ProcessGuid: processGuid})
		if err != nil && strings.Contains(err.Error(), "Invalid Response with status code: 404") {
			lrpGroups, err := client.ActualLRPGroupsByProcessGuid(logger, processGuid)
			Expect(err).NotTo(HaveOccurred())
			if len(lrpGroups) == 0 {
				return ""
			}
			foundLRP, _, err = lrpGroups[0].Resolve()
			Expect(err).NotTo(HaveOccurred())
		} else {
			Expect(err).NotTo(HaveOccurred())
			if len(lrps) == 0 {
				return ""
			}
			foundLRP = lrps[0]
		}
		if lrp != nil {
			*lrp = *foundLRP
		}
		return foundLRP.State
	}
}

func LRPInstanceStatePoller(logger lager.Logger, client bbs.InternalClient, processGuid string, index int, lrp *models.ActualLRP) func() string {
	return func() string {
		i := int32(index)
		var foundLRP *models.ActualLRP
		lrps, err := client.ActualLRPs(logger, models.ActualLRPFilter{ProcessGuid: processGuid, Index: &i})
		if err != nil && strings.Contains(err.Error(), "Invalid Response with status code: 404") {
			lrpGroup, err := client.ActualLRPGroupByProcessGuidAndIndex(logger, processGuid, index)
			Expect(err).NotTo(HaveOccurred())
			foundLRP, _, err = lrpGroup.Resolve()
			Expect(err).NotTo(HaveOccurred())
		} else {
			Expect(err).NotTo(HaveOccurred())
			Expect(lrps).To(HaveLen(1))
			foundLRP = lrps[0]
		}
		if lrp != nil {
			*lrp = *foundLRP
		}
		return foundLRP.State
	}
}
<|endoftext|>


<commit_before>package connpass

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

const baseURL string = "http://connpass.com/api/v1/event/"

type ResultSet struct {
	Returned  int     `json:"results_returned"`
	Available int     `json:"results_available"`
	Start     int     `json:"results_start"`
	Events    []Event `json:"events"`
}

type Event struct {
	Id            int     `json:"event_id"`
	Title         string  `json:"title"`
	Catch         string  `json:"catch"`
	Description   string  `json:"description"`
	Url           string  `json:"event_url"`
	Tag           string  `json:"hash_tag"`
	Start         string  `json:"started_at"`
	End           string  `json:"ended_at"`
	Limit         int     `json:"limit"`
	Etype         string  `json:"event_type"`
	Address       string  `json:"address"`
	Place         string  `json:"place"`
	Lat           float64 `json:"lat"`
	Lon           float64 `json:"lon"`
	OwnerID       int     `json:"owner_id"`
	OwnerNickname string  `json:"owner_nickname"`
	OwnerName     string  `json:"owner_display_name"`
	Accepted      int     `json:"accepted"`
	Waiting       int     `json:"waiting"`
	Updated       string  `json:"updated_at"`
}

type Order int

const (
	_      Order = iota
	UPDATE       // 1: descending in updated time
	START        // 2: descending in event start time
	CREATE       // 3: descending in created time
)

type Query struct {
	Start int
	Order Order
	Count int
}

func NewQery() Query {
	q := Query{0, CREATE, 100}
	return q
}

func (q Query) buildURL() string {
	return fmt.Sprint(baseURL, "?start=", q.Start, "&order=", q.Order, "&count=", q.Count)
}

func parse(jsonBlob []byte) (*ResultSet, error) {
	res := new(ResultSet)
	err := json.Unmarshal(jsonBlob, &res)
	if err != nil {
		return nil, err
	}
	return res, nil
}

func (q Query) Search() (*ResultSet, error) {
	res, err := http.Get(q.buildURL())
	if err != nil {
		return nil, err
	}
	body, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		return nil, err
	}
	return parse(body)
}
<commit_msg>Fix typo.<commit_after>package connpass

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

const baseURL string = "http://connpass.com/api/v1/event/"

type ResultSet struct {
	Returned  int     `json:"results_returned"`
	Available int     `json:"results_available"`
	Start     int     `json:"results_start"`
	Events    []Event `json:"events"`
}

type Event struct {
	Id            int     `json:"event_id"`
	Title         string  `json:"title"`
	Catch         string  `json:"catch"`
	Description   string  `json:"description"`
	Url           string  `json:"event_url"`
	Tag           string  `json:"hash_tag"`
	Start         string  `json:"started_at"`
	End           string  `json:"ended_at"`
	Limit         int     `json:"limit"`
	Etype         string  `json:"event_type"`
	Address       string  `json:"address"`
	Place         string  `json:"place"`
	Lat           float64 `json:"lat"`
	Lon           float64 `json:"lon"`
	OwnerID       int     `json:"owner_id"`
	OwnerNickname string  `json:"owner_nickname"`
	OwnerName     string  `json:"owner_display_name"`
	Accepted      int     `json:"accepted"`
	Waiting       int     `json:"waiting"`
	Updated       string  `json:"updated_at"`
}

type Order int

const (
	_      Order = iota
	UPDATE       // 1: descending in updated time
	START        // 2: descending in event start time
	CREATE       // 3: descending in created time
)

type Query struct {
	Start int
	Order Order
	Count int
}

func NewQuery() Query {
	q := Query{0, CREATE, 100}
	return q
}

func (q Query) buildURL() string {
	return fmt.Sprint(baseURL, "?start=", q.Start, "&order=", q.Order, "&count=", q.Count)
}

func parse(jsonBlob []byte) (*ResultSet, error) {
	res := new(ResultSet)
	err := json.Unmarshal(jsonBlob, &res)
	if err != nil {
		return nil, err
	}
	return res, nil
}

func (q Query) Search() (*ResultSet, error) {
	res, err := http.Get(q.buildURL())
	if err != nil {
		return nil, err
	}
	body, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		return nil, err
	}
	return parse(body)
}
<|endoftext|>


<commit_before>package schema

import (
	"fmt"
	"strconv"
	"strings"
	"sync"

	"github.com/hashicorp/terraform/terraform"
	"github.com/mitchellh/mapstructure"
)

// ConfigFieldReader reads fields out of an untyped map[string]string to the
// best of its ability. It also applies defaults from the Schema. (The other
// field readers do not need default handling because they source fully
// populated data structures.)
type ConfigFieldReader struct {
	Config *terraform.ResourceConfig
	Schema map[string]*Schema

	indexMaps map[string]map[string]int
	once      sync.Once
}

func (r *ConfigFieldReader) ReadField(address []string) (FieldReadResult, error) {
	r.once.Do(func() { r.indexMaps = make(map[string]map[string]int) })
	return r.readField(address, false)
}

func (r *ConfigFieldReader) readField(
	address []string, nested bool) (FieldReadResult, error) {
	schemaList := addrToSchema(address, r.Schema)
	if len(schemaList) == 0 {
		return FieldReadResult{}, nil
	}

	if !nested {
		// If we have a set anywhere in the address, then we need to
		// read that set out in order and actually replace that part of
		// the address with the real list index. i.e. set.50 might actually
		// map to set.12 in the config, since it is in list order in the
		// config, not indexed by set value.
		for i, v := range schemaList {
			// Sets are the only thing that cause this issue.
			if v.Type != TypeSet {
				continue
			}

			// If we're at the end of the list, then we don't have to worry
			// about this because we're just requesting the whole set.
			if i == len(schemaList)-1 {
				continue
			}

			// If we're looking for the count, then ignore...
			if address[i+1] == "#" {
				continue
			}

			indexMap, ok := r.indexMaps[strings.Join(address[:i+1], ".")]
			if !ok {
				// Get the set so we can get the index map that tells us the
				// mapping of the hash code to the list index
				_, err := r.readSet(address[:i+1], v)
				if err != nil {
					return FieldReadResult{}, err
				}
				indexMap = r.indexMaps[strings.Join(address[:i+1], ".")]
			}

			index, ok := indexMap[address[i+1]]
			if !ok {
				return FieldReadResult{}, nil
			}

			address[i+1] = strconv.FormatInt(int64(index), 10)
		}
	}

	k := strings.Join(address, ".")
	schema := schemaList[len(schemaList)-1]
	switch schema.Type {
	case TypeBool, TypeFloat, TypeInt, TypeString:
		return r.readPrimitive(k, schema)
	case TypeList:
		return readListField(&nestedConfigFieldReader{r}, address, schema)
	case TypeMap:
		return r.readMap(k)
	case TypeSet:
		return r.readSet(address, schema)
	case typeObject:
		return readObjectField(
			&nestedConfigFieldReader{r},
			address, schema.Elem.(map[string]*Schema))
	default:
		panic(fmt.Sprintf("Unknown type: %s", schema.Type))
	}
}

func (r *ConfigFieldReader) readMap(k string) (FieldReadResult, error) {
	// We want both the raw value and the interpolated. We use the interpolated
	// to store actual values and we use the raw one to check for
	// computed keys. Actual values are obtained in the switch, depending on
	// the type of the raw value.
	mraw, ok := r.Config.GetRaw(k)
	if !ok {
		return FieldReadResult{}, nil
	}

	result := make(map[string]interface{})
	computed := false
	switch m := mraw.(type) {
	case string:
		// This is a map which has come out of an interpolated variable, so we
		// can just get the value directly from config. Values cannot be computed
		// currently.
		v, _ := r.Config.Get(k)

		// If this isn't a map[string]interface, it must be computed.
		mapV, ok := v.(map[string]interface{})
		if !ok {
			return FieldReadResult{
				Exists:   true,
				Computed: true,
			}, nil
		}

		// Otherwise we can proceed as usual.
		for i, iv := range mapV {
			result[i] = iv
		}
	case []interface{}:
		for i, innerRaw := range m {
			for ik := range innerRaw.(map[string]interface{}) {
				key := fmt.Sprintf("%s.%d.%s", k, i, ik)
				if r.Config.IsComputed(key) {
					computed = true
					break
				}

				v, _ := r.Config.Get(key)
				result[ik] = v
			}
		}
	case []map[string]interface{}:
		for i, innerRaw := range m {
			for ik := range innerRaw {
				key := fmt.Sprintf("%s.%d.%s", k, i, ik)
				if r.Config.IsComputed(key) {
					computed = true
					break
				}

				v, _ := r.Config.Get(key)
				result[ik] = v
			}
		}
	case map[string]interface{}:
		for ik := range m {
			key := fmt.Sprintf("%s.%s", k, ik)
			if r.Config.IsComputed(key) {
				computed = true
				break
			}

			v, _ := r.Config.Get(key)
			result[ik] = v
		}
	default:
		panic(fmt.Sprintf("unknown type: %#v", mraw))
	}

	var value interface{}
	if !computed {
		value = result
	}

	return FieldReadResult{
		Value:    value,
		Exists:   true,
		Computed: computed,
	}, nil
}

func (r *ConfigFieldReader) readPrimitive(
	k string, schema *Schema) (FieldReadResult, error) {
	raw, ok := r.Config.Get(k)
	if !ok {
		// Nothing in config, but we might still have a default from the schema
		var err error
		raw, err = schema.DefaultValue()
		if err != nil {
			return FieldReadResult{}, fmt.Errorf("%s, error loading default: %s", k, err)
		}

		if raw == nil {
			return FieldReadResult{}, nil
		}
	}

	var result string
	if err := mapstructure.WeakDecode(raw, &result); err != nil {
		return FieldReadResult{}, err
	}

	computed := r.Config.IsComputed(k)
	returnVal, err := stringToPrimitive(result, computed, schema)
	if err != nil {
		return FieldReadResult{}, err
	}

	return FieldReadResult{
		Value:    returnVal,
		Exists:   true,
		Computed: computed,
	}, nil
}

func (r *ConfigFieldReader) readSet(
	address []string, schema *Schema) (FieldReadResult, error) {
	indexMap := make(map[string]int)
	// Create the set that will be our result
	set := schema.ZeroValue().(*Set)

	raw, err := readListField(&nestedConfigFieldReader{r}, address, schema)
	if err != nil {
		return FieldReadResult{}, err
	}
	if !raw.Exists {
		return FieldReadResult{Value: set}, nil
	}

	// If the list is computed, the set is necessarilly computed
	if raw.Computed {
		return FieldReadResult{
			Value:    set,
			Exists:   true,
			Computed: raw.Computed,
		}, nil
	}

	// Build up the set from the list elements
	for i, v := range raw.Value.([]interface{}) {
		// Check if any of the keys in this item are computed
		computed := r.hasComputedSubKeys(
			fmt.Sprintf("%s.%d", strings.Join(address, "."), i), schema)

		code := set.add(v, computed)
		indexMap[code] = i
	}

	r.indexMaps[strings.Join(address, ".")] = indexMap

	return FieldReadResult{
		Value:  set,
		Exists: true,
	}, nil
}

// hasComputedSubKeys walks through a schema and returns whether or not the
// given key contains any subkeys that are computed.
func (r *ConfigFieldReader) hasComputedSubKeys(key string, schema *Schema) bool {
	prefix := key + "."

	switch t := schema.Elem.(type) {
	case *Resource:
		for k, schema := range t.Schema {
			if r.Config.IsComputed(prefix + k) {
				return true
			}

			if r.hasComputedSubKeys(prefix+k, schema) {
				return true
			}
		}
	}

	return false
}

// nestedConfigFieldReader is a funny little thing that just wraps a
// ConfigFieldReader to call readField when ReadField is called so that
// we don't recalculate the set rewrites in the address, which leads to
// an infinite loop.
type nestedConfigFieldReader struct {
	Reader *ConfigFieldReader
}

func (r *nestedConfigFieldReader) ReadField(
	address []string) (FieldReadResult, error) {
	return r.Reader.readField(address, true)
}
<commit_msg>core: Ensure hasComputedSubKeys iterates over Sets and Lists properly<commit_after>package schema

import (
	"fmt"
	"strconv"
	"strings"
	"sync"

	"github.com/hashicorp/terraform/terraform"
	"github.com/mitchellh/mapstructure"
)

// ConfigFieldReader reads fields out of an untyped map[string]string to the
// best of its ability. It also applies defaults from the Schema. (The other
// field readers do not need default handling because they source fully
// populated data structures.)
type ConfigFieldReader struct {
	Config *terraform.ResourceConfig
	Schema map[string]*Schema

	indexMaps map[string]map[string]int
	once      sync.Once
}

func (r *ConfigFieldReader) ReadField(address []string) (FieldReadResult, error) {
	r.once.Do(func() { r.indexMaps = make(map[string]map[string]int) })
	return r.readField(address, false)
}

func (r *ConfigFieldReader) readField(
	address []string, nested bool) (FieldReadResult, error) {
	schemaList := addrToSchema(address, r.Schema)
	if len(schemaList) == 0 {
		return FieldReadResult{}, nil
	}

	if !nested {
		// If we have a set anywhere in the address, then we need to
		// read that set out in order and actually replace that part of
		// the address with the real list index. i.e. set.50 might actually
		// map to set.12 in the config, since it is in list order in the
		// config, not indexed by set value.
		for i, v := range schemaList {
			// Sets are the only thing that cause this issue.
			if v.Type != TypeSet {
				continue
			}

			// If we're at the end of the list, then we don't have to worry
			// about this because we're just requesting the whole set.
			if i == len(schemaList)-1 {
				continue
			}

			// If we're looking for the count, then ignore...
			if address[i+1] == "#" {
				continue
			}

			indexMap, ok := r.indexMaps[strings.Join(address[:i+1], ".")]
			if !ok {
				// Get the set so we can get the index map that tells us the
				// mapping of the hash code to the list index
				_, err := r.readSet(address[:i+1], v)
				if err != nil {
					return FieldReadResult{}, err
				}
				indexMap = r.indexMaps[strings.Join(address[:i+1], ".")]
			}

			index, ok := indexMap[address[i+1]]
			if !ok {
				return FieldReadResult{}, nil
			}

			address[i+1] = strconv.FormatInt(int64(index), 10)
		}
	}

	k := strings.Join(address, ".")
	schema := schemaList[len(schemaList)-1]
	switch schema.Type {
	case TypeBool, TypeFloat, TypeInt, TypeString:
		return r.readPrimitive(k, schema)
	case TypeList:
		return readListField(&nestedConfigFieldReader{r}, address, schema)
	case TypeMap:
		return r.readMap(k)
	case TypeSet:
		return r.readSet(address, schema)
	case typeObject:
		return readObjectField(
			&nestedConfigFieldReader{r},
			address, schema.Elem.(map[string]*Schema))
	default:
		panic(fmt.Sprintf("Unknown type: %s", schema.Type))
	}
}

func (r *ConfigFieldReader) readMap(k string) (FieldReadResult, error) {
	// We want both the raw value and the interpolated. We use the interpolated
	// to store actual values and we use the raw one to check for
	// computed keys. Actual values are obtained in the switch, depending on
	// the type of the raw value.
	mraw, ok := r.Config.GetRaw(k)
	if !ok {
		return FieldReadResult{}, nil
	}

	result := make(map[string]interface{})
	computed := false
	switch m := mraw.(type) {
	case string:
		// This is a map which has come out of an interpolated variable, so we
		// can just get the value directly from config. Values cannot be computed
		// currently.
		v, _ := r.Config.Get(k)

		// If this isn't a map[string]interface, it must be computed.
		mapV, ok := v.(map[string]interface{})
		if !ok {
			return FieldReadResult{
				Exists:   true,
				Computed: true,
			}, nil
		}

		// Otherwise we can proceed as usual.
		for i, iv := range mapV {
			result[i] = iv
		}
	case []interface{}:
		for i, innerRaw := range m {
			for ik := range innerRaw.(map[string]interface{}) {
				key := fmt.Sprintf("%s.%d.%s", k, i, ik)
				if r.Config.IsComputed(key) {
					computed = true
					break
				}

				v, _ := r.Config.Get(key)
				result[ik] = v
			}
		}
	case []map[string]interface{}:
		for i, innerRaw := range m {
			for ik := range innerRaw {
				key := fmt.Sprintf("%s.%d.%s", k, i, ik)
				if r.Config.IsComputed(key) {
					computed = true
					break
				}

				v, _ := r.Config.Get(key)
				result[ik] = v
			}
		}
	case map[string]interface{}:
		for ik := range m {
			key := fmt.Sprintf("%s.%s", k, ik)
			if r.Config.IsComputed(key) {
				computed = true
				break
			}

			v, _ := r.Config.Get(key)
			result[ik] = v
		}
	default:
		panic(fmt.Sprintf("unknown type: %#v", mraw))
	}

	var value interface{}
	if !computed {
		value = result
	}

	return FieldReadResult{
		Value:    value,
		Exists:   true,
		Computed: computed,
	}, nil
}

func (r *ConfigFieldReader) readPrimitive(
	k string, schema *Schema) (FieldReadResult, error) {
	raw, ok := r.Config.Get(k)
	if !ok {
		// Nothing in config, but we might still have a default from the schema
		var err error
		raw, err = schema.DefaultValue()
		if err != nil {
			return FieldReadResult{}, fmt.Errorf("%s, error loading default: %s", k, err)
		}

		if raw == nil {
			return FieldReadResult{}, nil
		}
	}

	var result string
	if err := mapstructure.WeakDecode(raw, &result); err != nil {
		return FieldReadResult{}, err
	}

	computed := r.Config.IsComputed(k)
	returnVal, err := stringToPrimitive(result, computed, schema)
	if err != nil {
		return FieldReadResult{}, err
	}

	return FieldReadResult{
		Value:    returnVal,
		Exists:   true,
		Computed: computed,
	}, nil
}

func (r *ConfigFieldReader) readSet(
	address []string, schema *Schema) (FieldReadResult, error) {
	indexMap := make(map[string]int)
	// Create the set that will be our result
	set := schema.ZeroValue().(*Set)

	raw, err := readListField(&nestedConfigFieldReader{r}, address, schema)
	if err != nil {
		return FieldReadResult{}, err
	}
	if !raw.Exists {
		return FieldReadResult{Value: set}, nil
	}

	// If the list is computed, the set is necessarilly computed
	if raw.Computed {
		return FieldReadResult{
			Value:    set,
			Exists:   true,
			Computed: raw.Computed,
		}, nil
	}

	// Build up the set from the list elements
	for i, v := range raw.Value.([]interface{}) {
		// Check if any of the keys in this item are computed
		computed := r.hasComputedSubKeys(
			fmt.Sprintf("%s.%d", strings.Join(address, "."), i), schema)

		code := set.add(v, computed)
		indexMap[code] = i
	}

	r.indexMaps[strings.Join(address, ".")] = indexMap

	return FieldReadResult{
		Value:  set,
		Exists: true,
	}, nil
}

// hasComputedSubKeys walks through a schema and returns whether or not the
// given key contains any subkeys that are computed.
func (r *ConfigFieldReader) hasComputedSubKeys(key string, schema *Schema) bool {
	prefix := key + "."

	switch t := schema.Elem.(type) {
	case *Resource:
		for k, schema := range t.Schema {
			addr := prefix + k
			if r.Config.IsComputed(addr) {
				return true
			}

			// We need to loop into sets and lists to ensure we pass the correct
			// address to the raw config - otherwise for sets we get something like
			// set.0.set.item instead of set.0.set.0.item, which renders an
			// inaccurate result.
			if schema.Type == TypeSet || schema.Type == TypeList {
				raw, err := readListField(&nestedConfigFieldReader{r}, strings.Split(addr, "."), schema)
				if err != nil {
					panic(fmt.Errorf("readListField failed when field was supposed to be list-like: %v", err))
				}
				// Just range into the address space here, we don't need the value.
				for i := range raw.Value.([]interface{}) {
					if r.hasComputedSubKeys(addr+"."+strconv.Itoa(i), schema) {
						return true
					}
				}
			} else {
				if r.hasComputedSubKeys(addr, schema) {
					return true
				}
			}
		}
	}

	return false
}

// nestedConfigFieldReader is a funny little thing that just wraps a
// ConfigFieldReader to call readField when ReadField is called so that
// we don't recalculate the set rewrites in the address, which leads to
// an infinite loop.
type nestedConfigFieldReader struct {
	Reader *ConfigFieldReader
}

func (r *nestedConfigFieldReader) ReadField(
	address []string) (FieldReadResult, error) {
	return r.Reader.readField(address, true)
}
<|endoftext|>