<commit_before>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package etcdmain

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path"
	"reflect"
	"runtime"
	"strings"
	"time"

	"github.com/coreos/etcd/discovery"
	"github.com/coreos/etcd/embed"
	"github.com/coreos/etcd/etcdserver"
	"github.com/coreos/etcd/pkg/cors"
	"github.com/coreos/etcd/pkg/fileutil"
	pkgioutil "github.com/coreos/etcd/pkg/ioutil"
	"github.com/coreos/etcd/pkg/osutil"
	"github.com/coreos/etcd/pkg/transport"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/proxy/httpproxy"
	"github.com/coreos/etcd/version"
	"github.com/coreos/go-systemd/daemon"
	systemdutil "github.com/coreos/go-systemd/util"
	"github.com/coreos/pkg/capnslog"
	"github.com/prometheus/client_golang/prometheus"
	"google.golang.org/grpc"
)

type dirType string

var plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "etcdmain")

var (
	dirMember = dirType("member")
	dirProxy  = dirType("proxy")
	dirEmpty  = dirType("empty")
)

func startEtcdOrProxyV2() {
	grpc.EnableTracing = false

	cfg := newConfig()
	err := cfg.parse(os.Args[1:])
	if err != nil {
		plog.Errorf("error verifying flags, %v. See 'etcd --help'.", err)
		switch err {
		case embed.ErrUnsetAdvertiseClientURLsFlag:
			plog.Errorf("When listening on specific address(es), this etcd process must advertise accessible url(s) to each connected client.")
		}
		os.Exit(1)
	}
	setupLogging(cfg)

	var stopped <-chan struct{}
	var errc <-chan error

	plog.Infof("etcd Version: %s\n", version.Version)
	plog.Infof("Git SHA: %s\n", version.GitSHA)
	plog.Infof("Go Version: %s\n", runtime.Version())
	plog.Infof("Go OS/Arch: %s/%s\n", runtime.GOOS, runtime.GOARCH)

	GoMaxProcs := runtime.GOMAXPROCS(0)
	plog.Infof("setting maximum number of CPUs to %d, total number of available CPUs is %d", GoMaxProcs, runtime.NumCPU())

	// TODO: check whether fields are set instead of whether fields have default value
	if cfg.Name != embed.DefaultName && cfg.InitialCluster == cfg.InitialClusterFromName(embed.DefaultName) {
		cfg.InitialCluster = cfg.InitialClusterFromName(cfg.Name)
	}

	if cfg.Dir == "" {
		cfg.Dir = fmt.Sprintf("%v.etcd", cfg.Name)
		plog.Warningf("no data-dir provided, using default data-dir ./%s", cfg.Dir)
	}

	which := identifyDataDirOrDie(cfg.Dir)
	if which != dirEmpty {
		plog.Noticef("the server is already initialized as %v before, starting as etcd %v...", which, which)
		switch which {
		case dirMember:
			stopped, errc, err = startEtcd(&cfg.Config)
		case dirProxy:
			err = startProxy(cfg)
		default:
			plog.Panicf("unhandled dir type %v", which)
		}
	} else {
		shouldProxy := cfg.isProxy()
		if !shouldProxy {
			stopped, errc, err = startEtcd(&cfg.Config)
			if derr, ok := err.(*etcdserver.DiscoveryError); ok && derr.Err == discovery.ErrFullCluster {
				if cfg.shouldFallbackToProxy() {
					plog.Noticef("discovery cluster full, falling back to %s", fallbackFlagProxy)
					shouldProxy = true
				}
			}
		}
		if shouldProxy {
			err = startProxy(cfg)
		}
	}

	if err != nil {
		if derr, ok := err.(*etcdserver.DiscoveryError); ok {
			switch derr.Err {
			case discovery.ErrDuplicateID:
				plog.Errorf("member %q has previously registered with discovery service token (%s).", cfg.Name, cfg.Durl)
				plog.Errorf("But etcd could not find valid cluster configuration in the given data dir (%s).", cfg.Dir)
				plog.Infof("Please check the given data dir path if the previous bootstrap succeeded")
				plog.Infof("or use a new discovery token if the previous bootstrap failed.")
			case discovery.ErrDuplicateName:
				plog.Errorf("member with duplicated name has registered with discovery service token(%s).", cfg.Durl)
				plog.Errorf("please check (cURL) the discovery token for more information.")
				plog.Errorf("please do not reuse the discovery token and generate a new one to bootstrap the cluster.")
			default:
				plog.Errorf("%v", err)
				plog.Infof("discovery token %s was used, but failed to bootstrap the cluster.", cfg.Durl)
				plog.Infof("please generate a new discovery token and try to bootstrap again.")
			}
			os.Exit(1)
		}

		if strings.Contains(err.Error(), "include") && strings.Contains(err.Error(), "--initial-cluster") {
			plog.Infof("%v", err)
			if cfg.InitialCluster == cfg.InitialClusterFromName(cfg.Name) {
				plog.Infof("forgot to set --initial-cluster flag?")
			}
			if types.URLs(cfg.APUrls).String() == embed.DefaultInitialAdvertisePeerURLs {
				plog.Infof("forgot to set --initial-advertise-peer-urls flag?")
			}
			if cfg.InitialCluster == cfg.InitialClusterFromName(cfg.Name) && len(cfg.Durl) == 0 {
				plog.Infof("if you want to use discovery service, please set --discovery flag.")
			}
			os.Exit(1)
		}
		plog.Fatalf("%v", err)
	}

	osutil.HandleInterrupts()

	if systemdutil.IsRunningSystemd() {
		// At this point, the initialization of etcd is done.
		// The listeners are listening on the TCP ports and ready
		// for accepting connections. The etcd instance should be
		// joined with the cluster and ready to serve incoming
		// connections.
		sent, err := daemon.SdNotify("READY=1")
		if err != nil {
			plog.Errorf("failed to notify systemd for readiness: %v", err)
		}
		if !sent {
			plog.Errorf("forgot to set Type=notify in systemd service file?")
		}
	}

	select {
	case lerr := <-errc:
		// fatal out on listener errors
		plog.Fatal(lerr)
	case <-stopped:
	}

	osutil.Exit(0)
}

// startEtcd runs StartEtcd in addition to hooks needed for standalone etcd.
func startEtcd(cfg *embed.Config) (<-chan struct{}, <-chan error, error) {
	e, err := embed.StartEtcd(cfg)
	if err != nil {
		return nil, nil, err
	}
	osutil.RegisterInterruptHandler(e.Server.Stop)
	return e.Server.StopNotify(), e.Err(), nil
}

// startProxy launches an HTTP proxy for client communication which proxies to other etcd nodes.
func startProxy(cfg *config) error {
	plog.Notice("proxy: this proxy supports v2 API only!")

	pt, err := transport.NewTimeoutTransport(cfg.PeerTLSInfo, time.Duration(cfg.ProxyDialTimeoutMs)*time.Millisecond, time.Duration(cfg.ProxyReadTimeoutMs)*time.Millisecond, time.Duration(cfg.ProxyWriteTimeoutMs)*time.Millisecond)
	if err != nil {
		return err
	}
	pt.MaxIdleConnsPerHost = httpproxy.DefaultMaxIdleConnsPerHost

	tr, err := transport.NewTimeoutTransport(cfg.PeerTLSInfo, time.Duration(cfg.ProxyDialTimeoutMs)*time.Millisecond, time.Duration(cfg.ProxyReadTimeoutMs)*time.Millisecond, time.Duration(cfg.ProxyWriteTimeoutMs)*time.Millisecond)
	if err != nil {
		return err
	}

	cfg.Dir = path.Join(cfg.Dir, "proxy")
	err = os.MkdirAll(cfg.Dir, fileutil.PrivateDirMode)
	if err != nil {
		return err
	}

	var peerURLs []string
	clusterfile := path.Join(cfg.Dir, "cluster")

	b, err := ioutil.ReadFile(clusterfile)
	switch {
	case err == nil:
		if cfg.Durl != "" {
			plog.Warningf("discovery token ignored since the proxy has already been initialized. Valid cluster file found at %q", clusterfile)
		}
		if cfg.DNSCluster != "" {
			plog.Warningf("DNS SRV discovery ignored since the proxy has already been initialized. Valid cluster file found at %q", clusterfile)
		}
		urls := struct{ PeerURLs []string }{}
		err = json.Unmarshal(b, &urls)
		if err != nil {
			return err
		}
		peerURLs = urls.PeerURLs
		plog.Infof("proxy: using peer urls %v from cluster file %q", peerURLs, clusterfile)
	case os.IsNotExist(err):
		var urlsmap types.URLsMap
		urlsmap, _, err = cfg.PeerURLsMapAndToken("proxy")
		if err != nil {
			return fmt.Errorf("error setting up initial cluster: %v", err)
		}

		if cfg.Durl != "" {
			var s string
			s, err = discovery.GetCluster(cfg.Durl, cfg.Dproxy)
			if err != nil {
				return err
			}
			if urlsmap, err = types.NewURLsMap(s); err != nil {
				return err
			}
		}
		peerURLs = urlsmap.URLs()
		plog.Infof("proxy: using peer urls %v ", peerURLs)
	default:
		return err
	}

	clientURLs := []string{}
	uf := func() []string {
		gcls, gerr := etcdserver.GetClusterFromRemotePeers(peerURLs, tr)

		if gerr != nil {
			plog.Warningf("proxy: %v", gerr)
			return []string{}
		}

		clientURLs = gcls.ClientURLs()

		urls := struct{ PeerURLs []string }{gcls.PeerURLs()}
		b, jerr := json.Marshal(urls)
		if jerr != nil {
			plog.Warningf("proxy: error on marshal peer urls %s", jerr)
			return clientURLs
		}

		err = pkgioutil.WriteAndSyncFile(clusterfile+".bak", b, 0600)
		if err != nil {
			plog.Warningf("proxy: error on writing urls %s", err)
			return clientURLs
		}
		err = os.Rename(clusterfile+".bak", clusterfile)
		if err != nil {
			plog.Warningf("proxy: error on updating clusterfile %s", err)
			return clientURLs
		}
		if !reflect.DeepEqual(gcls.PeerURLs(), peerURLs) {
			plog.Noticef("proxy: updated peer urls in cluster file from %v to %v", peerURLs, gcls.PeerURLs())
		}
		peerURLs = gcls.PeerURLs()

		return clientURLs
	}
	ph := httpproxy.NewHandler(pt, uf, time.Duration(cfg.ProxyFailureWaitMs)*time.Millisecond, time.Duration(cfg.ProxyRefreshIntervalMs)*time.Millisecond)
	ph = &cors.CORSHandler{
		Handler: ph,
		Info:    cfg.CorsInfo,
	}

	if cfg.isReadonlyProxy() {
		ph = httpproxy.NewReadonlyHandler(ph)
	}
	// Start a proxy server goroutine for each listen address
	for _, u := range cfg.LCUrls {
		var (
			l      net.Listener
			tlscfg *tls.Config
		)
		if !cfg.ClientTLSInfo.Empty() {
			tlscfg, err = cfg.ClientTLSInfo.ServerConfig()
			if err != nil {
				return err
			}
		}

		l, err := transport.NewListener(u.Host, u.Scheme, tlscfg)
		if err != nil {
			return err
		}

		host := u.String()
		go func() {
			plog.Info("proxy: listening for client requests on ", host)
			mux := http.NewServeMux()
			mux.Handle("/metrics", prometheus.Handler())
			mux.Handle("/", ph)
			plog.Fatal(http.Serve(l, mux))
		}()
	}
	return nil
}

// identifyDataDirOrDie returns the type of the data dir.
// Dies if the datadir is invalid.
func identifyDataDirOrDie(dir string) dirType {
	names, err := fileutil.ReadDir(dir)
	if err != nil {
		if os.IsNotExist(err) {
			return dirEmpty
		}
		plog.Fatalf("error listing data dir: %s", dir)
	}

	var m, p bool
	for _, name := range names {
		switch dirType(name) {
		case dirMember:
			m = true
		case dirProxy:
			p = true
		default:
			plog.Warningf("found invalid file/dir %s under data dir %s (Ignore this if you are upgrading etcd)", name, dir)
		}
	}

	if m && p {
		plog.Fatal("invalid datadir. Both member and proxy directories exist.")
	}
	if m {
		return dirMember
	}
	if p {
		return dirProxy
	}
	return dirEmpty
}

func setupLogging(cfg *config) {
	capnslog.SetGlobalLogLevel(capnslog.INFO)
	if cfg.Debug {
		capnslog.SetGlobalLogLevel(capnslog.DEBUG)
	}
	if cfg.LogPkgLevels != "" {
		repoLog := capnslog.MustRepoLogger("github.com/coreos/etcd")
		settings, err := repoLog.ParseLogLevelConfig(cfg.LogPkgLevels)
		if err != nil {
			plog.Warningf("couldn't parse log level string: %s, continuing with default levels", err.Error())
			return
		}
		repoLog.SetLogLevel(settings)
	}
}

func checkSupportArch() {
	// TODO qualify arm64
	if runtime.GOARCH == "amd64" {
		return
	}
	if env, ok := os.LookupEnv("ETCD_UNSUPPORTED_ARCH"); ok && env == runtime.GOARCH {
		plog.Warningf("running etcd on unsupported architecture %q since ETCD_UNSUPPORTED_ARCH is set", env)
		return
	}
	plog.Errorf("etcd on unsupported platform without ETCD_UNSUPPORTED_ARCH=%s set.", runtime.GOARCH)
	os.Exit(1)
}
<commit_msg>etcdmain: report default advertise detection / fallback<commit_after>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package etcdmain

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path"
	"reflect"
	"runtime"
	"strings"
	"time"

	"github.com/coreos/etcd/discovery"
	"github.com/coreos/etcd/embed"
	"github.com/coreos/etcd/etcdserver"
	"github.com/coreos/etcd/pkg/cors"
	"github.com/coreos/etcd/pkg/fileutil"
	pkgioutil "github.com/coreos/etcd/pkg/ioutil"
	"github.com/coreos/etcd/pkg/osutil"
	"github.com/coreos/etcd/pkg/transport"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/proxy/httpproxy"
	"github.com/coreos/etcd/version"
	"github.com/coreos/go-systemd/daemon"
	systemdutil "github.com/coreos/go-systemd/util"
	"github.com/coreos/pkg/capnslog"
	"github.com/prometheus/client_golang/prometheus"
	"google.golang.org/grpc"
)

type dirType string

var plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "etcdmain")

var (
	dirMember = dirType("member")
	dirProxy  = dirType("proxy")
	dirEmpty  = dirType("empty")
)

func startEtcdOrProxyV2() {
	grpc.EnableTracing = false

	cfg := newConfig()
	defaultInitialCluster := cfg.InitialCluster

	err := cfg.parse(os.Args[1:])
	if err != nil {
		plog.Errorf("error verifying flags, %v. See 'etcd --help'.", err)
		switch err {
		case embed.ErrUnsetAdvertiseClientURLsFlag:
			plog.Errorf("When listening on specific address(es), this etcd process must advertise accessible url(s) to each connected client.")
		}
		os.Exit(1)
	}
	setupLogging(cfg)

	var stopped <-chan struct{}
	var errc <-chan error

	plog.Infof("etcd Version: %s\n", version.Version)
	plog.Infof("Git SHA: %s\n", version.GitSHA)
	plog.Infof("Go Version: %s\n", runtime.Version())
	plog.Infof("Go OS/Arch: %s/%s\n", runtime.GOOS, runtime.GOARCH)

	GoMaxProcs := runtime.GOMAXPROCS(0)
	plog.Infof("setting maximum number of CPUs to %d, total number of available CPUs is %d", GoMaxProcs, runtime.NumCPU())

	// TODO: check whether fields are set instead of whether fields have default value
	defaultHost, defaultHostErr := cfg.IsDefaultHost()
	defaultHostOverride := defaultHost == "" || defaultHostErr == nil
	if (defaultHostOverride || cfg.Name != embed.DefaultName) && cfg.InitialCluster == defaultInitialCluster {
		cfg.InitialCluster = cfg.InitialClusterFromName(cfg.Name)
	}

	if cfg.Dir == "" {
		cfg.Dir = fmt.Sprintf("%v.etcd", cfg.Name)
		plog.Warningf("no data-dir provided, using default data-dir ./%s", cfg.Dir)
	}

	which := identifyDataDirOrDie(cfg.Dir)
	if which != dirEmpty {
		plog.Noticef("the server is already initialized as %v before, starting as etcd %v...", which, which)
		switch which {
		case dirMember:
			stopped, errc, err = startEtcd(&cfg.Config)
		case dirProxy:
			err = startProxy(cfg)
		default:
			plog.Panicf("unhandled dir type %v", which)
		}
	} else {
		shouldProxy := cfg.isProxy()
		if !shouldProxy {
			stopped, errc, err = startEtcd(&cfg.Config)
			if derr, ok := err.(*etcdserver.DiscoveryError); ok && derr.Err == discovery.ErrFullCluster {
				if cfg.shouldFallbackToProxy() {
					plog.Noticef("discovery cluster full, falling back to %s", fallbackFlagProxy)
					shouldProxy = true
				}
			}
		}
		if shouldProxy {
			err = startProxy(cfg)
		}
	}

	if err != nil {
		if derr, ok := err.(*etcdserver.DiscoveryError); ok {
			switch derr.Err {
			case discovery.ErrDuplicateID:
				plog.Errorf("member %q has previously registered with discovery service token (%s).", cfg.Name, cfg.Durl)
				plog.Errorf("But etcd could not find valid cluster configuration in the given data dir (%s).", cfg.Dir)
				plog.Infof("Please check the given data dir path if the previous bootstrap succeeded")
				plog.Infof("or use a new discovery token if the previous bootstrap failed.")
			case discovery.ErrDuplicateName:
				plog.Errorf("member with duplicated name has registered with discovery service token(%s).", cfg.Durl)
				plog.Errorf("please check (cURL) the discovery token for more information.")
				plog.Errorf("please do not reuse the discovery token and generate a new one to bootstrap the cluster.")
			default:
				plog.Errorf("%v", err)
				plog.Infof("discovery token %s was used, but failed to bootstrap the cluster.", cfg.Durl)
				plog.Infof("please generate a new discovery token and try to bootstrap again.")
			}
			os.Exit(1)
		}

		if strings.Contains(err.Error(), "include") && strings.Contains(err.Error(), "--initial-cluster") {
			plog.Infof("%v", err)
			if cfg.InitialCluster == cfg.InitialClusterFromName(cfg.Name) {
				plog.Infof("forgot to set --initial-cluster flag?")
			}
			if types.URLs(cfg.APUrls).String() == embed.DefaultInitialAdvertisePeerURLs {
				plog.Infof("forgot to set --initial-advertise-peer-urls flag?")
			}
			if cfg.InitialCluster == cfg.InitialClusterFromName(cfg.Name) && len(cfg.Durl) == 0 {
				plog.Infof("if you want to use discovery service, please set --discovery flag.")
			}
			os.Exit(1)
		}
		plog.Fatalf("%v", err)
	}

	osutil.HandleInterrupts()

	if systemdutil.IsRunningSystemd() {
		// At this point, the initialization of etcd is done.
		// The listeners are listening on the TCP ports and ready
		// for accepting connections. The etcd instance should be
		// joined with the cluster and ready to serve incoming
		// connections.
		sent, err := daemon.SdNotify("READY=1")
		if err != nil {
			plog.Errorf("failed to notify systemd for readiness: %v", err)
		}
		if !sent {
			plog.Errorf("forgot to set Type=notify in systemd service file?")
		}
	}

	select {
	case lerr := <-errc:
		// fatal out on listener errors
		plog.Fatal(lerr)
	case <-stopped:
	}

	osutil.Exit(0)
}

// startEtcd runs StartEtcd in addition to hooks needed for standalone etcd.
func startEtcd(cfg *embed.Config) (<-chan struct{}, <-chan error, error) {
	defaultHost, dhErr := cfg.IsDefaultHost()
	if defaultHost != "" {
		if dhErr == nil {
			plog.Infof("advertising using detected default host %q", defaultHost)
		} else {
			plog.Noticef("failed to detect default host, advertise falling back to %q (%v)", defaultHost, dhErr)
		}
	}

	e, err := embed.StartEtcd(cfg)
	if err != nil {
		return nil, nil, err
	}
	osutil.RegisterInterruptHandler(e.Server.Stop)
	return e.Server.StopNotify(), e.Err(), nil
}

// startProxy launches an HTTP proxy for client communication which proxies to other etcd nodes.
func startProxy(cfg *config) error {
	plog.Notice("proxy: this proxy supports v2 API only!")

	pt, err := transport.NewTimeoutTransport(cfg.PeerTLSInfo, time.Duration(cfg.ProxyDialTimeoutMs)*time.Millisecond, time.Duration(cfg.ProxyReadTimeoutMs)*time.Millisecond, time.Duration(cfg.ProxyWriteTimeoutMs)*time.Millisecond)
	if err != nil {
		return err
	}
	pt.MaxIdleConnsPerHost = httpproxy.DefaultMaxIdleConnsPerHost

	tr, err := transport.NewTimeoutTransport(cfg.PeerTLSInfo, time.Duration(cfg.ProxyDialTimeoutMs)*time.Millisecond, time.Duration(cfg.ProxyReadTimeoutMs)*time.Millisecond, time.Duration(cfg.ProxyWriteTimeoutMs)*time.Millisecond)
	if err != nil {
		return err
	}

	cfg.Dir = path.Join(cfg.Dir, "proxy")
	err = os.MkdirAll(cfg.Dir, fileutil.PrivateDirMode)
	if err != nil {
		return err
	}

	var peerURLs []string
	clusterfile := path.Join(cfg.Dir, "cluster")

	b, err := ioutil.ReadFile(clusterfile)
	switch {
	case err == nil:
		if cfg.Durl != "" {
			plog.Warningf("discovery token ignored since the proxy has already been initialized. Valid cluster file found at %q", clusterfile)
		}
		if cfg.DNSCluster != "" {
			plog.Warningf("DNS SRV discovery ignored since the proxy has already been initialized. Valid cluster file found at %q", clusterfile)
		}
		urls := struct{ PeerURLs []string }{}
		err = json.Unmarshal(b, &urls)
		if err != nil {
			return err
		}
		peerURLs = urls.PeerURLs
		plog.Infof("proxy: using peer urls %v from cluster file %q", peerURLs, clusterfile)
	case os.IsNotExist(err):
		var urlsmap types.URLsMap
		urlsmap, _, err = cfg.PeerURLsMapAndToken("proxy")
		if err != nil {
			return fmt.Errorf("error setting up initial cluster: %v", err)
		}

		if cfg.Durl != "" {
			var s string
			s, err = discovery.GetCluster(cfg.Durl, cfg.Dproxy)
			if err != nil {
				return err
			}
			if urlsmap, err = types.NewURLsMap(s); err != nil {
				return err
			}
		}
		peerURLs = urlsmap.URLs()
		plog.Infof("proxy: using peer urls %v ", peerURLs)
	default:
		return err
	}

	clientURLs := []string{}
	uf := func() []string {
		gcls, gerr := etcdserver.GetClusterFromRemotePeers(peerURLs, tr)

		if gerr != nil {
			plog.Warningf("proxy: %v", gerr)
			return []string{}
		}

		clientURLs = gcls.ClientURLs()

		urls := struct{ PeerURLs []string }{gcls.PeerURLs()}
		b, jerr := json.Marshal(urls)
		if jerr != nil {
			plog.Warningf("proxy: error on marshal peer urls %s", jerr)
			return clientURLs
		}

		err = pkgioutil.WriteAndSyncFile(clusterfile+".bak", b, 0600)
		if err != nil {
			plog.Warningf("proxy: error on writing urls %s", err)
			return clientURLs
		}
		err = os.Rename(clusterfile+".bak", clusterfile)
		if err != nil {
			plog.Warningf("proxy: error on updating clusterfile %s", err)
			return clientURLs
		}
		if !reflect.DeepEqual(gcls.PeerURLs(), peerURLs) {
			plog.Noticef("proxy: updated peer urls in cluster file from %v to %v", peerURLs, gcls.PeerURLs())
		}
		peerURLs = gcls.PeerURLs()

		return clientURLs
	}
	ph := httpproxy.NewHandler(pt, uf, time.Duration(cfg.ProxyFailureWaitMs)*time.Millisecond, time.Duration(cfg.ProxyRefreshIntervalMs)*time.Millisecond)
	ph = &cors.CORSHandler{
		Handler: ph,
		Info:    cfg.CorsInfo,
	}

	if cfg.isReadonlyProxy() {
		ph = httpproxy.NewReadonlyHandler(ph)
	}
	// Start a proxy server goroutine for each listen address
	for _, u := range cfg.LCUrls {
		var (
			l      net.Listener
			tlscfg *tls.Config
		)
		if !cfg.ClientTLSInfo.Empty() {
			tlscfg, err = cfg.ClientTLSInfo.ServerConfig()
			if err != nil {
				return err
			}
		}

		l, err := transport.NewListener(u.Host, u.Scheme, tlscfg)
		if err != nil {
			return err
		}

		host := u.String()
		go func() {
			plog.Info("proxy: listening for client requests on ", host)
			mux := http.NewServeMux()
			mux.Handle("/metrics", prometheus.Handler())
			mux.Handle("/", ph)
			plog.Fatal(http.Serve(l, mux))
		}()
	}
	return nil
}

// identifyDataDirOrDie returns the type of the data dir.
// Dies if the datadir is invalid.
func identifyDataDirOrDie(dir string) dirType {
	names, err := fileutil.ReadDir(dir)
	if err != nil {
		if os.IsNotExist(err) {
			return dirEmpty
		}
		plog.Fatalf("error listing data dir: %s", dir)
	}

	var m, p bool
	for _, name := range names {
		switch dirType(name) {
		case dirMember:
			m = true
		case dirProxy:
			p = true
		default:
			plog.Warningf("found invalid file/dir %s under data dir %s (Ignore this if you are upgrading etcd)", name, dir)
		}
	}

	if m && p {
		plog.Fatal("invalid datadir. Both member and proxy directories exist.")
	}
	if m {
		return dirMember
	}
	if p {
		return dirProxy
	}
	return dirEmpty
}

func setupLogging(cfg *config) {
	capnslog.SetGlobalLogLevel(capnslog.INFO)
	if cfg.Debug {
		capnslog.SetGlobalLogLevel(capnslog.DEBUG)
	}
	if cfg.LogPkgLevels != "" {
		repoLog := capnslog.MustRepoLogger("github.com/coreos/etcd")
		settings, err := repoLog.ParseLogLevelConfig(cfg.LogPkgLevels)
		if err != nil {
			plog.Warningf("couldn't parse log level string: %s, continuing with default levels", err.Error())
			return
		}
		repoLog.SetLogLevel(settings)
	}
}

func checkSupportArch() {
	// TODO qualify arm64
	if runtime.GOARCH == "amd64" {
		return
	}
	if env, ok := os.LookupEnv("ETCD_UNSUPPORTED_ARCH"); ok && env == runtime.GOARCH {
		plog.Warningf("running etcd on unsupported architecture %q since ETCD_UNSUPPORTED_ARCH is set", env)
		return
	}
	plog.Errorf("etcd on unsupported platform without ETCD_UNSUPPORTED_ARCH=%s set.", runtime.GOARCH)
	os.Exit(1)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"runtime"
	"time"

	"github.com/smartystreets/go-disruptor"
)

const (
	BufferSize = 1024 * 64
	BufferMask = BufferSize - 1
	Iterations = 1000000 * 100
)

var ringBuffer = [BufferSize]int64{}

func main() {
	runtime.GOMAXPROCS(2)

	written, read := disruptor.NewCursor(), disruptor.NewCursor()
	started := time.Now()

	go publish(written, read)
	consume(written, read)

	finished := time.Now()
	fmt.Println(Iterations, finished.Sub(started))
}

func publish(written, read *disruptor.Cursor) {
	previous := disruptor.InitialSequenceValue
	gate := disruptor.InitialSequenceValue

	for previous <= Iterations {
		next := previous + 1
		wrap := next - BufferSize

		for wrap > gate {
			gate = read.Sequence
		}

		ringBuffer[next&BufferMask] = next
		written.Sequence = next
		previous = next
	}
}
func consume(written, read *disruptor.Cursor) {
	sequence := int64(0)
	for sequence < Iterations {
		maximum := written.Sequence
		for maximum <= sequence {
			maximum = written.Sequence
			time.Sleep(time.Microsecond)
		}

		for sequence < maximum {
			if ringBuffer[sequence&BufferMask] > 0 {
			}
			sequence++
		}

		read.Sequence = maximum
		sequence = maximum + 1
	}
}
<commit_msg>More experiments.<commit_after>package main

import (
	"fmt"
	"runtime"
	"time"

	"github.com/smartystreets/go-disruptor"
)

const (
	BufferSize = 1024 * 64
	BufferMask = BufferSize - 1
	Iterations = 1000000 * 100
)

var ringBuffer = [BufferSize]int64{}

func main() {
	runtime.GOMAXPROCS(2)

	written, read := disruptor.NewCursor(), disruptor.NewCursor()
	started := time.Now()

	go publish(written, read)
	consume(written, read)

	finished := time.Now()
	fmt.Println(Iterations, finished.Sub(started))
}

func publish(written, read *disruptor.Cursor) {
	previous := disruptor.InitialSequenceValue
	gate := disruptor.InitialSequenceValue

	for previous <= Iterations {
		next := previous + 1
		wrap := next - BufferSize

		for wrap > gate {
			gate = read.Sequence
			// if wrap > gate {
			// 	time.Sleep(time.Microsecond)
			// }
		}

		ringBuffer[next&BufferMask] = next
		written.Sequence = next
		previous = next
	}
}
func consume(written, read *disruptor.Cursor) {
	for sequence, gate := int64(0), int64(0); sequence < Iterations; sequence++ {
		for gate <= sequence {
			gate = written.Sequence
			if gate <= sequence {
				time.Sleep(time.Microsecond)
			}
		}

		if ringBuffer[sequence&BufferMask] > 0 {
		}

		read.Sequence = sequence
	}
}
<|endoftext|>


<commit_before>// +build darwin linux

package main

import (
	"github.com/pankona/gomo-simra/example/scene"
	"github.com/pankona/gomo-simra/peer"
	"github.com/pankona/gomo-simra/simra"
)

func eventHandle(onStart, onStop chan bool) {
	for {
	loop:
		select {
		case <-onStart:
			peer.LogDebug("receive chan. onStart")
			engine := simra.GetInstance()
			engine.SetScene(&scene.Title{})
		case <-onStop:
			peer.LogDebug("receive chan. onStop")
			break loop
		}
	}
}

func main() {
	peer.LogDebug("[IN]")
	engine := simra.GetInstance()

	onStart := make(chan bool)
	onStop := make(chan bool)
	go eventHandle(onStart, onStop)
	engine.Start(onStart, onStop)
	peer.LogDebug("[OUT]")
}
<commit_msg>remove breaking inifinite loop at onStop.<commit_after>// +build darwin linux

package main

import (
	"github.com/pankona/gomo-simra/example/scene"
	"github.com/pankona/gomo-simra/peer"
	"github.com/pankona/gomo-simra/simra"
)

func eventHandle(onStart, onStop chan bool) {
	for {
		select {
		case <-onStart:
			peer.LogDebug("receive chan. onStart")
			engine := simra.GetInstance()
			engine.SetScene(&scene.Title{})
		case <-onStop:
			peer.LogDebug("receive chan. onStop")
		}
	}
}

func main() {
	peer.LogDebug("[IN]")
	engine := simra.GetInstance()

	onStart := make(chan bool)
	onStop := make(chan bool)
	go eventHandle(onStart, onStop)
	engine.Start(onStart, onStop)
	peer.LogDebug("[OUT]")
}
<|endoftext|>


<commit_before>package logrus

import "time"

const DefaultTimestampFormat = time.RFC3339

// The Formatter interface is used to implement a custom Formatter. It takes an
// `Entry`. It exposes all the fields, including the default ones:
//
// * `entry.Data["msg"]`. The message passed from Info, Warn, Error ..
// * `entry.Data["time"]`. The timestamp.
// * `entry.Data["level"]. The level the entry was logged at.
//
// Any additional fields added with `WithField` or `WithFields` are also in
// `entry.Data`. Format is expected to return an array of bytes which are then
// logged to `logger.Out`.
type Formatter interface {
	Format(*Entry) ([]byte, error)
}

// This is to not silently overwrite `time`, `msg` and `level` fields when
// dumping it. If this code wasn't there doing:
//
//  logrus.WithField("level", 1).Info("hello")
//
// Would just silently drop the user provided level. Instead with this code
// it'll logged as:
//
//  {"level": "info", "fields.level": 1, "msg": "hello", "time": "..."}
//
// It's not exported because it's still using Data in an opinionated way. It's to
// avoid code duplication between the two default formatters.
func prefixFieldClashes(data Fields) {
	_, ok := data["time"]
	if ok {
		data["fields.time"] = data["time"]
	}

	_, ok = data["msg"]
	if ok {
		data["fields.msg"] = data["msg"]
	}

	_, ok = data["level"]
	if ok {
		data["fields.level"] = data["level"]
	}
}
<commit_msg>formatter.go: simplify prefixFieldClashes(Fields)<commit_after>package logrus

import "time"

const DefaultTimestampFormat = time.RFC3339

// The Formatter interface is used to implement a custom Formatter. It takes an
// `Entry`. It exposes all the fields, including the default ones:
//
// * `entry.Data["msg"]`. The message passed from Info, Warn, Error ..
// * `entry.Data["time"]`. The timestamp.
// * `entry.Data["level"]. The level the entry was logged at.
//
// Any additional fields added with `WithField` or `WithFields` are also in
// `entry.Data`. Format is expected to return an array of bytes which are then
// logged to `logger.Out`.
type Formatter interface {
	Format(*Entry) ([]byte, error)
}

// This is to not silently overwrite `time`, `msg` and `level` fields when
// dumping it. If this code wasn't there doing:
//
//  logrus.WithField("level", 1).Info("hello")
//
// Would just silently drop the user provided level. Instead with this code
// it'll logged as:
//
//  {"level": "info", "fields.level": 1, "msg": "hello", "time": "..."}
//
// It's not exported because it's still using Data in an opinionated way. It's to
// avoid code duplication between the two default formatters.
func prefixFieldClashes(data Fields) {
	if t, ok := data["time"]; ok {
		data["fields.time"] = t
	}

	if m, ok := data["msg"]; ok {
		data["fields.msg"] = m
	}

	if l, ok := data["level"]; ok {
		data["fields.level"] = l
	}
}
<|endoftext|>


<commit_before>package fritz

import (
	"fmt"
	"net/http"
	"strings"

	"github.com/bpicode/fritzctl/httpread"
	"github.com/bpicode/fritzctl/logger"
)

// HomeAutomationAPI API definition, guided by
// https://avm.de/fileadmin/user_upload/Global/Service/Schnittstellen/AHA-HTTP-Interface.pdf.
type HomeAutomationAPI interface {
	ListDevices() (*Devicelist, error)
	NameToAinTable() (map[string]string, error)
	SwitchOn(ain string) (string, error)
	SwitchOff(ain string) (string, error)
	Toggle(ain string) (string, error)
	ApplyTemperature(value float64, ain string) (string, error)
}

// HomeAutomation creates a Fritz AHA API from a given client.
func HomeAutomation(client *Client) HomeAutomationAPI {
	return &ahaHTTP{client: client}
}

type ahaHTTP struct {
	client *Client
}

func (aha *ahaHTTP) getf(url string) func() (*http.Response, error) {
	return func() (*http.Response, error) {
		logger.Debug("GET", url)
		return aha.client.HTTPClient.Get(url)
	}
}

// ListDevices lists the basic data of the smart home devices.
func (aha *ahaHTTP) ListDevices() (*Devicelist, error) {
	url := aha.homeAutoSwitch().
		query("switchcmd", "getdevicelistinfos").
		build()
	var deviceList Devicelist
	errRead := httpread.ReadFullyXML(aha.getf(url), &deviceList)
	return &deviceList, errRead
}

// SwitchOn switches a device on. The device is identified by its AIN.
func (aha *ahaHTTP) SwitchOn(ain string) (string, error) {
	return aha.switchForAin(ain, "setswitchon")
}

// SwitchOff switches a device off. The device is identified by its AIN.
func (aha *ahaHTTP) SwitchOff(ain string) (string, error) {
	return aha.switchForAin(ain, "setswitchoff")
}

// Toggle toggles the on/off state of a device. The device is identified by its AIN.
func (aha *ahaHTTP) Toggle(ain string) (string, error) {
	url := aha.homeAutoSwitch().
		query("ain", ain).
		query("switchcmd", "setswitchtoggle").
		build()
	return httpread.ReadFullyString(aha.getf(url))
}

// ApplyTemperature sets the desired temperature on a "HKR" device. The device is identified by its AIN.
func (aha *ahaHTTP) ApplyTemperature(value float64, ain string) (string, error) {
	param, err := temperatureParam(value)
	if err != nil {
		return "", err
	}
	url := aha.homeAutoSwitch().
		query("ain", ain).
		query("switchcmd", "sethkrtsoll").
		query("param", fmt.Sprintf("%d", param)).
		build()
	return httpread.ReadFullyString(aha.getf(url))
}

func (aha *ahaHTTP) switchForAin(ain, command string) (string, error) {
	url := aha.homeAutoSwitch().
		query("ain", ain).
		query("switchcmd", command).
		build()
	return httpread.ReadFullyString(aha.getf(url))
}

// NameToAinTable returns a lookup name -> AIN.
func (aha *ahaHTTP) NameToAinTable() (map[string]string, error) {
	devList, err := aha.ListDevices()
	if err != nil {
		return nil, err
	}
	devs := devList.Devices
	table := make(map[string]string, len(devs))
	for _, dev := range devs {
		table[dev.Name] = strings.Replace(dev.Identifier, " ", "", -1)
	}
	return table, nil
}

func (aha *ahaHTTP) homeAutoSwitch() fritzURLBuilder {
	return newURLBuilder(aha.client.Config).path(homeAutomationURI).query("sid", aha.client.SessionInfo.SID)
}

func temperatureParam(t float64) (int64, error) {
	doubled := round(2 * t)
	regular := doubled >= 16 && doubled <= 56
	special := doubled == 253 || doubled == 254
	if ! (regular || special) {
		return 0, fmt.Errorf("invalid temperature value: %.1f°C is not contained in the set of acceptable values: 8-28°C, 126.5, 127", t)
	}
	return doubled, nil
}

// round rounds a float64 value to an integer.
func round(v float64) int64 {
	return int64(v + 0.5)
}
<commit_msg>gofmt: remove whitespace<commit_after>package fritz

import (
	"fmt"
	"net/http"
	"strings"

	"github.com/bpicode/fritzctl/httpread"
	"github.com/bpicode/fritzctl/logger"
)

// HomeAutomationAPI API definition, guided by
// https://avm.de/fileadmin/user_upload/Global/Service/Schnittstellen/AHA-HTTP-Interface.pdf.
type HomeAutomationAPI interface {
	ListDevices() (*Devicelist, error)
	NameToAinTable() (map[string]string, error)
	SwitchOn(ain string) (string, error)
	SwitchOff(ain string) (string, error)
	Toggle(ain string) (string, error)
	ApplyTemperature(value float64, ain string) (string, error)
}

// HomeAutomation creates a Fritz AHA API from a given client.
func HomeAutomation(client *Client) HomeAutomationAPI {
	return &ahaHTTP{client: client}
}

type ahaHTTP struct {
	client *Client
}

func (aha *ahaHTTP) getf(url string) func() (*http.Response, error) {
	return func() (*http.Response, error) {
		logger.Debug("GET", url)
		return aha.client.HTTPClient.Get(url)
	}
}

// ListDevices lists the basic data of the smart home devices.
func (aha *ahaHTTP) ListDevices() (*Devicelist, error) {
	url := aha.homeAutoSwitch().
		query("switchcmd", "getdevicelistinfos").
		build()
	var deviceList Devicelist
	errRead := httpread.ReadFullyXML(aha.getf(url), &deviceList)
	return &deviceList, errRead
}

// SwitchOn switches a device on. The device is identified by its AIN.
func (aha *ahaHTTP) SwitchOn(ain string) (string, error) {
	return aha.switchForAin(ain, "setswitchon")
}

// SwitchOff switches a device off. The device is identified by its AIN.
func (aha *ahaHTTP) SwitchOff(ain string) (string, error) {
	return aha.switchForAin(ain, "setswitchoff")
}

// Toggle toggles the on/off state of a device. The device is identified by its AIN.
func (aha *ahaHTTP) Toggle(ain string) (string, error) {
	url := aha.homeAutoSwitch().
		query("ain", ain).
		query("switchcmd", "setswitchtoggle").
		build()
	return httpread.ReadFullyString(aha.getf(url))
}

// ApplyTemperature sets the desired temperature on a "HKR" device. The device is identified by its AIN.
func (aha *ahaHTTP) ApplyTemperature(value float64, ain string) (string, error) {
	param, err := temperatureParam(value)
	if err != nil {
		return "", err
	}
	url := aha.homeAutoSwitch().
		query("ain", ain).
		query("switchcmd", "sethkrtsoll").
		query("param", fmt.Sprintf("%d", param)).
		build()
	return httpread.ReadFullyString(aha.getf(url))
}

func (aha *ahaHTTP) switchForAin(ain, command string) (string, error) {
	url := aha.homeAutoSwitch().
		query("ain", ain).
		query("switchcmd", command).
		build()
	return httpread.ReadFullyString(aha.getf(url))
}

// NameToAinTable returns a lookup name -> AIN.
func (aha *ahaHTTP) NameToAinTable() (map[string]string, error) {
	devList, err := aha.ListDevices()
	if err != nil {
		return nil, err
	}
	devs := devList.Devices
	table := make(map[string]string, len(devs))
	for _, dev := range devs {
		table[dev.Name] = strings.Replace(dev.Identifier, " ", "", -1)
	}
	return table, nil
}

func (aha *ahaHTTP) homeAutoSwitch() fritzURLBuilder {
	return newURLBuilder(aha.client.Config).path(homeAutomationURI).query("sid", aha.client.SessionInfo.SID)
}

func temperatureParam(t float64) (int64, error) {
	doubled := round(2 * t)
	regular := doubled >= 16 && doubled <= 56
	special := doubled == 253 || doubled == 254
	if !(regular || special) {
		return 0, fmt.Errorf("invalid temperature value: %.1f°C is not contained in the set of acceptable values: 8-28°C, 126.5, 127", t)
	}
	return doubled, nil
}

// round rounds a float64 value to an integer.
func round(v float64) int64 {
	return int64(v + 0.5)
}
<|endoftext|>


<commit_before>package utils

import (
	"reflect"
	"testing"
)

func TestDomainCreation(t *testing.T) {
	var tests = map[string]string{
		"foo":           "foo",
		"foo.":          "foo",
		".foo.docker.":  "foo.docker",
		".foo..docker.": "foo.docker",
		"foo.docker..":  "foo.docker",
	}

	for input, expected := range tests {
		t.Log(input)
		d := NewDomain(input)
		if actual := d.String(); actual != expected {
			t.Error(input, "Expected:", expected, "Got:", actual)
		}
	}
	input := "127.0.0.1,127.0.0.2"
	expected := nameservers{input}
	if actual := []string{"127.0.0.1", "127.0.0.2"}; reflect.DeepEqual(actual, expected) {
		t.Error(input, "Expected:", expected, "Got:", actual)
	}
}

func TestNewConfig(t *testing.T) {
	config := NewConfig()
	if !reflect.DeepEqual(config.DnsAddr, ":53") {
		t.Error("DnsAddr error")
	}
}

func TestNameserverString(t *testing.T) {
	ns := NewConfig().Nameservers
	t.Log(ns.String())
}

func TestNameserverSet(t *testing.T){
	ns := NewConfig().Nameservers.Set("8.8.4.4:53").String()
	t.Log(ns)
}
<commit_msg>test: config update<commit_after>package utils

import (
	"reflect"
	"testing"
)

func TestDomainCreation(t *testing.T) {
	var tests = map[string]string{
		"foo":           "foo",
		"foo.":          "foo",
		".foo.docker.":  "foo.docker",
		".foo..docker.": "foo.docker",
		"foo.docker..":  "foo.docker",
	}

	for input, expected := range tests {
		t.Log(input)
		d := NewDomain(input)
		if actual := d.String(); actual != expected {
			t.Error(input, "Expected:", expected, "Got:", actual)
		}
	}
	input := "127.0.0.1,127.0.0.2"
	expected := nameservers{input}
	if actual := []string{"127.0.0.1", "127.0.0.2"}; reflect.DeepEqual(actual, expected) {
		t.Error(input, "Expected:", expected, "Got:", actual)
	}
}

func TestNewConfig(t *testing.T) {
	config := NewConfig()
	if !reflect.DeepEqual(config.DnsAddr, ":53") {
		t.Error("DnsAddr error")
	}
}

func TestNameserverString(t *testing.T) {
	ns := NewConfig().Nameservers
	t.Log(ns.String())
}

func TestNameserverSet(t *testing.T){
	ns := NewConfig().Nameservers.Set("8.8.4.4:53")
	t.Log(ns.String())
}
<|endoftext|>


<commit_before><commit_msg>bugfix<commit_after><|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"strconv"
	"strings"
)

func main() {
	door := "cxdnnyjw"
	var password string
	for counter := 0; len(password) < 8; counter++ {
		sumBytes := md5.Sum([]byte(door + strconv.Itoa(counter)))
		sumStr := hex.EncodeToString(sumBytes[:])
		if strings.HasPrefix(sumStr, "00000") {
			password += sumStr[5:6]
		}
	}
	fmt.Println(password)
}
<commit_msg>Day 5 Part 2<commit_after>package main

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"os"
	"strconv"
	"strings"
)

func pass1(count int, password []byte, hexSum string) bool {
	password[count] = []byte(hexSum[5:6])[0]
	return true
}

func pass2(count int, password []byte, hexSum string) bool {
	posValue, _ := hex.DecodeString("0" + hexSum[5:6])
	pos := posValue[0]
	value := []byte(hexSum[6:7])[0]

	if pos > 7 {
		return false
	}

	if password[pos] != 32 {
		return false
	}

	password[pos] = value

	return true
}

func main() {

	passFunc := pass1
	if len(os.Args) >= 2 && os.Args[1] == "2" {
		passFunc = pass2
	}

	door := "cxdnnyjw"
	password := make([]byte, 8)
	for i := range password {
		password[i] = 32
	}

	for counter, pwcount := 0, 0; pwcount < 8; counter++ {
		sumBytes := md5.Sum([]byte(door + strconv.Itoa(counter)))
		sumStr := hex.EncodeToString(sumBytes[:])
		if strings.HasPrefix(sumStr, "00000") {
			if passFunc(pwcount, password, sumStr) {
				fmt.Println(string(password))
				pwcount++
			}
		}
	}
	fmt.Println(string(password))
}
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"github.com/hpehl/whatunga/model"
	"github.com/hpehl/whatunga/path"
	"github.com/oleiade/reflections"
	"reflect"
	"strings"
)

var cdUsage = "cd <path> | cd .. | cd /"

var cd = Command{
	"cd",
	"Changes the current context to the specified path.",
	cdUsage,
	`Changes the current context to the specified path. The path specifies
the name of an object in the project model like "config.templates.domain".

If the object is part of a collection you can also use index on the objects
type. Both numeric and name based indizes are supported:

	cd hosts[master].servers[4]

changes the current context to the fifth server of host "master".`,
	// tab completer
	func(project *model.Project, query, cmdline string) ([]string, int) {
		tokens := strings.Fields(cmdline)

		if len(tokens) == 1 && query == "" {
			// just the command was given, return matches based on the current path
			return keys(ls(path.CurrentPath, project)), 0

		} else if len(tokens) > 1 {
			// check the stuff after "cd"
			pathPart, reminder := split(tokens[1])

			if pathPart == "" {
				// no full path. return matches based on the reminder
				return matchesFor(path.CurrentPath, reminder, project)

			} else {
				// some kind of path given. append to path.CurrentPath
				p, err := path.Parse(pathPart)
				if err != nil {
					return nil, 0
				}
				fullPath := path.CurrentPath.Append(p)
				return matchesFor(fullPath, reminder, project)
			}
		}
		return nil, 0
	},
	// action
	func(project *model.Project, args []string) error {
		if len(args) == 0 {
			return fmt.Errorf("Missing argument. Usage: %s", cdUsage)
		}
		if len(args) > 1 {
			return fmt.Errorf("Too many arguments. Usage: %s", cdUsage)
		}

		if args[0] == ".." {
			if path.CurrentPath.IsEmpty() {
				return fmt.Errorf("Cannot go up one level: Already at root")
			}
			path.CurrentPath = path.CurrentPath[0 : len(path.CurrentPath)-1]
		} else if args[0] == "/" {
			if path.CurrentPath.IsEmpty() {
				return fmt.Errorf("Already at root")
			}
			path.CurrentPath = path.CurrentPath[:0]
		} else {
			changeTo, err := path.Parse(args[0])
			if err != nil {
				return err
			}
			full := path.CurrentPath.Append(changeTo)
			if _, err := full.Resolve(project); err != nil {
				return err
			}
			path.CurrentPath = full
		}
		return nil
	},
}

func matchesFor(context path.Path, reminder string, project *model.Project) ([]string, int) {
	children := ls(context, project)

	if contains(keys(children), reminder) {
		// yep. check the type to find the right CompletionAppendChar and return that match
		if children[reminder] == reflect.Struct {
			if hasNestedStructs(context, reminder, project) {
				return []string{reminder}, '.'
			} else {
				return []string{reminder}, ' '
			}
		} else {
			return []string{reminder}, '['
		}
	} else {
		// no. return a slice of matches based on the reminder
		var matches []string
		for _, name := range keys(children) {
			if strings.HasPrefix(name, reminder) {
				matches = append(matches, name)
			}
		}
		if len(matches) == 1 {
			if children[matches[0]] == reflect.Struct {
				if hasNestedStructs(context, matches[0], project) {
					return matches, '.'
				} else {
					return matches, ' '
				}
			} else {
				return matches, '['
			}
		} else {
			return matches, 0
		}
	}
}

func ls(context path.Path, project *model.Project) map[string]reflect.Kind {
	var matches = make(map[string]reflect.Kind)

	obj, err := context.Resolve(project)
	if err == nil {
		tags, err := reflections.Tags(obj, "json")
		if err == nil {
			for field, tag := range tags {
				kind, _ := reflections.GetFieldKind(obj, field)
				if kind == reflect.Struct || kind == reflect.Slice {
					matches[tag] = kind
				}
			}
		}
	}
	return matches
}

func hasNestedStructs(context path.Path, name string, project *model.Project) bool {
	nested, err := path.Parse(name)
	if err == nil {
		children := ls(context.Append(nested), project)
		return len(children) > 0
	}
	return false
}

func keys(m map[string]reflect.Kind) []string {
	keys := make([]string, 0, len(m))
	for k := range m {
		keys = append(keys, k)
	}
	return keys
}

func split(arg string) (string, string) {
	var path, reminder string
	lastDot := strings.LastIndex(arg, ".")
	if lastDot != -1 {
		path = arg[0:lastDot]
		reminder = arg[lastDot+1:]
	} else {
		path = ""
		reminder = arg
	}
	return path, reminder
}

func contains(s []string, e string) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}
	return false
}
<commit_msg>Fix cd command<commit_after>package command

import (
	"fmt"
	"github.com/hpehl/whatunga/model"
	"github.com/hpehl/whatunga/path"
	"github.com/oleiade/reflections"
	"reflect"
	"strings"
)

var cdUsage = "cd <path> | cd .. | cd /"

var cd = Command{
	"cd",
	"Changes the current context to the specified path.",
	cdUsage,
	`Changes the current context to the specified path. The path specifies
the name of an object in the project model like "config.templates.domain".

If the object is part of a collection you can also use index on the objects
type. Both numeric and name based indizes are supported:

	cd hosts[master].servers[4]

changes the current context to the fifth server of host "master".`,
	// tab completer
	func(project *model.Project, query, cmdline string) ([]string, int) {
		tokens := strings.Fields(cmdline)

		if len(tokens) == 1 && query == "" {
			// just the command was given, return matches based on the current path
			return keys(ls(path.CurrentPath, project)), 0

		} else if len(tokens) > 1 {
			// check the stuff after "cd"
			pathPart, reminder := split(tokens[1])

			if pathPart == "" {
				// no full path. return matches based on the reminder
				return matchesFor(path.CurrentPath, reminder, project)

			} else {
				// some kind of path given. append to path.CurrentPath
				p, err := path.Parse(pathPart)
				if err != nil {
					return nil, 0
				}
				fullPath := path.CurrentPath.Append(p)
				return matchesFor(fullPath, reminder, project)
			}
		}
		return nil, 0
	},
	// action
	func(project *model.Project, args []string) error {
		if len(args) == 0 {
			return fmt.Errorf("Missing argument. Usage: %s", cdUsage)
		}
		if len(args) > 1 {
			return fmt.Errorf("Too many arguments. Usage: %s", cdUsage)
		}

		if args[0] == ".." {
			if path.CurrentPath.IsEmpty() {
				return fmt.Errorf("Cannot go up one level: Already at root")
			}
			path.CurrentPath = path.CurrentPath[0 : len(path.CurrentPath)-1]
		} else if args[0] == "/" {
			if path.CurrentPath.IsEmpty() {
				return fmt.Errorf("Already at root")
			}
			path.CurrentPath = path.CurrentPath[:0]
		} else {
			changeTo, err := path.Parse(args[0])
			if err != nil {
				return err
			}
			full := path.CurrentPath.Append(changeTo)
			if _, err := full.Resolve(project); err != nil {
				return err
			}
			path.CurrentPath = full
		}
		return nil
	},
}

func matchesFor(context path.Path, reminder string, project *model.Project) ([]string, int) {
	children := ls(context, project)

	if contains(keys(children), reminder) {
		if children[reminder] == reflect.Struct {
			return []string{reminder}, 0
		} else {
			return []string{reminder}, '['
		}
	} else {
		var matches []string
		for _, name := range keys(children) {
			if strings.HasPrefix(name, reminder) {
				matches = append(matches, name)
			}
		}
		if len(matches) == 1 {
			if children[matches[0]] == reflect.Struct {
				return matches, 0
			} else {
				return matches, '['
			}
		} else {
			return matches, 0
		}
	}
}

func ls(context path.Path, project *model.Project) map[string]reflect.Kind {
	var matches = make(map[string]reflect.Kind)

	obj, err := context.Resolve(project)
	if err == nil {
		tags, err := reflections.Tags(obj, "json")
		if err == nil {
			for field, tag := range tags {
				kind, _ := reflections.GetFieldKind(obj, field)
				if kind == reflect.Struct || kind == reflect.Slice {
					matches[tag] = kind
				}
			}
		}
	}
	return matches
}

func keys(m map[string]reflect.Kind) []string {
	keys := make([]string, 0, len(m))
	for k := range m {
		keys = append(keys, k)
	}
	return keys
}

func split(arg string) (string, string) {
	var path, reminder string
	lastDot := strings.LastIndex(arg, ".")
	if lastDot != -1 {
		path = arg[0:lastDot]
		reminder = arg[lastDot+1:]
	} else {
		path = ""
		reminder = arg
	}
	return path, reminder
}

func contains(s []string, e string) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package cupaloy

import (
	"fmt"
	"os"
	"strings"
	"testing"
)

// Snapshotter is the API for taking snapshots of values in your tests.
type Snapshotter interface {
	// Snapshot compares the given value to the it's previous value stored on the filesystem.
	// An error containing a diff is returned if the snapshots do not match.
	// Snapshot determines the snapshot file automatically from the name of the calling function.
	Snapshot(i ...interface{}) error

	// SnapshotMulti is identical to Snapshot but can be called multiple times from the same function.
	// This is done by providing a unique snapshotId for each invocation.
	SnapshotMulti(snapshotID string, i ...interface{}) error

	// SnapshotT is identical to Snapshot but gets the snapshot name using
	// t.Name() and calls t.Fail() directly if the snapshots do not match.
	SnapshotT(t *testing.T, i ...interface{})
}

// New constructs a new, configured instance of cupaloy using the given Configurators.
func New(configurators ...Configurator) Snapshotter {
	config := defaultConfig()

	for _, configurator := range configurators {
		configurator(config)
	}

	return config
}

// Snapshot calls Snapshotter.Snapshot with the default config.
func Snapshot(i ...interface{}) error {
	return defaultConfig().snapshot(getNameOfCaller(), i...)
}

// SnapshotMulti calls Snapshotter.SnapshotMulti with the default config.
func SnapshotMulti(snapshotID string, i ...interface{}) error {
	snapshotName := fmt.Sprintf("%s-%s", getNameOfCaller(), snapshotID)
	return defaultConfig().snapshot(snapshotName, i...)
}

// SnapshotT calls Snapshotter.SnapshotT with the default config.
func SnapshotT(t *testing.T, i ...interface{}) {
	t.Helper()
	snapshotName := strings.Replace(t.Name(), "/", "-", -1)
	err := defaultConfig().snapshot(snapshotName, i...)
	if err != nil {
		t.Error(err)
	}
}

func (c *config) Snapshot(i ...interface{}) error {
	return c.snapshot(getNameOfCaller(), i...)
}

func (c *config) SnapshotMulti(snapshotID string, i ...interface{}) error {
	snapshotName := fmt.Sprintf("%s-%s", getNameOfCaller(), snapshotID)
	return c.snapshot(snapshotName, i...)
}

func (c *config) SnapshotT(t *testing.T, i ...interface{}) {
	t.Helper()
	snapshotName := strings.Replace(t.Name(), "/", "-", -1)
	err := c.snapshot(snapshotName, i...)
	if err != nil {
		t.Error(err)
	}
}

func (c *config) snapshot(snapshotName string, i ...interface{}) error {
	snapshot := takeSnapshot(i...)

	prevSnapshot, err := c.readSnapshot(snapshotName)
	if os.IsNotExist(err) {
		return c.updateSnapshot(snapshotName, snapshot)
	}
	if err != nil {
		return err
	}

	if snapshot == prevSnapshot || takeV1Snapshot(i...) == prevSnapshot {
		// previous snapshot matches current value
		return nil
	}

	if c.shouldUpdate() {
		// updates snapshot to current value and upgrades snapshot format
		return c.updateSnapshot(snapshotName, snapshot)
	}

	diff := diffSnapshots(prevSnapshot, snapshot)
	return fmt.Errorf("snapshot not equal:\n%s", diff)
}
<commit_msg>Make SnapshotT() a no-op for (already) failed tests (#33)<commit_after>package cupaloy

import (
	"fmt"
	"os"
	"strings"
	"testing"
)

// Snapshotter is the API for taking snapshots of values in your tests.
type Snapshotter interface {
	// Snapshot compares the given value to the it's previous value stored on the filesystem.
	// An error containing a diff is returned if the snapshots do not match.
	// Snapshot determines the snapshot file automatically from the name of the calling function.
	Snapshot(i ...interface{}) error

	// SnapshotMulti is identical to Snapshot but can be called multiple times from the same function.
	// This is done by providing a unique snapshotId for each invocation.
	SnapshotMulti(snapshotID string, i ...interface{}) error

	// SnapshotT is identical to Snapshot but gets the snapshot name using
	// t.Name() and calls t.Fail() directly if the snapshots do not match.
	SnapshotT(t *testing.T, i ...interface{})
}

// New constructs a new, configured instance of cupaloy using the given Configurators.
func New(configurators ...Configurator) Snapshotter {
	config := defaultConfig()

	for _, configurator := range configurators {
		configurator(config)
	}

	return config
}

// Snapshot calls Snapshotter.Snapshot with the default config.
func Snapshot(i ...interface{}) error {
	return defaultConfig().snapshot(getNameOfCaller(), i...)
}

// SnapshotMulti calls Snapshotter.SnapshotMulti with the default config.
func SnapshotMulti(snapshotID string, i ...interface{}) error {
	snapshotName := fmt.Sprintf("%s-%s", getNameOfCaller(), snapshotID)
	return defaultConfig().snapshot(snapshotName, i...)
}

// SnapshotT calls Snapshotter.SnapshotT with the default config.
func SnapshotT(t *testing.T, i ...interface{}) {
	t.Helper()
	defaultConfig().SnapshotT(t, i...)
}

func (c *config) Snapshot(i ...interface{}) error {
	return c.snapshot(getNameOfCaller(), i...)
}

func (c *config) SnapshotMulti(snapshotID string, i ...interface{}) error {
	snapshotName := fmt.Sprintf("%s-%s", getNameOfCaller(), snapshotID)
	return c.snapshot(snapshotName, i...)
}

func (c *config) SnapshotT(t *testing.T, i ...interface{}) {
	t.Helper()
	if t.Failed() {
		return
	}

	snapshotName := strings.Replace(t.Name(), "/", "-", -1)
	err := c.snapshot(snapshotName, i...)
	if err != nil {
		t.Error(err)
	}
}

func (c *config) snapshot(snapshotName string, i ...interface{}) error {
	snapshot := takeSnapshot(i...)

	prevSnapshot, err := c.readSnapshot(snapshotName)
	if os.IsNotExist(err) {
		return c.updateSnapshot(snapshotName, snapshot)
	}
	if err != nil {
		return err
	}

	if snapshot == prevSnapshot || takeV1Snapshot(i...) == prevSnapshot {
		// previous snapshot matches current value
		return nil
	}

	if c.shouldUpdate() {
		// updates snapshot to current value and upgrades snapshot format
		return c.updateSnapshot(snapshotName, snapshot)
	}

	diff := diffSnapshots(prevSnapshot, snapshot)
	return fmt.Errorf("snapshot not equal:\n%s", diff)
}
<|endoftext|>


<commit_before>package learn

import (
	"fmt"
	"github.com/tsinghua-io/api-server/adapter"
	"net/http"
	"net/url"
)

const (
	BaseURL = "http://learn.cic.tsinghua.edu.cn"
	AuthURL = "https://id.tsinghua.edu.cn/do/off/ui/auth/login/post/fa8077873a7a80b1cd6b185d5a796617/0?/j_spring_security_thauth_roaming_entry"
)

type Adapter struct{ adapter.Adapter }

func New(userId, password string) (ada *Adapter, status int, errMsg error) {
	ada = new(Adapter)
	ada.WithJar()
	status = http.StatusOK

	form := url.Values{}
	form.Add("i_user", userId)
	form.Add("i_pass", password)

	resp, err := ada.PostForm(AuthURL, form)
	if err != nil {
		return nil, http.StatusBadGateway, fmt.Errorf("Failed to post login form to %s: %s", AuthURL, err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, http.StatusUnauthorized, fmt.Errorf("Failed to login to %s: %s", AuthURL, http.StatusText(resp.StatusCode))
	}

	return
}
<commit_msg>Fix login fail in cic.<commit_after>package learn

import (
	"fmt"
	"github.com/tsinghua-io/api-server/adapter"
	"net/http"
	"net/url"
)

const (
	BaseURL = "http://learn.cic.tsinghua.edu.cn"
	AuthURL = "https://id.tsinghua.edu.cn/do/off/ui/auth/login/post/fa8077873a7a80b1cd6b185d5a796617/0?/j_spring_security_thauth_roaming_entry"
)

type Adapter struct{ adapter.Adapter }

func New(userId, password string) (ada *Adapter, status int, errMsg error) {
	ada = new(Adapter)
	ada.WithJar()
	status = http.StatusOK

	form := url.Values{}
	form.Add("i_user", userId)
	form.Add("i_pass", password)

	resp, err := ada.PostForm(AuthURL, form)
	if err != nil {
		return nil, http.StatusBadGateway, fmt.Errorf("Failed to post login form to %s: %s", AuthURL, err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK || resp.Request.URL.Host != "learn.cic.tsinghua.edu.cn" {
		return nil, http.StatusUnauthorized, fmt.Errorf("Failed to login to %s: %s", AuthURL, http.StatusText(resp.StatusCode))
	}

	return
}
<|endoftext|>


<commit_before>package imagebuildah

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/containers/buildah"
	"github.com/containers/common/pkg/config"
	"github.com/containers/image/v5/docker/reference"
	"github.com/containers/image/v5/types"
	encconfig "github.com/containers/ocicrypt/config"
	"github.com/containers/storage"
	"github.com/containers/storage/pkg/archive"
	specs "github.com/opencontainers/runtime-spec/specs-go"
	"github.com/openshift/imagebuilder"
	"github.com/openshift/imagebuilder/dockerfile/parser"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const (
	PullIfMissing = buildah.PullIfMissing
	PullAlways    = buildah.PullAlways
	PullIfNewer   = buildah.PullIfNewer
	PullNever     = buildah.PullNever

	Gzip         = archive.Gzip
	Bzip2        = archive.Bzip2
	Xz           = archive.Xz
	Zstd         = archive.Zstd
	Uncompressed = archive.Uncompressed
)

// Mount is a mountpoint for the build container.
type Mount specs.Mount

// BuildOptions can be used to alter how an image is built.
type BuildOptions struct {
	// ContextDirectory is the default source location for COPY and ADD
	// commands.
	ContextDirectory string
	// PullPolicy controls whether or not we pull images.  It should be one
	// of PullIfMissing, PullAlways, PullIfNewer, or PullNever.
	PullPolicy buildah.PullPolicy
	// Registry is a value which is prepended to the image's name, if it
	// needs to be pulled and the image name alone can not be resolved to a
	// reference to a source image.  No separator is implicitly added.
	Registry string
	// IgnoreUnrecognizedInstructions tells us to just log instructions we
	// don't recognize, and try to keep going.
	IgnoreUnrecognizedInstructions bool
	// Quiet tells us whether or not to announce steps as we go through them.
	Quiet bool
	// Isolation controls how Run() runs things.
	Isolation buildah.Isolation
	// Runtime is the name of the command to run for RUN instructions when
	// Isolation is either IsolationDefault or IsolationOCI.  It should
	// accept the same arguments and flags that runc does.
	Runtime string
	// RuntimeArgs adds global arguments for the runtime.
	RuntimeArgs []string
	// TransientMounts is a list of mounts that won't be kept in the image.
	TransientMounts []string
	// Compression specifies the type of compression which is applied to
	// layer blobs.  The default is to not use compression, but
	// archive.Gzip is recommended.
	Compression archive.Compression
	// Arguments which can be interpolated into Dockerfiles
	Args map[string]string
	// Name of the image to write to.
	Output string
	// Additional tags to add to the image that we write, if we know of a
	// way to add them.
	AdditionalTags []string
	// Log is a callback that will print a progress message.  If no value
	// is supplied, the message will be sent to Err (or os.Stderr, if Err
	// is nil) by default.
	Log func(format string, args ...interface{})
	// In is connected to stdin for RUN instructions.
	In io.Reader
	// Out is a place where non-error log messages are sent.
	Out io.Writer
	// Err is a place where error log messages should be sent.
	Err io.Writer
	// SignaturePolicyPath specifies an override location for the signature
	// policy which should be used for verifying the new image as it is
	// being written.  Except in specific circumstances, no value should be
	// specified, indicating that the shared, system-wide default policy
	// should be used.
	SignaturePolicyPath string
	// ReportWriter is an io.Writer which will be used to report the
	// progress of the (possible) pulling of the source image and the
	// writing of the new image.
	ReportWriter io.Writer
	// OutputFormat is the format of the output image's manifest and
	// configuration data.
	// Accepted values are buildah.OCIv1ImageManifest and buildah.Dockerv2ImageManifest.
	OutputFormat string
	// SystemContext holds parameters used for authentication.
	SystemContext *types.SystemContext
	// NamespaceOptions controls how we set up namespaces processes that we
	// might need when handling RUN instructions.
	NamespaceOptions []buildah.NamespaceOption
	// ConfigureNetwork controls whether or not network interfaces and
	// routing are configured for a new network namespace (i.e., when not
	// joining another's namespace and not just using the host's
	// namespace), effectively deciding whether or not the process has a
	// usable network.
	ConfigureNetwork buildah.NetworkConfigurationPolicy
	// CNIPluginPath is the location of CNI plugin helpers, if they should be
	// run from a location other than the default location.
	CNIPluginPath string
	// CNIConfigDir is the location of CNI configuration files, if the files in
	// the default configuration directory shouldn't be used.
	CNIConfigDir string
	// ID mapping options to use if we're setting up our own user namespace
	// when handling RUN instructions.
	IDMappingOptions *buildah.IDMappingOptions
	// AddCapabilities is a list of capabilities to add to the default set when
	// handling RUN instructions.
	AddCapabilities []string
	// DropCapabilities is a list of capabilities to remove from the default set
	// when handling RUN instructions. If a capability appears in both lists, it
	// will be dropped.
	DropCapabilities []string
	// CommonBuildOpts is *required*.
	CommonBuildOpts *buildah.CommonBuildOptions
	// DefaultMountsFilePath is the file path holding the mounts to be mounted in "host-path:container-path" format
	DefaultMountsFilePath string
	// IIDFile tells the builder to write the image ID to the specified file
	IIDFile string
	// Squash tells the builder to produce an image with a single layer
	// instead of with possibly more than one layer.
	Squash bool
	// Labels metadata for an image
	Labels []string
	// Annotation metadata for an image
	Annotations []string
	// OnBuild commands to be run by images based on this image
	OnBuild []string
	// Layers tells the builder to create a cache of images for each step in the Dockerfile
	Layers bool
	// NoCache tells the builder to build the image from scratch without checking for a cache.
	// It creates a new set of cached images for the build.
	NoCache bool
	// RemoveIntermediateCtrs tells the builder whether to remove intermediate containers used
	// during the build process. Default is true.
	RemoveIntermediateCtrs bool
	// ForceRmIntermediateCtrs tells the builder to remove all intermediate containers even if
	// the build was unsuccessful.
	ForceRmIntermediateCtrs bool
	// BlobDirectory is a directory which we'll use for caching layer blobs.
	BlobDirectory string
	// Target the targeted FROM in the Dockerfile to build.
	Target string
	// Devices are the additional devices to add to the containers.
	Devices []string
	// SignBy is the fingerprint of a GPG key to use for signing images.
	SignBy string
	// Architecture specifies the target architecture of the image to be built.
	Architecture string
	// Timestamp sets the created timestamp to the specified time, allowing
	// for deterministic, content-addressable builds.
	Timestamp *time.Time
	// OS is the specifies the operating system of the image to be built.
	OS string
	// MaxPullPushRetries is the maximum number of attempts we'll make to pull or push any one
	// image from or to an external registry if the first attempt fails.
	MaxPullPushRetries int
	// PullPushRetryDelay is how long to wait before retrying a pull or push attempt.
	PullPushRetryDelay time.Duration
	// OciDecryptConfig contains the config that can be used to decrypt an image if it is
	// encrypted if non-nil. If nil, it does not attempt to decrypt an image.
	OciDecryptConfig *encconfig.DecryptConfig
	// Jobs is the number of stages to run in parallel.  If not specified it defaults to 1.
	Jobs *int
	// LogRusage logs resource usage for each step.
	LogRusage bool
}

// BuildDockerfiles parses a set of one or more Dockerfiles (which may be
// URLs), creates a new Executor, and then runs Prepare/Execute/Commit/Delete
// over the entire set of instructions.
func BuildDockerfiles(ctx context.Context, store storage.Store, options BuildOptions, paths ...string) (string, reference.Canonical, error) {
	if len(paths) == 0 {
		return "", nil, errors.Errorf("error building: no dockerfiles specified")
	}
	var dockerfiles []io.ReadCloser
	defer func(dockerfiles ...io.ReadCloser) {
		for _, d := range dockerfiles {
			d.Close()
		}
	}(dockerfiles...)

	for _, dfile := range paths {
		var data io.ReadCloser

		if strings.HasPrefix(dfile, "http://") || strings.HasPrefix(dfile, "https://") {
			logrus.Debugf("reading remote Dockerfile %q", dfile)
			resp, err := http.Get(dfile)
			if err != nil {
				return "", nil, err
			}
			if resp.ContentLength == 0 {
				resp.Body.Close()
				return "", nil, errors.Errorf("no contents in %q", dfile)
			}
			data = resp.Body
		} else {
			// If the Dockerfile isn't found try prepending the
			// context directory to it.
			dinfo, err := os.Stat(dfile)
			if os.IsNotExist(err) {
				// If they are "/workDir/Dockerfile" and "/workDir"
				// so don't joint it
				if !strings.HasPrefix(dfile, options.ContextDirectory) {
					dfile = filepath.Join(options.ContextDirectory, dfile)
				}
				dinfo, err = os.Stat(dfile)
				if err != nil {
					return "", nil, err
				}
			}
			// If given a directory, add '/Dockerfile' to it.
			if dinfo.Mode().IsDir() {
				dfile = filepath.Join(dfile, "Dockerfile")
			}
			logrus.Debugf("reading local Dockerfile %q", dfile)
			contents, err := os.Open(dfile)
			if err != nil {
				return "", nil, err
			}
			dinfo, err = contents.Stat()
			if err != nil {
				contents.Close()
				return "", nil, errors.Wrapf(err, "error reading info about %q", dfile)
			}
			if dinfo.Mode().IsRegular() && dinfo.Size() == 0 {
				contents.Close()
				return "", nil, errors.Errorf("no contents in %q", dfile)
			}
			data = contents
		}

		// pre-process Dockerfiles with ".in" suffix
		if strings.HasSuffix(dfile, ".in") {
			pData, err := preprocessDockerfileContents(data, options.ContextDirectory)
			if err != nil {
				return "", nil, err
			}
			data = *pData
		}

		dockerfiles = append(dockerfiles, data)
	}

	mainNode, err := imagebuilder.ParseDockerfile(dockerfiles[0])
	if err != nil {
		return "", nil, errors.Wrapf(err, "error parsing main Dockerfile: %s", dockerfiles[0])
	}

	warnOnUnsetBuildArgs(mainNode, options.Args)

	for _, d := range dockerfiles[1:] {
		additionalNode, err := imagebuilder.ParseDockerfile(d)
		if err != nil {
			return "", nil, errors.Wrapf(err, "error parsing additional Dockerfile %s", d)
		}
		mainNode.Children = append(mainNode.Children, additionalNode.Children...)
	}
	exec, err := NewExecutor(store, options, mainNode)
	if err != nil {
		return "", nil, errors.Wrapf(err, "error creating build executor")
	}
	b := imagebuilder.NewBuilder(options.Args)
	defaultContainerConfig, err := config.Default()
	if err != nil {
		return "", nil, errors.Wrapf(err, "failed to get container config")
	}
	b.Env = append(defaultContainerConfig.GetDefaultEnv(), b.Env...)
	stages, err := imagebuilder.NewStages(mainNode, b)
	if err != nil {
		return "", nil, errors.Wrap(err, "error reading multiple stages")
	}
	if options.Target != "" {
		stagesTargeted, ok := stages.ThroughTarget(options.Target)
		if !ok {
			return "", nil, errors.Errorf("The target %q was not found in the provided Dockerfile", options.Target)
		}
		stages = stagesTargeted
	}
	return exec.Build(ctx, stages)
}

func warnOnUnsetBuildArgs(node *parser.Node, args map[string]string) {
	for _, child := range node.Children {
		switch strings.ToUpper(child.Value) {
		case "ARG":
			argName := child.Next.Value
			if _, ok := args[argName]; !strings.Contains(argName, "=") && !ok {
				logrus.Warnf("missing %q build argument. Try adding %q to the command line", argName, fmt.Sprintf("--build-arg %s=<VALUE>", argName))
			}
		default:
			continue
		}
	}
}

// preprocessDockerfileContents runs CPP(1) in preprocess-only mode on the input
// dockerfile content and will use ctxDir as the base include path.
//
// Note: we cannot use cmd.StdoutPipe() as cmd.Wait() closes it.
func preprocessDockerfileContents(r io.Reader, ctxDir string) (rdrCloser *io.ReadCloser, err error) {
	cppPath := "/usr/bin/cpp"
	if _, err = os.Stat(cppPath); err != nil {
		if os.IsNotExist(err) {
			err = errors.Errorf("error: Dockerfile.in support requires %s to be installed", cppPath)
		}
		return nil, err
	}

	stdout := bytes.Buffer{}
	stderr := bytes.Buffer{}

	cmd := exec.Command(cppPath, "-E", "-iquote", ctxDir, "-traditional", "-undef", "-")
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr

	pipe, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}

	defer func() {
		if err != nil {
			pipe.Close()
		}
	}()

	if err = cmd.Start(); err != nil {
		return nil, err
	}

	if _, err = io.Copy(pipe, r); err != nil {
		return nil, err
	}

	pipe.Close()
	if err = cmd.Wait(); err != nil {
		if stderr.Len() > 0 {
			err = errors.Wrapf(err, "%v", strings.TrimSpace(stderr.String()))
		}
		return nil, errors.Wrapf(err, "error pre-processing Dockerfile")
	}

	rc := ioutil.NopCloser(bytes.NewReader(stdout.Bytes()))
	return &rc, nil
}
<commit_msg>Fix NPE when Dockerfile path contains non-directory entries<commit_after>package imagebuildah

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/containers/buildah"
	"github.com/containers/common/pkg/config"
	"github.com/containers/image/v5/docker/reference"
	"github.com/containers/image/v5/types"
	encconfig "github.com/containers/ocicrypt/config"
	"github.com/containers/storage"
	"github.com/containers/storage/pkg/archive"
	specs "github.com/opencontainers/runtime-spec/specs-go"
	"github.com/openshift/imagebuilder"
	"github.com/openshift/imagebuilder/dockerfile/parser"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const (
	PullIfMissing = buildah.PullIfMissing
	PullAlways    = buildah.PullAlways
	PullIfNewer   = buildah.PullIfNewer
	PullNever     = buildah.PullNever

	Gzip         = archive.Gzip
	Bzip2        = archive.Bzip2
	Xz           = archive.Xz
	Zstd         = archive.Zstd
	Uncompressed = archive.Uncompressed
)

// Mount is a mountpoint for the build container.
type Mount specs.Mount

// BuildOptions can be used to alter how an image is built.
type BuildOptions struct {
	// ContextDirectory is the default source location for COPY and ADD
	// commands.
	ContextDirectory string
	// PullPolicy controls whether or not we pull images.  It should be one
	// of PullIfMissing, PullAlways, PullIfNewer, or PullNever.
	PullPolicy buildah.PullPolicy
	// Registry is a value which is prepended to the image's name, if it
	// needs to be pulled and the image name alone can not be resolved to a
	// reference to a source image.  No separator is implicitly added.
	Registry string
	// IgnoreUnrecognizedInstructions tells us to just log instructions we
	// don't recognize, and try to keep going.
	IgnoreUnrecognizedInstructions bool
	// Quiet tells us whether or not to announce steps as we go through them.
	Quiet bool
	// Isolation controls how Run() runs things.
	Isolation buildah.Isolation
	// Runtime is the name of the command to run for RUN instructions when
	// Isolation is either IsolationDefault or IsolationOCI.  It should
	// accept the same arguments and flags that runc does.
	Runtime string
	// RuntimeArgs adds global arguments for the runtime.
	RuntimeArgs []string
	// TransientMounts is a list of mounts that won't be kept in the image.
	TransientMounts []string
	// Compression specifies the type of compression which is applied to
	// layer blobs.  The default is to not use compression, but
	// archive.Gzip is recommended.
	Compression archive.Compression
	// Arguments which can be interpolated into Dockerfiles
	Args map[string]string
	// Name of the image to write to.
	Output string
	// Additional tags to add to the image that we write, if we know of a
	// way to add them.
	AdditionalTags []string
	// Log is a callback that will print a progress message.  If no value
	// is supplied, the message will be sent to Err (or os.Stderr, if Err
	// is nil) by default.
	Log func(format string, args ...interface{})
	// In is connected to stdin for RUN instructions.
	In io.Reader
	// Out is a place where non-error log messages are sent.
	Out io.Writer
	// Err is a place where error log messages should be sent.
	Err io.Writer
	// SignaturePolicyPath specifies an override location for the signature
	// policy which should be used for verifying the new image as it is
	// being written.  Except in specific circumstances, no value should be
	// specified, indicating that the shared, system-wide default policy
	// should be used.
	SignaturePolicyPath string
	// ReportWriter is an io.Writer which will be used to report the
	// progress of the (possible) pulling of the source image and the
	// writing of the new image.
	ReportWriter io.Writer
	// OutputFormat is the format of the output image's manifest and
	// configuration data.
	// Accepted values are buildah.OCIv1ImageManifest and buildah.Dockerv2ImageManifest.
	OutputFormat string
	// SystemContext holds parameters used for authentication.
	SystemContext *types.SystemContext
	// NamespaceOptions controls how we set up namespaces processes that we
	// might need when handling RUN instructions.
	NamespaceOptions []buildah.NamespaceOption
	// ConfigureNetwork controls whether or not network interfaces and
	// routing are configured for a new network namespace (i.e., when not
	// joining another's namespace and not just using the host's
	// namespace), effectively deciding whether or not the process has a
	// usable network.
	ConfigureNetwork buildah.NetworkConfigurationPolicy
	// CNIPluginPath is the location of CNI plugin helpers, if they should be
	// run from a location other than the default location.
	CNIPluginPath string
	// CNIConfigDir is the location of CNI configuration files, if the files in
	// the default configuration directory shouldn't be used.
	CNIConfigDir string
	// ID mapping options to use if we're setting up our own user namespace
	// when handling RUN instructions.
	IDMappingOptions *buildah.IDMappingOptions
	// AddCapabilities is a list of capabilities to add to the default set when
	// handling RUN instructions.
	AddCapabilities []string
	// DropCapabilities is a list of capabilities to remove from the default set
	// when handling RUN instructions. If a capability appears in both lists, it
	// will be dropped.
	DropCapabilities []string
	// CommonBuildOpts is *required*.
	CommonBuildOpts *buildah.CommonBuildOptions
	// DefaultMountsFilePath is the file path holding the mounts to be mounted in "host-path:container-path" format
	DefaultMountsFilePath string
	// IIDFile tells the builder to write the image ID to the specified file
	IIDFile string
	// Squash tells the builder to produce an image with a single layer
	// instead of with possibly more than one layer.
	Squash bool
	// Labels metadata for an image
	Labels []string
	// Annotation metadata for an image
	Annotations []string
	// OnBuild commands to be run by images based on this image
	OnBuild []string
	// Layers tells the builder to create a cache of images for each step in the Dockerfile
	Layers bool
	// NoCache tells the builder to build the image from scratch without checking for a cache.
	// It creates a new set of cached images for the build.
	NoCache bool
	// RemoveIntermediateCtrs tells the builder whether to remove intermediate containers used
	// during the build process. Default is true.
	RemoveIntermediateCtrs bool
	// ForceRmIntermediateCtrs tells the builder to remove all intermediate containers even if
	// the build was unsuccessful.
	ForceRmIntermediateCtrs bool
	// BlobDirectory is a directory which we'll use for caching layer blobs.
	BlobDirectory string
	// Target the targeted FROM in the Dockerfile to build.
	Target string
	// Devices are the additional devices to add to the containers.
	Devices []string
	// SignBy is the fingerprint of a GPG key to use for signing images.
	SignBy string
	// Architecture specifies the target architecture of the image to be built.
	Architecture string
	// Timestamp sets the created timestamp to the specified time, allowing
	// for deterministic, content-addressable builds.
	Timestamp *time.Time
	// OS is the specifies the operating system of the image to be built.
	OS string
	// MaxPullPushRetries is the maximum number of attempts we'll make to pull or push any one
	// image from or to an external registry if the first attempt fails.
	MaxPullPushRetries int
	// PullPushRetryDelay is how long to wait before retrying a pull or push attempt.
	PullPushRetryDelay time.Duration
	// OciDecryptConfig contains the config that can be used to decrypt an image if it is
	// encrypted if non-nil. If nil, it does not attempt to decrypt an image.
	OciDecryptConfig *encconfig.DecryptConfig
	// Jobs is the number of stages to run in parallel.  If not specified it defaults to 1.
	Jobs *int
	// LogRusage logs resource usage for each step.
	LogRusage bool
}

// BuildDockerfiles parses a set of one or more Dockerfiles (which may be
// URLs), creates a new Executor, and then runs Prepare/Execute/Commit/Delete
// over the entire set of instructions.
func BuildDockerfiles(ctx context.Context, store storage.Store, options BuildOptions, paths ...string) (string, reference.Canonical, error) {
	if len(paths) == 0 {
		return "", nil, errors.Errorf("error building: no dockerfiles specified")
	}
	var dockerfiles []io.ReadCloser
	defer func(dockerfiles ...io.ReadCloser) {
		for _, d := range dockerfiles {
			d.Close()
		}
	}(dockerfiles...)

	for _, dfile := range paths {
		var data io.ReadCloser

		if strings.HasPrefix(dfile, "http://") || strings.HasPrefix(dfile, "https://") {
			logrus.Debugf("reading remote Dockerfile %q", dfile)
			resp, err := http.Get(dfile)
			if err != nil {
				return "", nil, err
			}
			if resp.ContentLength == 0 {
				resp.Body.Close()
				return "", nil, errors.Errorf("no contents in %q", dfile)
			}
			data = resp.Body
		} else {
			dinfo, err := os.Stat(dfile)
			if err != nil {
				// If the Dockerfile isn't available, try again with
				// context directory prepended (if not prepended yet).
				if !strings.HasPrefix(dfile, options.ContextDirectory) {
					dfile = filepath.Join(options.ContextDirectory, dfile)
					dinfo, err = os.Stat(dfile)
				}
			}
			if err != nil {
				return "", nil, err
			}

			// If given a directory, add '/Dockerfile' to it.
			if dinfo.Mode().IsDir() {
				dfile = filepath.Join(dfile, "Dockerfile")
			}
			logrus.Debugf("reading local Dockerfile %q", dfile)
			contents, err := os.Open(dfile)
			if err != nil {
				return "", nil, err
			}
			dinfo, err = contents.Stat()
			if err != nil {
				contents.Close()
				return "", nil, errors.Wrapf(err, "error reading info about %q", dfile)
			}
			if dinfo.Mode().IsRegular() && dinfo.Size() == 0 {
				contents.Close()
				return "", nil, errors.Errorf("no contents in %q", dfile)
			}
			data = contents
		}

		// pre-process Dockerfiles with ".in" suffix
		if strings.HasSuffix(dfile, ".in") {
			pData, err := preprocessDockerfileContents(data, options.ContextDirectory)
			if err != nil {
				return "", nil, err
			}
			data = *pData
		}

		dockerfiles = append(dockerfiles, data)
	}

	mainNode, err := imagebuilder.ParseDockerfile(dockerfiles[0])
	if err != nil {
		return "", nil, errors.Wrapf(err, "error parsing main Dockerfile: %s", dockerfiles[0])
	}

	warnOnUnsetBuildArgs(mainNode, options.Args)

	for _, d := range dockerfiles[1:] {
		additionalNode, err := imagebuilder.ParseDockerfile(d)
		if err != nil {
			return "", nil, errors.Wrapf(err, "error parsing additional Dockerfile %s", d)
		}
		mainNode.Children = append(mainNode.Children, additionalNode.Children...)
	}
	exec, err := NewExecutor(store, options, mainNode)
	if err != nil {
		return "", nil, errors.Wrapf(err, "error creating build executor")
	}
	b := imagebuilder.NewBuilder(options.Args)
	defaultContainerConfig, err := config.Default()
	if err != nil {
		return "", nil, errors.Wrapf(err, "failed to get container config")
	}
	b.Env = append(defaultContainerConfig.GetDefaultEnv(), b.Env...)
	stages, err := imagebuilder.NewStages(mainNode, b)
	if err != nil {
		return "", nil, errors.Wrap(err, "error reading multiple stages")
	}
	if options.Target != "" {
		stagesTargeted, ok := stages.ThroughTarget(options.Target)
		if !ok {
			return "", nil, errors.Errorf("The target %q was not found in the provided Dockerfile", options.Target)
		}
		stages = stagesTargeted
	}
	return exec.Build(ctx, stages)
}

func warnOnUnsetBuildArgs(node *parser.Node, args map[string]string) {
	for _, child := range node.Children {
		switch strings.ToUpper(child.Value) {
		case "ARG":
			argName := child.Next.Value
			if _, ok := args[argName]; !strings.Contains(argName, "=") && !ok {
				logrus.Warnf("missing %q build argument. Try adding %q to the command line", argName, fmt.Sprintf("--build-arg %s=<VALUE>", argName))
			}
		default:
			continue
		}
	}
}

// preprocessDockerfileContents runs CPP(1) in preprocess-only mode on the input
// dockerfile content and will use ctxDir as the base include path.
//
// Note: we cannot use cmd.StdoutPipe() as cmd.Wait() closes it.
func preprocessDockerfileContents(r io.Reader, ctxDir string) (rdrCloser *io.ReadCloser, err error) {
	cppPath := "/usr/bin/cpp"
	if _, err = os.Stat(cppPath); err != nil {
		if os.IsNotExist(err) {
			err = errors.Errorf("error: Dockerfile.in support requires %s to be installed", cppPath)
		}
		return nil, err
	}

	stdout := bytes.Buffer{}
	stderr := bytes.Buffer{}

	cmd := exec.Command(cppPath, "-E", "-iquote", ctxDir, "-traditional", "-undef", "-")
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr

	pipe, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}

	defer func() {
		if err != nil {
			pipe.Close()
		}
	}()

	if err = cmd.Start(); err != nil {
		return nil, err
	}

	if _, err = io.Copy(pipe, r); err != nil {
		return nil, err
	}

	pipe.Close()
	if err = cmd.Wait(); err != nil {
		if stderr.Len() > 0 {
			err = errors.Wrapf(err, "%v", strings.TrimSpace(stderr.String()))
		}
		return nil, errors.Wrapf(err, "error pre-processing Dockerfile")
	}

	rc := ioutil.NopCloser(bytes.NewReader(stdout.Bytes()))
	return &rc, nil
}
<|endoftext|>


<commit_before>// Package authtest provides testing helper functions for the
// auth package
package authtest

import (
	"context"
	"testing"

	"github.com/gilcrest/go-api-basic/domain/user"
)

// NewMockAuthorizer is an initializer for MockAuthorizer
func NewMockAuthorizer(t *testing.T) MockAuthorizer {
	return MockAuthorizer{t: t}
}

// MockAuthorizer mocks authorizing access for
// a user to a given object to perform a given action.
type MockAuthorizer struct {
	t *testing.T
}

// Authorize mocks authorizing access for
// a user to a given object to perform a given action. Authorize
// never returns an error, thus everything sent in is always authorized
func (ma MockAuthorizer) Authorize(ctx context.Context, sub user.User, obj string, act string) error {
	ma.t.Helper()

	return nil
}
<commit_msg>no longer necessary<commit_after><|endoftext|>


<commit_before>// Package bootstrap implements some helper functions intended
// to be used with the Bootstrap front-end framework.
// See http://twitter.github.io/bootstrap/ for more details.
//
// This package defines the "bootstrap" asset, which serves bootstrap
// from http://www.bootstrapcdn.com. It receives a single argument with the
// desired bootstrap version. Only version 3 is supported. e.g.
//
//  bootstrap: 3.0.0
//
// This asset also supports the following options:
//
//  nojs (bool): disables loading bootstrap's javascript library
//  e.g. bootstrap|nojs: 2.3.2
//
// See gnd.la/template and gnd.la/template/assets for more information
// about template functions and the assets pipeline.
package bootstrap
<commit_msg>Note that gnd.la/frontend/bootstrap will set the default gnd.la/form renderer<commit_after>// Package bootstrap implements some helper functions intended
// to be used with the Bootstrap front-end framework.
// See http://twitter.github.io/bootstrap/ for more details.
//
// This package defines the "bootstrap" asset, which serves bootstrap
// from http://www.bootstrapcdn.com. It receives a single argument with the
// desired bootstrap version. Only version 3 is supported. e.g.
//
//  bootstrap: 3.0.0
//
// This asset also supports the following options:
//
//  nojs (bool): disables loading bootstrap's javascript library
//  e.g. bootstrap|nojs: 2.3.2
//
// See gnd.la/template and gnd.la/template/assets for more information
// about template functions and the assets pipeline.
//
// Importing this package will also register FormRenderer as the default
// form renderer.
package bootstrap
<|endoftext|>


<commit_before>package gate

import (
	"github.com/name5566/leaf/chanrpc"
	"github.com/name5566/leaf/log"
	"github.com/name5566/leaf/network"
	"reflect"
	"time"
)

type Gate struct {
	MaxConnNum      int
	PendingWriteNum int
	MaxMsgLen       uint32
	Processor       network.Processor
	AgentChanRPC    *chanrpc.Server

	// websocket
	WSAddr      string
	HTTPTimeout time.Duration

	// tcp
	TCPAddr      string
	LenMsgLen    int
	LittleEndian bool
}

func (gate *Gate) Run(closeSig chan bool) {
	var wsServer *network.WSServer
	if gate.WSAddr != "" {
		wsServer = new(network.WSServer)
		wsServer.Addr = gate.WSAddr
		wsServer.MaxConnNum = gate.MaxConnNum
		wsServer.PendingWriteNum = gate.PendingWriteNum
		wsServer.MaxMsgLen = gate.MaxMsgLen
		wsServer.HTTPTimeout = gate.HTTPTimeout
		wsServer.NewAgent = func(conn *network.WSConn) network.Agent {
			a := &agent{conn: conn, gate: gate}
			if gate.AgentChanRPC != nil {
				gate.AgentChanRPC.Go("NewAgent", a)
			}
			return a
		}
	}

	var tcpServer *network.TCPServer
	if gate.TCPAddr != "" {
		tcpServer = new(network.TCPServer)
		tcpServer.Addr = gate.TCPAddr
		tcpServer.MaxConnNum = gate.MaxConnNum
		tcpServer.PendingWriteNum = gate.PendingWriteNum
		tcpServer.LenMsgLen = gate.LenMsgLen
		tcpServer.MaxMsgLen = gate.MaxMsgLen
		tcpServer.LittleEndian = gate.LittleEndian
		tcpServer.NewAgent = func(conn *network.TCPConn) network.Agent {
			a := &agent{conn: conn, gate: gate}
			if gate.AgentChanRPC != nil {
				gate.AgentChanRPC.Go("NewAgent", a)
			}
			return a
		}
	}

	if wsServer != nil {
		wsServer.Start()
	}
	if tcpServer != nil {
		tcpServer.Start()
	}
	<-closeSig
	if wsServer != nil {
		wsServer.Close()
	}
	if tcpServer != nil {
		tcpServer.Close()
	}
}

func (gate *Gate) OnDestroy() {}

type agent struct {
	conn     network.Conn
	gate     *Gate
	userData interface{}
}

func (a *agent) Run() {
	for {
		data, err := a.conn.ReadMsg()
		if err != nil {
			log.Debug("read message: %v", err)
			break
		}

		if a.gate.Processor != nil {
			msg, err := a.gate.Processor.Unmarshal(data)
			if err != nil {
				log.Debug("unmarshal message error: %v", err)
				break
			}
			err = a.gate.Processor.Route(msg, a)
			if err != nil {
				log.Debug("route message error: %v", err)
				break
			}
		}
	}
}

func (a *agent) OnClose() {
	if a.gate.AgentChanRPC != nil {
		err := a.gate.AgentChanRPC.Open(0).Call0("CloseAgent", a)
		if err != nil {
			log.Error("chanrpc error: %v", err)
		}
	}
}

func (a *agent) WriteMsg(msg interface{}) {
	if a.gate.Processor != nil {
		data, err := a.gate.Processor.Marshal(msg)
		if err != nil {
			log.Error("marshal message %v error: %v", reflect.TypeOf(msg), err)
			return
		}
		a.conn.WriteMsg(data...)
	}
}

func (a *agent) Close() {
	a.conn.Close()
}

func (a *agent) Destroy() {
	a.conn.Destroy()
}

func (a *agent) UserData() interface{} {
	return a.userData
}

func (a *agent) SetUserData(data interface{}) {
	a.userData = data
}
<commit_msg>log write message error<commit_after>package gate

import (
	"github.com/name5566/leaf/chanrpc"
	"github.com/name5566/leaf/log"
	"github.com/name5566/leaf/network"
	"reflect"
	"time"
)

type Gate struct {
	MaxConnNum      int
	PendingWriteNum int
	MaxMsgLen       uint32
	Processor       network.Processor
	AgentChanRPC    *chanrpc.Server

	// websocket
	WSAddr      string
	HTTPTimeout time.Duration

	// tcp
	TCPAddr      string
	LenMsgLen    int
	LittleEndian bool
}

func (gate *Gate) Run(closeSig chan bool) {
	var wsServer *network.WSServer
	if gate.WSAddr != "" {
		wsServer = new(network.WSServer)
		wsServer.Addr = gate.WSAddr
		wsServer.MaxConnNum = gate.MaxConnNum
		wsServer.PendingWriteNum = gate.PendingWriteNum
		wsServer.MaxMsgLen = gate.MaxMsgLen
		wsServer.HTTPTimeout = gate.HTTPTimeout
		wsServer.NewAgent = func(conn *network.WSConn) network.Agent {
			a := &agent{conn: conn, gate: gate}
			if gate.AgentChanRPC != nil {
				gate.AgentChanRPC.Go("NewAgent", a)
			}
			return a
		}
	}

	var tcpServer *network.TCPServer
	if gate.TCPAddr != "" {
		tcpServer = new(network.TCPServer)
		tcpServer.Addr = gate.TCPAddr
		tcpServer.MaxConnNum = gate.MaxConnNum
		tcpServer.PendingWriteNum = gate.PendingWriteNum
		tcpServer.LenMsgLen = gate.LenMsgLen
		tcpServer.MaxMsgLen = gate.MaxMsgLen
		tcpServer.LittleEndian = gate.LittleEndian
		tcpServer.NewAgent = func(conn *network.TCPConn) network.Agent {
			a := &agent{conn: conn, gate: gate}
			if gate.AgentChanRPC != nil {
				gate.AgentChanRPC.Go("NewAgent", a)
			}
			return a
		}
	}

	if wsServer != nil {
		wsServer.Start()
	}
	if tcpServer != nil {
		tcpServer.Start()
	}
	<-closeSig
	if wsServer != nil {
		wsServer.Close()
	}
	if tcpServer != nil {
		tcpServer.Close()
	}
}

func (gate *Gate) OnDestroy() {}

type agent struct {
	conn     network.Conn
	gate     *Gate
	userData interface{}
}

func (a *agent) Run() {
	for {
		data, err := a.conn.ReadMsg()
		if err != nil {
			log.Debug("read message: %v", err)
			break
		}

		if a.gate.Processor != nil {
			msg, err := a.gate.Processor.Unmarshal(data)
			if err != nil {
				log.Debug("unmarshal message error: %v", err)
				break
			}
			err = a.gate.Processor.Route(msg, a)
			if err != nil {
				log.Debug("route message error: %v", err)
				break
			}
		}
	}
}

func (a *agent) OnClose() {
	if a.gate.AgentChanRPC != nil {
		err := a.gate.AgentChanRPC.Open(0).Call0("CloseAgent", a)
		if err != nil {
			log.Error("chanrpc error: %v", err)
		}
	}
}

func (a *agent) WriteMsg(msg interface{}) {
	if a.gate.Processor != nil {
		data, err := a.gate.Processor.Marshal(msg)
		if err != nil {
			log.Error("marshal message %v error: %v", reflect.TypeOf(msg), err)
			return
		}
		err = a.conn.WriteMsg(data...)
		if err != nil {
			log.Error("write message %v error: %v", reflect.TypeOf(msg), err)
		}
	}
}

func (a *agent) Close() {
	a.conn.Close()
}

func (a *agent) Destroy() {
	a.conn.Destroy()
}

func (a *agent) UserData() interface{} {
	return a.userData
}

func (a *agent) SetUserData(data interface{}) {
	a.userData = data
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"strconv"
)

var (
	// 用于获取帖子信息
	getPage = regexp.MustCompile(`
\n
\n\n(?:<[^>]+>\n)+	\n\n\[]*>([^<]+)\]\s*]*>([^<]+)\n(?:<[^\n]+>\n){0,}?	\n\n\n]*>([^<]+)\n(?:)?([^<]+)(){1,2}\n	\n(?:<[^\n]+>\n){7}`)

	// 用于获取帖子列表总页数
	getMaxPagesNum = regexp.MustCompile(`<a href="[^"]+" class="last">\.\.\. ([0-9]+)</a>`)
)

func main() {
	fid := "139"
	resp, err := http.Get("http://www.mcbbs.net/forum.php?mod=forumdisplay&fid=" + fid + "&orderby=dateline&page=1")
	if err != nil {
		printError("GetPage", err)
		os.Exit(1)
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		printError("ReadAll", err)
		os.Exit(1)
	}

	// 获取帖子列表总页数
	buf := getMaxPagesNum.FindStringSubmatch(string(body))
	maxPagesNum, err := strconv.Atoi(buf[1])
	if err != nil {
		printError("Atoi", err)
		os.Exit(1)
	}
	// 获取每一页的所有帖子
	for i := 0; i < maxPagesNum; i++ {
		pageList, err := getPagesList(fid, i+1)
		if err != nil {
			printError("getPagesList"+string(i), err)
			return
		}
		printInfo("List", *pageList)
	}
}

// 获取单页面的所有帖子
func getPagesList(fid string, pageNum int) (*[]pageInfo, error) {
	resp, err := http.Get("http://www.mcbbs.net/forum.php?mod=forumdisplay&fid=" + fid + "&orderby=dateline&page=" + strconv.Itoa(pageNum))
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	/*
		// getPage所匹配的信息
		// 获取到的参数已用“m[x]说明”标出

		[m[1]分类] m[3]标题

		 ...234
		New

		m[4]作者
		m[6]发帖时间2/span>

		111189

		谢普
		5 天前

	*/
	m := getPage.FindAllStringSubmatch(string(body), -1)
	var pageList = new([]pageInfo)

	for _, v := range m {
		date := v[5]
		// 如果发帖时间1为空，那么使用发帖时间2
		if date == "" {
			date = v[6]
		}
		pageInf := pageInfo{
			category: v[1],
			url:      v[2],
			title:    v[3],
			author:   v[4],
			date:     date,
		}
		*pageList = append(*pageList, pageInf)
	}

	return pageList, nil
}

func printInfo(s string, v ...interface{}) {
	log.Println("[INFO]", s, v)
}

func printError(s string, v ...interface{}) {
	log.Println("[ERROR]", s, v)
}

type pageInfo struct {
	category string
	url      string
	title    string
	author   string
	date     string
}
<commit_msg>add: 添加文章内容抓取<commit_after>package main

import (
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"strconv"
)

var (
	// 用于获取帖子信息
	getPage = regexp.MustCompile(`
\n
\n\n(?:<[^>]+>\n)+	\n\n\[]*>([^<]+)\]\s*]*>([^<]+)\n(?:<[^\n]+>\n){0,}?	\n\n\n]*>([^<]+)\n(?:)?([^<]+)(){1,2}\n	\n(?:<[^\n]+>\n){7}`)

	// 用于获取帖子列表总页数
	getMaxPagesNum = regexp.MustCompile(`<a href="[^"]+" class="last">\.\.\. ([0-9]+)</a>`)

	getPostBody = regexp.MustCompile(`<td class="t_f" id="postmessage_[0-9]*">\n((?:.*\n){0,}?)</td>`)
)

func main() {
	fid := "139"
	resp, err := http.Get("http://www.mcbbs.net/forum.php?mod=forumdisplay&fid=" + fid + "&orderby=dateline&page=1")
	if err != nil {
		printError("GetPage", err)
		os.Exit(1)
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		printError("ReadAll", err)
		os.Exit(1)
	}

	// 获取帖子列表总页数
	buf := string(getMaxPagesNum.FindSubmatch(body)[1])
	maxPagesNum, err := strconv.Atoi(buf)
	if err != nil {
		printError("Atoi", err)
		os.Exit(1)
	}
	// 获取每一页的所有帖子
	for i := 0; i < maxPagesNum; i++ {
		pageList, err := getPagesList(fid, i+1)
		if err != nil {
			printError("getPagesList"+string(i), err)
			return
		}
		for _, v := range pageList {
			resp, err := http.Get("http://www.mcbbs.net/" + v.url)
			if err != nil {
				printError("GetPage", err)
				return
			}
			defer resp.Body.Close()

			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				printError("ReadAll", err)
				return
			}

			postBody := string(getPostBody.FindSubmatch(body)[1])
			log.Println(postBody)
		}
	}
}

// 获取单页面的所有帖子
func getPagesList(fid string, pageNum int) (*[]pageInfo, error) {
	resp, err := http.Get("http://www.mcbbs.net/forum.php?mod=forumdisplay&fid=" + fid + "&orderby=dateline&page=" + strconv.Itoa(pageNum))
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	/*
		// getPage所匹配的信息
		// 获取到的参数已用“m[x]说明”标出

		[m[1]分类] m[3]标题

		 ...234
		New

		m[4]作者
		m[6]发帖时间2/span>

		111189

		谢普
		5 天前

	*/
	m := getPage.FindAllStringSubmatch(string(body), -1)
	var pageList = new([]pageInfo)

	for _, v := range m {
		date := v[5]
		// 如果发帖时间1为空，那么使用发帖时间2
		if date == "" {
			date = v[6]
		}
		pageInf := pageInfo{
			category: v[1],
			url:      v[2],
			title:    v[3],
			author:   v[4],
			date:     date,
		}
		*pageList = append(*pageList, pageInf)
	}

	return pageList, nil
}

func printInfo(s string, v ...interface{}) {
	log.Println("[INFO]", s, v)
}

func printError(s string, v ...interface{}) {
	log.Println("[ERROR]", s, v)
}

type pageInfo struct {
	category string
	url      string
	title    string
	author   string
	date     string
}
<|endoftext|>