<commit_before>package sstable

import (
	"bytes"
	"encoding/binary"
	"io"
	"log"

	"github.com/golang/protobuf/proto"

	pb "github.com/akmistry/simple-sstable/proto"
)

type keyLengthPair struct {
	key    []byte
	length uint32
}

type Builder struct {
	w  io.Writer
	vf ValueFunc

	indexBuf proto.Buffer
	keys     []keyLengthPair
	valuePos uint64

	started bool
	prev    []byte
}

type ValueFunc func(key []byte, p []byte) error

func NewBuilder(w io.Writer, vf ValueFunc) *Builder {
	return &Builder{w: w, vf: vf}
}

func (b *Builder) Add(key []byte, valueLength uint32, meta []byte) {
	if !b.started {
		b.started = true
	} else if bytes.Compare(b.prev, key) != -1 {
		log.Panicf("Key %d is before previous %d", key, b.prev)
	}

	if len(key) > 256 {
		log.Panicf("Key length %d > 256", len(key))
	}

	keyDup := dup(key)
	b.prev = keyDup

	var entry pb.IndexEntry
	entry.Key = key
	entry.Offset = b.valuePos
	b.valuePos += uint64(valueLength)
	entry.Length = valueLength
	entry.Extra = meta
	b.keys = append(b.keys, keyLengthPair{keyDup, valueLength})
	if err := b.indexBuf.EncodeMessage(&entry); err != nil {
		log.Panicln("Unexpected error encoding index", err)
	}
}

func (b *Builder) Build() error {
	var header pb.TableHeader
	header.Version = 1
	header.IndexLength = uint32(len(b.indexBuf.Bytes()))
	// TODO: Implement index compression.

	headerBuf, err := proto.Marshal(&header)
	if err != nil {
		return err
	}

	var headerSize [4]byte
	binary.LittleEndian.PutUint32(headerSize[:], uint32(len(headerBuf)))
	_, err = writeMany(b.w, headerSize[:], headerBuf, b.indexBuf.Bytes())
	if err != nil {
		return err
	}

	for _, pair := range b.keys {
		if pair.length == 0 {
			continue
		}
		val := make([]byte, pair.length)
		err := b.vf(pair.key, val)
		if err != nil {
			return err
		}

		_, err = b.w.Write(val)
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>Recycle value buffer when building, if possible.<commit_after>package sstable

import (
	"bytes"
	"encoding/binary"
	"io"
	"log"

	"github.com/golang/protobuf/proto"

	pb "github.com/akmistry/simple-sstable/proto"
)

type keyLengthPair struct {
	key    []byte
	length uint32
}

type Builder struct {
	w  io.Writer
	vf ValueFunc

	indexBuf proto.Buffer
	keys     []keyLengthPair
	valuePos uint64

	started bool
	prev    []byte
}

type ValueFunc func(key []byte, p []byte) error

func NewBuilder(w io.Writer, vf ValueFunc) *Builder {
	return &Builder{w: w, vf: vf}
}

func (b *Builder) Add(key []byte, valueLength uint32, meta []byte) {
	if !b.started {
		b.started = true
	} else if bytes.Compare(b.prev, key) != -1 {
		log.Panicf("Key %d is before previous %d", key, b.prev)
	}

	if len(key) > 256 {
		log.Panicf("Key length %d > 256", len(key))
	}

	keyDup := dup(key)
	b.prev = keyDup

	var entry pb.IndexEntry
	entry.Key = key
	entry.Offset = b.valuePos
	b.valuePos += uint64(valueLength)
	entry.Length = valueLength
	entry.Extra = meta
	b.keys = append(b.keys, keyLengthPair{keyDup, valueLength})
	if err := b.indexBuf.EncodeMessage(&entry); err != nil {
		log.Panicln("Unexpected error encoding index", err)
	}
}

func (b *Builder) Build() error {
	var header pb.TableHeader
	header.Version = 1
	header.IndexLength = uint32(len(b.indexBuf.Bytes()))
	// TODO: Implement index compression.

	headerBuf, err := proto.Marshal(&header)
	if err != nil {
		return err
	}

	var headerSize [4]byte
	binary.LittleEndian.PutUint32(headerSize[:], uint32(len(headerBuf)))
	_, err = writeMany(b.w, headerSize[:], headerBuf, b.indexBuf.Bytes())
	if err != nil {
		return err
	}

	var val []byte
	for _, pair := range b.keys {
		if pair.length == 0 {
			continue
		}
		if uint32(cap(val)) < pair.length {
			val = make([]byte, pair.length)
		} else {
			val = val[:pair.length]
		}
		err := b.vf(pair.key, val)
		if err != nil {
			return err
		}

		_, err = b.w.Write(val)
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"flag"
	"github.com/golang/glog"
	"os"
	"mime/multipart"
	"io"
	)

func main() {
    flag.Parse()

    /* TODO add parametrization */
    glog.Info("Starting File Server on port 6543")
    http.Handle("/cdn/", http.StripPrefix("/cdn", http.FileServer(http.Dir("."))))
    glog.Info("Starting API Server on port 6543 path")
    http.HandleFunc("/api/upload", func(res http.ResponseWriter, req *http.Request) {
				var err error
				defer func() {
				    if nil != err {
					//http.Error(res, err.Error(), status)
					glog.Error("Error string: errStr", err)
					res.Write([]byte(`{"status":"error", "message":"` + err.Error()  + `"}`))
				    }
				}()
				// max bytes in mem at a time
				const _24K = (1 << 20) * 24
				err = req.ParseMultipartForm(_24K)
				if nil != err {
				    return
				}
				for _, fheaders := range req.MultipartForm.File {
				    for _, hdr := range fheaders {
					// open uploaded
					var infile multipart.File
					infile, err = hdr.Open()
					if nil != err {
					     return
					}
					// open destination
					var outfile *os.File
					outfile, err = os.Create(hdr.Filename)
					if nil != err {
					     return
					}
					// 32K buffer copy
					var written int64
					written, err = io.Copy(outfile, infile)
					if nil != err || 0 == written {
					     return
					}
					//res.Write([]byte("uploaded file:" + hdr.Filename + ";length:" + strconv.Itoa(int(written))))
					res.Write([]byte(`{"status":"ok", "resource_url":"` + `path_to_storage` + hdr.Filename + `"}`))
				    }
				}
    })
    http.ListenAndServe(":6543", nil)
}

<commit_msg>chore: added todos<commit_after>package main

import (
	"net/http"
	"flag"
	"github.com/golang/glog"
	"os"
	"mime/multipart"
	"io"
	"strconv"
	)

func main() {
    flag.Parse()

    /* TODO add parametrization
	- port number
	- server path to cdn TODO
	- server base url
	- server upload url TODO
	- cdn dir
    */

    /*
    * issues: files with same name can be overridden
    * there is literally no security
    * They will be solved on the second iteration of the cdn
    */

    a := os.Getenv("GOPATH1")
    glog.Info("AAA: ", a)
    portNumber, err := strconv.Atoi(os.Getenv("ABC_CDN_PORTNUM"))
    if nil != err {
	glog.Fatal("Error: ", err, portNumber)
    }

    glog.Info(`Starting File Server on port`, os.Getenv("ABC_CDN_PORTNUM"))
    http.Handle("/cdn/", http.StripPrefix("/cdn", http.FileServer(http.Dir("."))))
    glog.Info("Starting API Server on port 6543 path")
    http.HandleFunc("/api/upload", func(res http.ResponseWriter, req *http.Request) {
				var err error
				defer func() {
				    if nil != err {
					glog.Error("Error string: errStr", err)
					res.Write([]byte(`{"status":"error", "message":"` + err.Error()  + `"}`))
				    }
				}()
				// max bytes in mem at a time
				const _24K = (1 << 20) * 24
				err = req.ParseMultipartForm(_24K)
				if nil != err {
				    return
				}
				for _, fheaders := range req.MultipartForm.File {
				    for _, hdr := range fheaders {
					// open uploaded
					var infile multipart.File
					infile, err = hdr.Open()
					if nil != err {
					     return
					}
					// open destination
					var outfile *os.File
					outfile, err = os.Create(hdr.Filename)
					if nil != err {
					     return
					}
					var written int64
					written, err = io.Copy(outfile, infile)
					if nil != err || 0 == written {
					     return
					}
					res.Write([]byte(`{"status":"ok", "resource_url":"` + `path_to_storage` + hdr.Filename + `"}`))
				    }
				}
    })
    http.ListenAndServe(`:` + os.Getenv("ABC_CDN_PORTNUM"), nil)
}

<|endoftext|>


<commit_before>package views

const pre = `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<!--

    ( )      / __      ___      __      ___   / //
    / /      //   ) ) //   ) ) //  ) ) //   ) / // //   / /
    / /      //   / / //   / / //      //   / / // ((___/ /
    / /      //   / / ((___( ( //      ((___/ / //      / /

    / ___       __      ___                      / __      ___      __
    //\ \     //   ) ) //   ) ) //  / /  / /     //   ) ) //___) ) //  ) )
    //  \ \   //   / / //   / / //  / /  / /     //   / / //       //
    //    \ \ //   / / ((___/ / ((__( (__/ /     //   / / ((____   //

  -->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
  <head>
    <script type="text/javascript">var _sf_startpt=(new Date()).getTime()</script>
    <title>{{.Title}}</title>
    <meta http-equiv="content-type" content="text/html; charset=utf-8" />
    <meta http-equiv="imagetoolbar" content="no" />
    <meta name="warning" content="HC SVNT DRACONES" />
    <meta name="viewport" content="width=500;initial-scale=1" />
    <style>
      html{color:#000;background:#FFF;}body,div,dl,dt,dd,ul,ol,li,h1,h2,h3,h4,h5,h6,pre,code,form,fieldset,legend,input,button,textarea,p,blockquote,th,td{margin:0;padding:0;}table{border-collapse:collapse;border-spacing:0;}fieldset,img{border:0;}address,caption,cite,code,dfn,em,strong,th,var,optgroup{font-style:inherit;font-weight:inherit;}del,ins{text-decoration:none;}li{list-style:none;}caption,th{text-align:left;}h1,h2,h3,h4,h5,h6{font-size:100%;font-weight:normal;}q:before,q:after{content:'';}abbr,acronym{border:0;font-variant:normal;}sup{vertical-align:baseline;}sub{vertical-align:baseline;}legend{color:#000;}input,button,textarea,select,optgroup,option{font-family:inherit;font-size:inherit;font-style:inherit;font-weight:inherit;}input,button,textarea,select{*font-size:100%;}

      div.regular { font: 12px/20px helvetica,sans-serif; padding: 2em; }
      div.regular a { color: #000; }
      div.regular a:hover { text-decoration: none; }
      div.regular p, div.regular h1 { margin-bottom: 1.5em; }
      div.regular strong { font-weight: bold; }
      div.regular em { font-family: courier; font-style: italic; font-size: 11px; background: #e3e3e3; margin-left: 0.4em; }
      div.regular li.footer { color: #ddd; font-size: 11px; margin-top: 2em; }
      div.regular li.footer a { color: #ddd; }
      div.regular li.footer a:hover { text-decoration: none; }

      div.faq h2 { font-weight: bold; }
      div.faq p { width: 400px; margin-bottom: 2em; }

      div.settings form { margin-bottom: 1.5em; ;}
      div.settings form p { margin-top: 1.5em; margin-bottom: 0; }
      div.settings form input[type='radio'] { margin-left: 2em; }

      div.photos { margin-top: 30px; }
      div.photos ul { display: flex; justify-content: center; flex-direction: column; }
      div.photos li { margin: 0 auto; line-height: 0; padding: 1rem 0; }
      div.photos img { max-width: 95vw; max-height: 95vh; }
      div.photos li.pagers, div.photos li.footer { color: #333; text-align: center; position: relative; font: 14px helvetica,sans-serif; }
      div.photos li.pagers div.left { position: absolute; bottom: 0; left: 0; }
      div.photos li.pagers div.right { position: absolute; bottom: 0; right: 0; }
      div.photos li.pagers a { color: #333; }
      div.photos li.pagers a:hover { text-decoration: none; }
      div.photos li.pagers a.homelink { font-size: 12px; text-transform: uppercase; }
      div.photos li.footer { color: #ddd; font-size: 11px; }
      div.photos li.footer a { color: #ddd; }
      div.photos li.footer a:hover { text-decoration: none; }
    </style>
  </head>
  <body>
    `

const post = `
  </body>
</html>`
<commit_msg>Remove top margin on photo list<commit_after>package views

const pre = `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<!--

    ( )      / __      ___      __      ___   / //
    / /      //   ) ) //   ) ) //  ) ) //   ) / // //   / /
    / /      //   / / //   / / //      //   / / // ((___/ /
    / /      //   / / ((___( ( //      ((___/ / //      / /

    / ___       __      ___                      / __      ___      __
    //\ \     //   ) ) //   ) ) //  / /  / /     //   ) ) //___) ) //  ) )
    //  \ \   //   / / //   / / //  / /  / /     //   / / //       //
    //    \ \ //   / / ((___/ / ((__( (__/ /     //   / / ((____   //

  -->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
  <head>
    <script type="text/javascript">var _sf_startpt=(new Date()).getTime()</script>
    <title>{{.Title}}</title>
    <meta http-equiv="content-type" content="text/html; charset=utf-8" />
    <meta http-equiv="imagetoolbar" content="no" />
    <meta name="warning" content="HC SVNT DRACONES" />
    <meta name="viewport" content="width=500;initial-scale=1" />
    <style>
      html{color:#000;background:#FFF;}body,div,dl,dt,dd,ul,ol,li,h1,h2,h3,h4,h5,h6,pre,code,form,fieldset,legend,input,button,textarea,p,blockquote,th,td{margin:0;padding:0;}table{border-collapse:collapse;border-spacing:0;}fieldset,img{border:0;}address,caption,cite,code,dfn,em,strong,th,var,optgroup{font-style:inherit;font-weight:inherit;}del,ins{text-decoration:none;}li{list-style:none;}caption,th{text-align:left;}h1,h2,h3,h4,h5,h6{font-size:100%;font-weight:normal;}q:before,q:after{content:'';}abbr,acronym{border:0;font-variant:normal;}sup{vertical-align:baseline;}sub{vertical-align:baseline;}legend{color:#000;}input,button,textarea,select,optgroup,option{font-family:inherit;font-size:inherit;font-style:inherit;font-weight:inherit;}input,button,textarea,select{*font-size:100%;}

      div.regular { font: 12px/20px helvetica,sans-serif; padding: 2em; }
      div.regular a { color: #000; }
      div.regular a:hover { text-decoration: none; }
      div.regular p, div.regular h1 { margin-bottom: 1.5em; }
      div.regular strong { font-weight: bold; }
      div.regular em { font-family: courier; font-style: italic; font-size: 11px; background: #e3e3e3; margin-left: 0.4em; }
      div.regular li.footer { color: #ddd; font-size: 11px; margin-top: 2em; }
      div.regular li.footer a { color: #ddd; }
      div.regular li.footer a:hover { text-decoration: none; }

      div.faq h2 { font-weight: bold; }
      div.faq p { width: 400px; margin-bottom: 2em; }

      div.settings form { margin-bottom: 1.5em; ;}
      div.settings form p { margin-top: 1.5em; margin-bottom: 0; }
      div.settings form input[type='radio'] { margin-left: 2em; }

      div.photos ul { display: flex; justify-content: center; flex-direction: column; }
      div.photos li { margin: 0 auto; line-height: 0; padding: 1rem 0; }
      div.photos img { max-width: 95vw; max-height: 95vh; }
      div.photos li.pagers, div.photos li.footer { color: #333; text-align: center; position: relative; font: 14px helvetica,sans-serif; }
      div.photos li.pagers div.left { position: absolute; bottom: 0; left: 0; }
      div.photos li.pagers div.right { position: absolute; bottom: 0; right: 0; }
      div.photos li.pagers a { color: #333; }
      div.photos li.pagers a:hover { text-decoration: none; }
      div.photos li.pagers a.homelink { font-size: 12px; text-transform: uppercase; }
      div.photos li.footer { color: #ddd; font-size: 11px; }
      div.photos li.footer a { color: #ddd; }
      div.photos li.footer a:hover { text-decoration: none; }
    </style>
  </head>
  <body>
    `

const post = `
  </body>
</html>`
<|endoftext|>


<commit_before>package web

import (
	"fmt"
	"os"
	"os/exec"
	"strings"

	"github.com/gorilla/mux"
	"github.com/zenoss/glog"
	"github.com/zenoss/go-json-rest"
	"github.com/zenoss/serviced"
	"github.com/zenoss/serviced/dao"

	"mime"
	"net/http"
	"net/http/httputil"
	"net/url"
)

type ServiceConfig struct {
	bindPort    string
	agentPort   string
	zookeepers  []string
	stats       bool
	hostaliases []string
}

func NewServiceConfig(bindPort string, agentPort string, zookeepers []string, stats bool, hostaliases string) *ServiceConfig {
	cfg := ServiceConfig{bindPort, agentPort, zookeepers, stats, []string{}}
	if hostaliases != "" {
		cfg.hostaliases = strings.Split(hostaliases, ":")
	}
	if len(cfg.bindPort) == 0 {
		cfg.bindPort = ":8787"
	}
	if len(cfg.agentPort) == 0 {
		cfg.agentPort = "127.0.0.1:4979"
	}
	if len(cfg.zookeepers) == 0 {
		cfg.zookeepers = []string{"127.0.0.1:2181"}
	}
	return &cfg
}

// Serve handles control plane web UI requests and virtual host requests for zenoss web based services.
// The UI server actually listens on port 7878, the uihandler defined here just reverse proxies to it.
// Virutal host routing to zenoss web based services is done by the vhosthandler function.
func (sc *ServiceConfig) Serve() {
	client, err := sc.getClient()
	if err != nil {
		glog.Errorf("Unable to get control plane client: %v", err)
		return
	}

	// Reverse proxy to the web UI server.
	uihandler := func(w http.ResponseWriter, r *http.Request) {
		uiUrl, err := url.Parse("http://127.0.0.1:7878")
		if err != nil {
			glog.Errorf("Can't parse UI URL: %v", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		ui := httputil.NewSingleHostReverseProxy(uiUrl)
		if ui == nil {
			glog.Errorf("Can't proxy UI request: %v", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		ui.ServeHTTP(w, r)
	}

	// Lookup the appropriate virtual host and forward the request to it.
	// TODO: when zookeeper registration is integrated we can be more event
	// driven and only refresh the vhost map when service states change.
	vhosthandler := func(w http.ResponseWriter, r *http.Request) {
		var empty interface{}
		services := []*dao.RunningService{}
		client.GetRunningServices(&empty, &services)

		vhosts := make(map[string][]*dao.ServiceState, 0)

		for _, s := range services {
			var svc dao.Service

			if err := client.GetService(s.ServiceId, &svc); err != nil {
				glog.Errorf("Can't get service: %s (%v)", s.Id, err)
			}

			vheps := svc.GetServiceVHosts()

			for _, vhep := range vheps {
				for _, vh := range vhep.VHosts {
					svcstates := []*dao.ServiceState{}
					if err := client.GetServiceStates(s.ServiceId, &svcstates); err != nil {
						http.Error(w, fmt.Sprintf("can't retrieve service states for %s (%v)", s.ServiceId, err), http.StatusInternalServerError)
						return
					}

					for _, ss := range svcstates {
						vhosts[vh] = append(vhosts[vh], ss)
					}
				}
			}

		}

		muxvars := mux.Vars(r)
		svcstates, ok := vhosts[muxvars["subdomain"]]
		if !ok {
			http.Error(w, fmt.Sprintf("unknown vhost: %v", muxvars["subdomain"]), http.StatusNotFound)
			return
		}

		// TODO: implement a more intelligent strategy than "always pick the first one" when more
		// than one service state is mapped to a given virtual host
		for _, svcep := range svcstates[0].Endpoints {
			for _, vh := range svcep.VHosts {
				if vh == muxvars["subdomain"] {
					rp := httputil.NewSingleHostReverseProxy(&url.URL{Scheme: "http", Host: fmt.Sprintf("%s:%d", svcstates[0].PrivateIp, svcep.PortNumber)})
					rp.ServeHTTP(w, r)
					return
				}
			}
		}

		http.Error(w, fmt.Sprintf("unrecognized endpoint: %s", muxvars["subdomain"]), http.StatusNotImplemented)
	}

	r := mux.NewRouter()

	if hnm, err := os.Hostname(); err == nil {
		sc.hostaliases = append(sc.hostaliases, hnm)
	}

	cmd := exec.Command("hostname", "--fqdn")
	if hnm, err := cmd.CombinedOutput(); err == nil {
		sc.hostaliases = append(sc.hostaliases, string(hnm[:len(hnm)-1]))
	}

	for _, ha := range sc.hostaliases {
		r.HandleFunc("/", vhosthandler).Host(fmt.Sprintf("{subdomain}.%s", ha))
		r.HandleFunc("/{path:.*}", vhosthandler).Host(fmt.Sprintf("{subdomain}.%s", ha))
	}

	r.HandleFunc("/{path:.*}", uihandler)

	http.Handle("/", r)
	certfile, err := serviced.TempCertFile()
	if err != nil {
		glog.Fatal("Could not prepare cert.pem file.")
	}
	keyfile, err := serviced.TempKeyFile()
	if err != nil {
		glog.Fatal("Could not prepare key.pem file.")
	}
	http.ListenAndServeTLS(sc.bindPort, certfile, keyfile, nil)
}

func (this *ServiceConfig) ServeUI() {
	mime.AddExtensionType(".json", "application/json")
	mime.AddExtensionType(".woff", "application/font-woff")

	handler := rest.ResourceHandler{
		EnableRelaxedContentType: true,
	}
	routes := []rest.Route{
		rest.Route{"GET", "/", MainPage},
		rest.Route{"GET", "/test", TestPage},
		rest.Route{"GET", "/stats", this.IsCollectingStats()},
		// Hosts
		rest.Route{"GET", "/hosts", this.AuthorizedClient(RestGetHosts)},
		rest.Route{"POST", "/hosts/add", this.AuthorizedClient(RestAddHost)},
		rest.Route{"DELETE", "/hosts/:hostId", this.AuthorizedClient(RestRemoveHost)},
		rest.Route{"PUT", "/hosts/:hostId", this.AuthorizedClient(RestUpdateHost)},
		rest.Route{"GET", "/hosts/:hostId/running", this.AuthorizedClient(RestGetRunningForHost)},
		rest.Route{"DELETE", "/hosts/:hostId/:serviceStateId", this.AuthorizedClient(RestKillRunning)},
		// Pools
		rest.Route{"POST", "/pools/add", this.AuthorizedClient(RestAddPool)},
		rest.Route{"GET", "/pools/:poolId/hosts", this.AuthorizedClient(RestGetHostsForResourcePool)},
		rest.Route{"DELETE", "/pools/:poolId", this.AuthorizedClient(RestRemovePool)},
		rest.Route{"PUT", "/pools/:poolId", this.AuthorizedClient(RestUpdatePool)},
		rest.Route{"GET", "/pools", this.AuthorizedClient(RestGetPools)},
		// Services (Apps)
		rest.Route{"GET", "/services", this.AuthorizedClient(RestGetAllServices)},
		rest.Route{"GET", "/services/:serviceId", this.AuthorizedClient(RestGetService)},
		rest.Route{"GET", "/services/:serviceId/running", this.AuthorizedClient(RestGetRunningForService)},
		rest.Route{"GET", "/services/:serviceId/running/:serviceStateId", this.AuthorizedClient(RestGetRunningService)},
		rest.Route{"GET", "/services/:serviceId/:serviceStateId/logs", this.AuthorizedClient(RestGetServiceStateLogs)},
		rest.Route{"POST", "/services/add", this.AuthorizedClient(RestAddService)},
		rest.Route{"DELETE", "/services/:serviceId", this.AuthorizedClient(RestRemoveService)},
		rest.Route{"GET", "/services/:serviceId/logs", this.AuthorizedClient(RestGetServiceLogs)},
		rest.Route{"PUT", "/services/:serviceId", this.AuthorizedClient(RestUpdateService)},
		rest.Route{"GET", "/services/:serviceId/snapshot", this.AuthorizedClient(RestSnapshotService)},
		rest.Route{"PUT", "/services/:serviceId/startService", this.AuthorizedClient(RestStartService)},
		rest.Route{"PUT", "/services/:serviceId/stopService", this.AuthorizedClient(RestStopService)},

		// Services (Virtual Host)
		rest.Route{"GET", "/vhosts", this.AuthorizedClient(RestGetVirtualHosts)},
		rest.Route{"POST", "/vhosts/:serviceId/:application/:vhostName", this.AuthorizedClient(RestAddVirtualHost)},
		rest.Route{"DELETE", "/vhosts/:serviceId/:application/:vhostName", this.AuthorizedClient(RestRemoveVirtualHost)},

		// Service templates (App templates)
		rest.Route{"GET", "/templates", this.AuthorizedClient(RestGetAppTemplates)},
		rest.Route{"POST", "/templates/deploy", this.AuthorizedClient(RestDeployAppTemplate)},
		// Login
		rest.Route{"POST", "/login", this.UnAuthorizedClient(RestLogin)},
		rest.Route{"DELETE", "/login", RestLogout},
		// "Misc" stuff
		rest.Route{"GET", "/top/services", this.AuthorizedClient(RestGetTopServices)},
		rest.Route{"GET", "/running", this.AuthorizedClient(RestGetAllRunning)},
		// Generic static data
		rest.Route{"GET", "/favicon.ico", FavIcon},
		rest.Route{"GET", "/static*resource", StaticData},
	}

	// Hardcoding these target URLs for now.
	// TODO: When internal services are allowed to run on other hosts, look that up.
	routes = routeToInternalServiceProxy("/elastic", "http://127.0.0.1:9200/", routes)
	routes = routeToInternalServiceProxy("/metrics", "http://127.0.0.1:8888/", routes)

	handler.SetRoutes(routes...)

	http.ListenAndServe(":7878", &handler)
}

var methods []string = []string{"GET", "POST", "PUT", "DELETE"}

func routeToInternalServiceProxy(path string, target string, routes []rest.Route) []rest.Route {
	targetUrl, err := url.Parse(target)
	if err != nil {
		glog.Errorf("Unable to parse proxy target URL: %s", target)
		return routes
	}
	// Wrap the normal http.Handler in a rest.HandlerFunc
	handlerFunc := func(w *rest.ResponseWriter, r *rest.Request) {
		proxy := serviced.NewReverseProxy(path, targetUrl)
		proxy.ServeHTTP(w.ResponseWriter, r.Request)
	}
	// Add on a glob to match subpaths
	andsubpath := path + "*x"
	for _, method := range methods {
		routes = append(routes, rest.Route{method, path, handlerFunc})
		routes = append(routes, rest.Route{method, andsubpath, handlerFunc})
	}
	return routes
}

func (this *ServiceConfig) UnAuthorizedClient(realfunc HandlerClientFunc) HandlerFunc {
	return func(w *rest.ResponseWriter, r *rest.Request) {
		client, err := this.getClient()
		if err != nil {
			glog.Errorf("Unable to acquire client: %v", err)
			RestServerError(w)
			return
		}
		defer client.Close()
		realfunc(w, r, client)
	}
}

func (this *ServiceConfig) AuthorizedClient(realfunc HandlerClientFunc) HandlerFunc {
	return func(w *rest.ResponseWriter, r *rest.Request) {
		if !LoginOk(r) {
			RestUnauthorized(w)
			return
		}
		client, err := this.getClient()
		if err != nil {
			glog.Errorf("Unable to acquire client: %v", err)
			RestServerError(w)
			return
		}
		defer client.Close()
		realfunc(w, r, client)
	}
}

func (this *ServiceConfig) IsCollectingStats() HandlerFunc {
	if this.stats {
		return func(w *rest.ResponseWriter, r *rest.Request) {
			w.WriteHeader(http.StatusOK)
		}
	} else {
		return func(w *rest.ResponseWriter, r *rest.Request) {
			w.WriteHeader(http.StatusNotImplemented)
		}
	}
}

func (this *ServiceConfig) getClient() (c *serviced.ControlClient, err error) {
	// setup the client
	c, err = serviced.NewControlClient(this.agentPort)
	if err != nil {
		glog.Fatalf("Could not create a control plane client: %v", err)
	}
	return c, err
}
<commit_msg>style<commit_after>package web

import (
	"fmt"
	"os"
	"os/exec"
	"strings"

	"github.com/gorilla/mux"
	"github.com/zenoss/glog"
	"github.com/zenoss/go-json-rest"
	"github.com/zenoss/serviced"
	"github.com/zenoss/serviced/dao"

	"mime"
	"net/http"
	"net/http/httputil"
	"net/url"
)

type ServiceConfig struct {
	bindPort    string
	agentPort   string
	zookeepers  []string
	stats       bool
	hostaliases []string
}

func NewServiceConfig(bindPort string, agentPort string, zookeepers []string, stats bool, hostaliases string) *ServiceConfig {
	cfg := ServiceConfig{bindPort, agentPort, zookeepers, stats, []string{}}
	if hostaliases != "" {
		cfg.hostaliases = strings.Split(hostaliases, ":")
	}
	if len(cfg.bindPort) == 0 {
		cfg.bindPort = ":8787"
	}
	if len(cfg.agentPort) == 0 {
		cfg.agentPort = "127.0.0.1:4979"
	}
	if len(cfg.zookeepers) == 0 {
		cfg.zookeepers = []string{"127.0.0.1:2181"}
	}
	return &cfg
}

// Serve handles control plane web UI requests and virtual host requests for zenoss web based services.
// The UI server actually listens on port 7878, the uihandler defined here just reverse proxies to it.
// Virutal host routing to zenoss web based services is done by the vhosthandler function.
func (sc *ServiceConfig) Serve() {
	client, err := sc.getClient()
	if err != nil {
		glog.Errorf("Unable to get control plane client: %v", err)
		return
	}

	// Reverse proxy to the web UI server.
	uihandler := func(w http.ResponseWriter, r *http.Request) {
		uiUrl, err := url.Parse("http://127.0.0.1:7878")
		if err != nil {
			glog.Errorf("Can't parse UI URL: %v", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		ui := httputil.NewSingleHostReverseProxy(uiUrl)
		if ui == nil {
			glog.Errorf("Can't proxy UI request: %v", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		ui.ServeHTTP(w, r)
	}

	// Lookup the appropriate virtual host and forward the request to it.
	// TODO: when zookeeper registration is integrated we can be more event
	// driven and only refresh the vhost map when service states change.
	vhosthandler := func(w http.ResponseWriter, r *http.Request) {
		var empty interface{}
		services := []*dao.RunningService{}
		client.GetRunningServices(&empty, &services)

		vhosts := make(map[string][]*dao.ServiceState, 0)

		for _, s := range services {
			var svc dao.Service

			if err := client.GetService(s.ServiceId, &svc); err != nil {
				glog.Errorf("Can't get service: %s (%v)", s.Id, err)
			}

			vheps := svc.GetServiceVHosts()

			for _, vhep := range vheps {
				for _, vh := range vhep.VHosts {
					svcstates := []*dao.ServiceState{}
					if err := client.GetServiceStates(s.ServiceId, &svcstates); err != nil {
						http.Error(w, fmt.Sprintf("can't retrieve service states for %s (%v)", s.ServiceId, err), http.StatusInternalServerError)
						return
					}

					for _, ss := range svcstates {
						vhosts[vh] = append(vhosts[vh], ss)
					}
				}
			}

		}

		muxvars := mux.Vars(r)
		svcstates, ok := vhosts[muxvars["subdomain"]]
		if !ok {
			http.Error(w, fmt.Sprintf("unknown vhost: %v", muxvars["subdomain"]), http.StatusNotFound)
			return
		}

		// TODO: implement a more intelligent strategy than "always pick the first one" when more
		// than one service state is mapped to a given virtual host
		for _, svcep := range svcstates[0].Endpoints {
			for _, vh := range svcep.VHosts {
				if vh == muxvars["subdomain"] {
					rp := httputil.NewSingleHostReverseProxy(&url.URL{Scheme: "http", Host: fmt.Sprintf("%s:%d", svcstates[0].PrivateIp, svcep.PortNumber)})
					rp.ServeHTTP(w, r)
					return
				}
			}
		}

		http.Error(w, fmt.Sprintf("unrecognized endpoint: %s", muxvars["subdomain"]), http.StatusNotImplemented)
	}

	r := mux.NewRouter()

	if hnm, err := os.Hostname(); err == nil {
		sc.hostaliases = append(sc.hostaliases, hnm)
	}

	cmd := exec.Command("hostname", "--fqdn")
	if hnm, err := cmd.CombinedOutput(); err == nil {
		sc.hostaliases = append(sc.hostaliases, string(hnm[:len(hnm)-1]))
	}

	for _, ha := range sc.hostaliases {
		r.HandleFunc("/", vhosthandler).Host(fmt.Sprintf("{subdomain}.%s", ha))
		r.HandleFunc("/{path:.*}", vhosthandler).Host(fmt.Sprintf("{subdomain}.%s", ha))
	}

	r.HandleFunc("/{path:.*}", uihandler)

	http.Handle("/", r)

	certfile, err := serviced.TempCertFile()
	if err != nil {
		glog.Fatal("Could not prepare cert.pem file.")
	}
	keyfile, err := serviced.TempKeyFile()
	if err != nil {
		glog.Fatal("Could not prepare key.pem file.")
	}
	http.ListenAndServeTLS(sc.bindPort, certfile, keyfile, nil)
}

func (this *ServiceConfig) ServeUI() {
	mime.AddExtensionType(".json", "application/json")
	mime.AddExtensionType(".woff", "application/font-woff")

	handler := rest.ResourceHandler{
		EnableRelaxedContentType: true,
	}
	routes := []rest.Route{
		rest.Route{"GET", "/", MainPage},
		rest.Route{"GET", "/test", TestPage},
		rest.Route{"GET", "/stats", this.IsCollectingStats()},
		// Hosts
		rest.Route{"GET", "/hosts", this.AuthorizedClient(RestGetHosts)},
		rest.Route{"POST", "/hosts/add", this.AuthorizedClient(RestAddHost)},
		rest.Route{"DELETE", "/hosts/:hostId", this.AuthorizedClient(RestRemoveHost)},
		rest.Route{"PUT", "/hosts/:hostId", this.AuthorizedClient(RestUpdateHost)},
		rest.Route{"GET", "/hosts/:hostId/running", this.AuthorizedClient(RestGetRunningForHost)},
		rest.Route{"DELETE", "/hosts/:hostId/:serviceStateId", this.AuthorizedClient(RestKillRunning)},
		// Pools
		rest.Route{"POST", "/pools/add", this.AuthorizedClient(RestAddPool)},
		rest.Route{"GET", "/pools/:poolId/hosts", this.AuthorizedClient(RestGetHostsForResourcePool)},
		rest.Route{"DELETE", "/pools/:poolId", this.AuthorizedClient(RestRemovePool)},
		rest.Route{"PUT", "/pools/:poolId", this.AuthorizedClient(RestUpdatePool)},
		rest.Route{"GET", "/pools", this.AuthorizedClient(RestGetPools)},
		// Services (Apps)
		rest.Route{"GET", "/services", this.AuthorizedClient(RestGetAllServices)},
		rest.Route{"GET", "/services/:serviceId", this.AuthorizedClient(RestGetService)},
		rest.Route{"GET", "/services/:serviceId/running", this.AuthorizedClient(RestGetRunningForService)},
		rest.Route{"GET", "/services/:serviceId/running/:serviceStateId", this.AuthorizedClient(RestGetRunningService)},
		rest.Route{"GET", "/services/:serviceId/:serviceStateId/logs", this.AuthorizedClient(RestGetServiceStateLogs)},
		rest.Route{"POST", "/services/add", this.AuthorizedClient(RestAddService)},
		rest.Route{"DELETE", "/services/:serviceId", this.AuthorizedClient(RestRemoveService)},
		rest.Route{"GET", "/services/:serviceId/logs", this.AuthorizedClient(RestGetServiceLogs)},
		rest.Route{"PUT", "/services/:serviceId", this.AuthorizedClient(RestUpdateService)},
		rest.Route{"GET", "/services/:serviceId/snapshot", this.AuthorizedClient(RestSnapshotService)},
		rest.Route{"PUT", "/services/:serviceId/startService", this.AuthorizedClient(RestStartService)},
		rest.Route{"PUT", "/services/:serviceId/stopService", this.AuthorizedClient(RestStopService)},

		// Services (Virtual Host)
		rest.Route{"GET", "/vhosts", this.AuthorizedClient(RestGetVirtualHosts)},
		rest.Route{"POST", "/vhosts/:serviceId/:application/:vhostName", this.AuthorizedClient(RestAddVirtualHost)},
		rest.Route{"DELETE", "/vhosts/:serviceId/:application/:vhostName", this.AuthorizedClient(RestRemoveVirtualHost)},

		// Service templates (App templates)
		rest.Route{"GET", "/templates", this.AuthorizedClient(RestGetAppTemplates)},
		rest.Route{"POST", "/templates/deploy", this.AuthorizedClient(RestDeployAppTemplate)},
		// Login
		rest.Route{"POST", "/login", this.UnAuthorizedClient(RestLogin)},
		rest.Route{"DELETE", "/login", RestLogout},
		// "Misc" stuff
		rest.Route{"GET", "/top/services", this.AuthorizedClient(RestGetTopServices)},
		rest.Route{"GET", "/running", this.AuthorizedClient(RestGetAllRunning)},
		// Generic static data
		rest.Route{"GET", "/favicon.ico", FavIcon},
		rest.Route{"GET", "/static*resource", StaticData},
	}

	// Hardcoding these target URLs for now.
	// TODO: When internal services are allowed to run on other hosts, look that up.
	routes = routeToInternalServiceProxy("/elastic", "http://127.0.0.1:9200/", routes)
	routes = routeToInternalServiceProxy("/metrics", "http://127.0.0.1:8888/", routes)

	handler.SetRoutes(routes...)

	http.ListenAndServe(":7878", &handler)
}

var methods []string = []string{"GET", "POST", "PUT", "DELETE"}

func routeToInternalServiceProxy(path string, target string, routes []rest.Route) []rest.Route {
	targetUrl, err := url.Parse(target)
	if err != nil {
		glog.Errorf("Unable to parse proxy target URL: %s", target)
		return routes
	}
	// Wrap the normal http.Handler in a rest.HandlerFunc
	handlerFunc := func(w *rest.ResponseWriter, r *rest.Request) {
		proxy := serviced.NewReverseProxy(path, targetUrl)
		proxy.ServeHTTP(w.ResponseWriter, r.Request)
	}
	// Add on a glob to match subpaths
	andsubpath := path + "*x"
	for _, method := range methods {
		routes = append(routes, rest.Route{method, path, handlerFunc})
		routes = append(routes, rest.Route{method, andsubpath, handlerFunc})
	}
	return routes
}

func (this *ServiceConfig) UnAuthorizedClient(realfunc HandlerClientFunc) HandlerFunc {
	return func(w *rest.ResponseWriter, r *rest.Request) {
		client, err := this.getClient()
		if err != nil {
			glog.Errorf("Unable to acquire client: %v", err)
			RestServerError(w)
			return
		}
		defer client.Close()
		realfunc(w, r, client)
	}
}

func (this *ServiceConfig) AuthorizedClient(realfunc HandlerClientFunc) HandlerFunc {
	return func(w *rest.ResponseWriter, r *rest.Request) {
		if !LoginOk(r) {
			RestUnauthorized(w)
			return
		}
		client, err := this.getClient()
		if err != nil {
			glog.Errorf("Unable to acquire client: %v", err)
			RestServerError(w)
			return
		}
		defer client.Close()
		realfunc(w, r, client)
	}
}

func (this *ServiceConfig) IsCollectingStats() HandlerFunc {
	if this.stats {
		return func(w *rest.ResponseWriter, r *rest.Request) {
			w.WriteHeader(http.StatusOK)
		}
	} else {
		return func(w *rest.ResponseWriter, r *rest.Request) {
			w.WriteHeader(http.StatusNotImplemented)
		}
	}
}

func (this *ServiceConfig) getClient() (c *serviced.ControlClient, err error) {
	// setup the client
	c, err = serviced.NewControlClient(this.agentPort)
	if err != nil {
		glog.Fatalf("Could not create a control plane client: %v", err)
	}
	return c, err
}
<|endoftext|>


<commit_before>// webhog is a package that stores and downloads a
// given URL (including js, css, and images) for offline
// use and uploads it to a given AWS-S3 account.
package webhog

import (
	"bytes"
	"code.google.com/p/go.net/html"
	uuid "github.com/nu7hatch/gouuid"
	"log"
	"net/http"
	"regexp"
	"sync"
)

var addCounter = 0
var doneCounter = 0

// Regex to match css and js extentions.
var rxExt = regexp.MustCompile(`(\.(?:css|js|gif|png|jpg))\/?$`)

// HTML nodes and their respective attrs that we need.
var matchVals = map[string]string{"link": "href", "script": "src", "img": "src"}

// Start the scraping process.
func NewScraper(url string) (e *Entity, err error) {
	entity := new(Entity)

	// Return existing entity if it exists.
	e, exists := checkExistingEntity(url, entity)
	if exists {
		return e, nil
	}

	// Create a new entity.
	e, err = createNewEntity(url, entity)

	return e, err
}

// Make a GET request to the given URL and start parsing
// its HTML.
func ExtractData(entity *Entity, url string) {
	// Parsing completion channel.
	done := make(chan bool, 1)

	res, err := http.Get(url)
	if err != nil {
		log.Println("Error requesting URL data: ", err)
	}

	defer res.Body.Close()

	doc, err := html.Parse(res.Body)
	if err != nil {
		log.Println("Error parsing URL body: ", err)
	}

	go ParseHTML(doc, entity, done)

	for {
		select {
		case <-done:
			var finalHTML bytes.Buffer
			bl := html.Render(&finalHTML, doc)
			if bl != nil {
				log.Println(bl)
			}

			err := StoreHTML(finalHTML, EntityDir)
			if err != nil {
				log.Println("Error in StoreHTML: ", err)
			}
		default:
		}
	}
}

// Parse the HTML - pull the href/src attributes for js,
// css, and images for download.
func ParseHTML(n *html.Node, entity *Entity, done chan bool) {
	var wg sync.WaitGroup

	wg.Add(1)
	go extractAttrs(n, entity, &wg)
	wg.Wait()
	done <- true
}

func extractAttrs(n *html.Node, entity *Entity, wg *sync.WaitGroup) {
	defer wg.Done()

	// loop through file types and their extensions to check for matches
	for j, p := range matchVals {
		if n.Type == html.ElementNode && n.Data == j {
			for i := range n.Attr {
				attr := &n.Attr[i]
				if attr.Key == p {
					// match css and js extentions
					match := rxExt.FindStringSubmatch(attr.Val)

					// check if a full URL is given to the entity, otherwise
					// we need to create one for the resource.
					matched, err := regexp.MatchString("http.*", attr.Val)
					if err != nil {
						log.Println("Error matching regex http(s): ", err)
					}

					if len(match) > 0 && matched {
						// Download and persist the current resource and insert it's
						// new name in the value of the HTML tree.
						name, err := StoreResource(attr.Val, string(match[0]), EntityDir)
						if err != nil {
							log.Println("Error storing resource: ", err)
						}
						attr.Val = name
					}
				}
			}
		}
	}

	for c := n.FirstChild; c != nil; c = c.NextSibling {
		wg.Add(1)
		go extractAttrs(c, entity, wg)
	}
}

// See if this URL has already been saved into the
// database - if so, return it.
func checkExistingEntity(url string, e *Entity) (entity *Entity, exists bool) {
	exists = false

	en := e.Find(url)
	if en.UUID != "" {
		exists = true
	}

	return en, exists
}

// Create a new entity to persist into the database - start HTML extraction.
func createNewEntity(url string, entity *Entity) (e *Entity, err error) {
	err = NewEntityDir()
	if err != nil {
		log.Println("Error creating entity dir: ", err)
	}

	go ExtractData(entity, url)

	id, err := uuid.NewV4()
	if err != nil {
		log.Println("Error creating UUID: ", err)
	}

	// Set new entities' initial data from the request.
	entity.Status = ParsingStatus
	entity.Url = url
	entity.UUID = id.String()

	// Persist new entity into the database.
	entity = entity.Create()

	return entity, err
}
<commit_msg>url support for the inclusion or removal of trailing slash<commit_after>// webhog is a package that stores and downloads a
// given URL (including js, css, and images) for offline
// use and uploads it to a given AWS-S3 account.
package webhog

import (
	"bytes"
	"code.google.com/p/go.net/html"
	uuid "github.com/nu7hatch/gouuid"
	"log"
	"net/http"
	"regexp"
	"sync"
)

var addCounter = 0
var doneCounter = 0

// Regex to match css and js extentions.
var rxExt = regexp.MustCompile(`(\.(?:css|js|gif|png|jpg))\/?$`)

// HTML nodes and their respective attrs that we need.
var matchVals = map[string]string{"link": "href", "script": "src", "img": "src"}

// Start the scraping process.
func NewScraper(url string) (e *Entity, err error) {
	entity := new(Entity)

	// Return existing entity if it exists.
	e, exists := checkExistingEntity(url, entity)
	if exists {
		return e, nil
	}

	// Create a new entity.
	e, err = createNewEntity(url, entity)

	return e, err
}

// Make a GET request to the given URL and start parsing
// its HTML.
func ExtractData(entity *Entity, url string) {
	// Parsing completion channel.
	done := make(chan bool, 1)

	res, err := http.Get(url)
	if err != nil {
		log.Println("Error requesting URL data: ", err)
	}

	defer res.Body.Close()

	doc, err := html.Parse(res.Body)
	if err != nil {
		log.Println("Error parsing URL body: ", err)
	}

	go ParseHTML(doc, entity, done)

	for {
		select {
		case <-done:
			var finalHTML bytes.Buffer
			bl := html.Render(&finalHTML, doc)
			if bl != nil {
				log.Println(bl)
			}

			err := StoreHTML(finalHTML, EntityDir)
			if err != nil {
				log.Println("Error in StoreHTML: ", err)
			}
		default:
		}
	}
}

// Parse the HTML - pull the href/src attributes for js,
// css, and images for download.
func ParseHTML(n *html.Node, entity *Entity, done chan bool) {
	var wg sync.WaitGroup

	wg.Add(1)
	go extractAttrs(n, entity, &wg)
	wg.Wait()
	done <- true
}

func extractAttrs(n *html.Node, entity *Entity, wg *sync.WaitGroup) {
	defer wg.Done()

	// loop through file types and their extensions to check for matches
	for j, p := range matchVals {
		if n.Type == html.ElementNode && n.Data == j {
			for i := range n.Attr {
				attr := &n.Attr[i]
				if attr.Key == p {
					matchAttrs(attr, entity)
				}
			}
		}
	}

	for c := n.FirstChild; c != nil; c = c.NextSibling {
		wg.Add(1)
		go extractAttrs(c, entity, wg)
	}
}

// match found attr file types and check src/href for a
// fullpath URL
func matchAttrs(attr *html.Attribute, entity *Entity) {
	// match css and js extentions
	match := rxExt.FindStringSubmatch(attr.Val)

	// check if a full URL is given to the entity, otherwise
	// we need to create one for the resource.
	matched, err := regexp.MatchString("http.*", attr.Val)
	if err != nil {
		log.Println("Error matching regex http(s): ", err)
	}

	// valid filetype and there is a full URL we can use
	if len(match) > 0 && matched {
		// Download and persist the current resource and insert it's
		// new name in the value of the HTML tree.
		name, err := StoreResource(attr.Val, string(match[0]), EntityDir)
		if err != nil {
			log.Println("Error storing resource: ", err)
		}
		attr.Val = name
	}

	// valid filetype but there is a relative URL
	if len(match) > 0 && !matched {
		// new resource name after adding in full URL
		var updName string
		// check for trailing slash on the entities' URL
		ln := len(entity.Url)
		if string(entity.Url[ln-1]) == "/" {
			updName = entity.Url + attr.Val
		} else {
			updName = entity.Url + "/" + attr.Val
		}

		name, err := StoreResource(updName, string(match[0]), EntityDir)
		if err != nil {
			log.Println("Error storing resource: ", err)
		}
		attr.Val = name
	}
}

// See if this URL has already been saved into the
// database - if so, return it.
func checkExistingEntity(url string, e *Entity) (entity *Entity, exists bool) {
	exists = false

	en := e.Find(url)
	if en.UUID != "" {
		exists = true
	}

	return en, exists
}

// Create a new entity to persist into the database - start HTML extraction.
func createNewEntity(url string, entity *Entity) (e *Entity, err error) {
	err = NewEntityDir()
	if err != nil {
		log.Println("Error creating entity dir: ", err)
	}

	go ExtractData(entity, url)

	id, err := uuid.NewV4()
	if err != nil {
		log.Println("Error creating UUID: ", err)
	}

	// Set new entities' initial data from the request.
	entity.Status = ParsingStatus
	entity.Url = url
	entity.UUID = id.String()

	// Persist new entity into the database.
	entity = entity.Create()

	return entity, err
}
<|endoftext|>


<commit_before>package whisper

import (
	"fmt"
	"time"

	"github.com/ethereum/go-ethereum/p2p"
	"github.com/ethereum/go-ethereum/rlp"
	"gopkg.in/fatih/set.v0"
)

const (
	protocolVersion = 0x02
)

type peer struct {
	host *Whisper
	peer *p2p.Peer
	ws   p2p.MsgReadWriter

	// XXX Eventually this is going to reach exceptional large space. We need an expiry here
	known *set.Set

	quit chan struct{}
}

func NewPeer(host *Whisper, p *p2p.Peer, ws p2p.MsgReadWriter) *peer {
	return &peer{host, p, ws, set.New(), make(chan struct{})}
}

func (self *peer) init() error {
	if err := self.handleStatus(); err != nil {
		return err
	}

	return nil
}

func (self *peer) start() {
	go self.update()
	self.peer.Infoln("whisper started")
}

func (self *peer) stop() {
	self.peer.Infoln("whisper stopped")

	close(self.quit)
}

func (self *peer) update() {
	relay := time.NewTicker(300 * time.Millisecond)
out:
	for {
		select {
		case <-relay.C:
			err := self.broadcast(self.host.envelopes())
			if err != nil {
				self.peer.Infoln("broadcast err:", err)
				break out
			}

		case <-self.quit:
			break out
		}
	}
}

func (self *peer) broadcast(envelopes []*Envelope) error {
	envs := make([]interface{}, len(envelopes))
	i := 0
	for _, envelope := range envelopes {
		if !self.known.Has(envelope.Hash()) {
			envs[i] = envelope
			self.known.Add(envelope.Hash())
			i++
		}
	}

	if i > 0 {
		if err := p2p.EncodeMsg(self.ws, envelopesMsg, envs[:i]...); err != nil {
			return err
		}
		self.peer.DebugDetailln("broadcasted", i, "message(s)")
	}

	return nil
}

func (self *peer) addKnown(envelope *Envelope) {
	self.known.Add(envelope.Hash())
}

func (self *peer) handleStatus() error {
	ws := self.ws
	if err := ws.WriteMsg(self.statusMsg()); err != nil {
		return err
	}
	msg, err := ws.ReadMsg()
	if err != nil {
		return err
	}
	if msg.Code != statusMsg {
		return fmt.Errorf("peer send %x before status msg", msg.Code)
	}
	s := rlp.NewStream(msg.Payload)
	if _, err := s.List(); err != nil {
		return fmt.Errorf("bad status message: %v", err)
	}
	pv, err := s.Uint()
	if err != nil {
		return fmt.Errorf("bad status message: %v", err)
	}
	if pv != protocolVersion {
		return fmt.Errorf("protocol version mismatch %d != %d", pv, protocolVersion)
	}
	return msg.Discard() // ignore anything after protocol version
}

func (self *peer) statusMsg() p2p.Msg {
	return p2p.NewMsg(statusMsg, protocolVersion)
}
<commit_msg>don't log whisper start ups<commit_after>package whisper

import (
	"fmt"
	"time"

	"github.com/ethereum/go-ethereum/p2p"
	"github.com/ethereum/go-ethereum/rlp"
	"gopkg.in/fatih/set.v0"
)

const (
	protocolVersion = 0x02
)

type peer struct {
	host *Whisper
	peer *p2p.Peer
	ws   p2p.MsgReadWriter

	// XXX Eventually this is going to reach exceptional large space. We need an expiry here
	known *set.Set

	quit chan struct{}
}

func NewPeer(host *Whisper, p *p2p.Peer, ws p2p.MsgReadWriter) *peer {
	return &peer{host, p, ws, set.New(), make(chan struct{})}
}

func (self *peer) init() error {
	if err := self.handleStatus(); err != nil {
		return err
	}

	return nil
}

func (self *peer) start() {
	go self.update()
	self.peer.Debugln("whisper started")
}

func (self *peer) stop() {
	self.peer.Debugln("whisper stopped")

	close(self.quit)
}

func (self *peer) update() {
	relay := time.NewTicker(300 * time.Millisecond)
out:
	for {
		select {
		case <-relay.C:
			err := self.broadcast(self.host.envelopes())
			if err != nil {
				self.peer.Infoln("broadcast err:", err)
				break out
			}

		case <-self.quit:
			break out
		}
	}
}

func (self *peer) broadcast(envelopes []*Envelope) error {
	envs := make([]interface{}, len(envelopes))
	i := 0
	for _, envelope := range envelopes {
		if !self.known.Has(envelope.Hash()) {
			envs[i] = envelope
			self.known.Add(envelope.Hash())
			i++
		}
	}

	if i > 0 {
		if err := p2p.EncodeMsg(self.ws, envelopesMsg, envs[:i]...); err != nil {
			return err
		}
		self.peer.DebugDetailln("broadcasted", i, "message(s)")
	}

	return nil
}

func (self *peer) addKnown(envelope *Envelope) {
	self.known.Add(envelope.Hash())
}

func (self *peer) handleStatus() error {
	ws := self.ws
	if err := ws.WriteMsg(self.statusMsg()); err != nil {
		return err
	}
	msg, err := ws.ReadMsg()
	if err != nil {
		return err
	}
	if msg.Code != statusMsg {
		return fmt.Errorf("peer send %x before status msg", msg.Code)
	}
	s := rlp.NewStream(msg.Payload)
	if _, err := s.List(); err != nil {
		return fmt.Errorf("bad status message: %v", err)
	}
	pv, err := s.Uint()
	if err != nil {
		return fmt.Errorf("bad status message: %v", err)
	}
	if pv != protocolVersion {
		return fmt.Errorf("protocol version mismatch %d != %d", pv, protocolVersion)
	}
	return msg.Discard() // ignore anything after protocol version
}

func (self *peer) statusMsg() p2p.Msg {
	return p2p.NewMsg(statusMsg, protocolVersion)
}
<|endoftext|>


<commit_before>// Implementation (work in progress) of the resequencing analysis pipeline used
// to teach the introductory NGS bioinformatics analysis course at SciLifeLab
// as described on this page:
// http://uppnex.se/twiki/do/view/Courses/NgsIntro1502/ResequencingAnalysis.html
//
// Prerequisites: Samtools, BWA, Picard, GATK.  You can install all tools
// except GATK on X/L/K/Ubuntu, with this command:
//
//   sudo apt-get install samtools bwa picard-tools
//
// GATK needs to be downloaded and installed manually from:
// http://www.broadinstitute.org/gatk
package main

import (
	"fmt"

	sp "github.com/scipipe/scipipe"
)

// ------------------------------------------------------------------------------------
// Set up static stuff like paths
// ------------------------------------------------------------------------------------
const (
	fastqBaseUrl = "http://bioinfo.perdanauniversity.edu.my/tein4ngs/ngspractice/"
	fastQFilePtn = "%s.ILLUMINA.low_coverage.4p_%s.fq"
	refBaseUrl   = "http://ftp.ensembl.org/pub/release-75/fasta/homo_sapiens/dna/"
	refFile      = "Homo_sapiens.GRCh37.75.dna.chromosome.17.fa"
	refFileGz    = "Homo_sapiens.GRCh37.75.dna.chromosome.17.fa.gz"
	vcfBaseUrl   = "http://ftp.1000genomes.ebi.ac.uk/vol1/ftp/phase1/analysis_results/integrated_call_sets/"
	vcfFile      = "ALL.chr17.integrated_phase1_v3.20101123.snps_indels_svs.genotypes.vcf.gz"
)

// ------------------------------------------------------------------------------------
// Set up the main input parameters to the workflow
// ------------------------------------------------------------------------------------
var (
	individuals = []string{"NA06984", "NA12489"}
	samples     = []string{"1", "2"}
)

func main() {
	// --------------------------------------------------------------------------------
	// Initialize Workflow
	// --------------------------------------------------------------------------------
	wf := sp.NewWorkflow("resequencing_workflow", 4)

	// --------------------------------------------------------------------------------
	// Download Reference Genome
	// --------------------------------------------------------------------------------
	dlRef := wf.NewProc("download_ref", "wget -O {o:outfile} "+refBaseUrl+refFileGz)
	dlRef.SetOut("outfile", refFileGz)

	// --------------------------------------------------------------------------------
	// Unzip ref file
	// --------------------------------------------------------------------------------
	ungzipRef := wf.NewProc("ugzip_ref", "gunzip -c {i:in} > {o:out}")
	ungzipRef.In("in").From(dlRef.Out("outfile"))
	ungzipRef.SetOut("out", "{i:in|%.gz}")

	// --------------------------------------------------------------------------------
	// Index Reference Genome
	// --------------------------------------------------------------------------------
	indexRef := wf.NewProc("index_ref", "bwa index -a bwtsw {i:index}; echo done > {o:done}")
	indexRef.In("index").From(ungzipRef.Out("out"))
	indexRef.SetOut("done", "{i:index}.indexed")

	// Create (multi-level) maps where we can gather outports from processes
	// for each for loop iteration and access them in the merge step later
	outs := map[string]map[string]map[string]*sp.OutPort{}
	for _, ind := range individuals {
		outs[ind] = map[string]map[string]*sp.OutPort{}
		for _, spl := range samples {
			outs[ind][spl] = map[string]*sp.OutPort{}
			indSpl := "_" + ind + "_" + spl

			// ------------------------------------------------------------------------
			// Download FastQ component
			// ------------------------------------------------------------------------
			fastqFile := fmt.Sprintf(fastQFilePtn, ind, spl)
			dlFastq := wf.NewProc("download_fastq"+indSpl, "wget -O {o:fastq} "+fastqBaseUrl+fastqFile)
			dlFastq.SetOut("fastq", fastqFile)

			// Save outPorts for later use
			outs[ind][spl]["fastq"] = dlFastq.Out("fastq")

			// ------------------------------------------------------------------------
			// BWA Align
			// ------------------------------------------------------------------------
			align := wf.NewProc("bwa_aln"+indSpl, "bwa aln {i:ref} {i:fastq} > {o:sai} # {i:idxdone}")
			align.In("ref").From(ungzipRef.Out("out"))
			align.In("idxdone").From(indexRef.Out("done"))
			align.In("fastq").From(dlFastq.Out("fastq"))
			align.SetOut("sai", "{i:fastq}.sai")

			// Save outPorts for later use
			outs[ind][spl]["sai"] = align.Out("sai")
		}

		// ---------------------------------------------------------------------------
		// Merge
		// ---------------------------------------------------------------------------
		merge := wf.NewProc("merge_"+ind, "bwa sampe {i:ref} {i:sai1} {i:sai2} {i:fq1} {i:fq2} > {o:merged} # {i:refdone} {p:ind}")
		merge.InParam("ind").FromStr(ind)
		merge.In("ref").From(ungzipRef.Out("out"))
		merge.In("refdone").From(indexRef.Out("done"))
		merge.In("sai1").From(outs[ind]["1"]["sai"])
		merge.In("sai2").From(outs[ind]["2"]["sai"])
		merge.In("fq1").From(outs[ind]["1"]["fastq"])
		merge.In("fq2").From(outs[ind]["2"]["fastq"])
		merge.SetOut("merged", "{p:ind}.merged.sam")
	}
	// -------------------------------------------------------------------------------
	// Run Workflow
	// -------------------------------------------------------------------------------
	wf.Run()
}
<commit_msg>Use better naming for in-port in resequencing example<commit_after>// Implementation (work in progress) of the resequencing analysis pipeline used
// to teach the introductory NGS bioinformatics analysis course at SciLifeLab
// as described on this page:
// http://uppnex.se/twiki/do/view/Courses/NgsIntro1502/ResequencingAnalysis.html
//
// Prerequisites: Samtools, BWA, Picard, GATK.  You can install all tools
// except GATK on X/L/K/Ubuntu, with this command:
//
//   sudo apt-get install samtools bwa picard-tools
//
// GATK needs to be downloaded and installed manually from:
// http://www.broadinstitute.org/gatk
package main

import (
	"fmt"

	sp "github.com/scipipe/scipipe"
)

// ------------------------------------------------------------------------------------
// Set up static stuff like paths
// ------------------------------------------------------------------------------------
const (
	fastqBaseUrl = "http://bioinfo.perdanauniversity.edu.my/tein4ngs/ngspractice/"
	fastQFilePtn = "%s.ILLUMINA.low_coverage.4p_%s.fq"
	refBaseUrl   = "http://ftp.ensembl.org/pub/release-75/fasta/homo_sapiens/dna/"
	refFile      = "Homo_sapiens.GRCh37.75.dna.chromosome.17.fa"
	refFileGz    = "Homo_sapiens.GRCh37.75.dna.chromosome.17.fa.gz"
	vcfBaseUrl   = "http://ftp.1000genomes.ebi.ac.uk/vol1/ftp/phase1/analysis_results/integrated_call_sets/"
	vcfFile      = "ALL.chr17.integrated_phase1_v3.20101123.snps_indels_svs.genotypes.vcf.gz"
)

// ------------------------------------------------------------------------------------
// Set up the main input parameters to the workflow
// ------------------------------------------------------------------------------------
var (
	individuals = []string{"NA06984", "NA12489"}
	samples     = []string{"1", "2"}
)

func main() {
	// --------------------------------------------------------------------------------
	// Initialize Workflow
	// --------------------------------------------------------------------------------
	wf := sp.NewWorkflow("resequencing_workflow", 4)

	// --------------------------------------------------------------------------------
	// Download Reference Genome
	// --------------------------------------------------------------------------------
	dlRef := wf.NewProc("download_ref", "wget -O {o:outfile} "+refBaseUrl+refFileGz)
	dlRef.SetOut("outfile", refFileGz)

	// --------------------------------------------------------------------------------
	// Unzip ref file
	// --------------------------------------------------------------------------------
	ungzipRef := wf.NewProc("ugzip_ref", "gunzip -c {i:in} > {o:out}")
	ungzipRef.In("in").From(dlRef.Out("outfile"))
	ungzipRef.SetOut("out", "{i:in|%.gz}")

	// --------------------------------------------------------------------------------
	// Index Reference Genome
	// --------------------------------------------------------------------------------
	indexRef := wf.NewProc("index_ref", "bwa index -a bwtsw {i:ref}; echo done > {o:done}")
	indexRef.In("ref").From(ungzipRef.Out("out"))
	indexRef.SetOut("done", "{i:ref}.indexed")

	// Create (multi-level) maps where we can gather outports from processes
	// for each for loop iteration and access them in the merge step later
	outs := map[string]map[string]map[string]*sp.OutPort{}
	for _, ind := range individuals {
		outs[ind] = map[string]map[string]*sp.OutPort{}
		for _, spl := range samples {
			outs[ind][spl] = map[string]*sp.OutPort{}
			indSpl := "_" + ind + "_" + spl

			// ------------------------------------------------------------------------
			// Download FastQ component
			// ------------------------------------------------------------------------
			fastqFile := fmt.Sprintf(fastQFilePtn, ind, spl)
			dlFastq := wf.NewProc("download_fastq"+indSpl, "wget -O {o:fastq} "+fastqBaseUrl+fastqFile)
			dlFastq.SetOut("fastq", fastqFile)

			// Save outPorts for later use
			outs[ind][spl]["fastq"] = dlFastq.Out("fastq")

			// ------------------------------------------------------------------------
			// BWA Align
			// ------------------------------------------------------------------------
			align := wf.NewProc("bwa_aln"+indSpl, "bwa aln {i:ref} {i:fastq} > {o:sai} # {i:idxdone}")
			align.In("ref").From(ungzipRef.Out("out"))
			align.In("idxdone").From(indexRef.Out("done"))
			align.In("fastq").From(dlFastq.Out("fastq"))
			align.SetOut("sai", "{i:fastq}.sai")

			// Save outPorts for later use
			outs[ind][spl]["sai"] = align.Out("sai")
		}

		// ---------------------------------------------------------------------------
		// Merge
		// ---------------------------------------------------------------------------
		merge := wf.NewProc("merge_"+ind, "bwa sampe {i:ref} {i:sai1} {i:sai2} {i:fq1} {i:fq2} > {o:merged} # {i:refdone} {p:ind}")
		merge.InParam("ind").FromStr(ind)
		merge.In("ref").From(ungzipRef.Out("out"))
		merge.In("refdone").From(indexRef.Out("done"))
		merge.In("sai1").From(outs[ind]["1"]["sai"])
		merge.In("sai2").From(outs[ind]["2"]["sai"])
		merge.In("fq1").From(outs[ind]["1"]["fastq"])
		merge.In("fq2").From(outs[ind]["2"]["fastq"])
		merge.SetOut("merged", "{p:ind}.merged.sam")
	}
	// -------------------------------------------------------------------------------
	// Run Workflow
	// -------------------------------------------------------------------------------
	wf.Run()
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor: Zack Mullaly zmullaly@mozilla.com [:zack]

package api

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"gopkg.in/gcfg.v1"
	"net/http"
	"path/filepath"
	"strings"
	"testing"
	"time"

	_ "github.com/lib/pq"
)

const migAPIPort uint16 = 12345

type testState struct {
	AgentID       float64
	QueueLocation string
	ActionID      float64
}

type operation struct {
	Module     string                 `json:"module"`
	Parameters map[string]interface{} `json:"parameters"`
}

type TestConfig struct {
	Postgres struct {
		Host     string
		User     string
		Password string
		DBName   string
		SSLMode  string
		Port     int
		MaxConn  int
	}
}

type action struct {
	Name          string      `json:"name"`
	Target        string      `json:"target"`
	ValidFrom     time.Time   `json:"validFrom"`
	ExpireAfter   time.Time   `json:"expireAfter"`
	Operations    []operation `json:"operations"`
	Signatures    []string    `json:"signatures"`
	Status        string      `json:"status"`
	SyntaxVersion uint        `json:"syntaxVersion"`
}

type listActionsResponse struct {
	Error   *string `json:"error"`
	Actions []action
}

func listActions(port uint16, queue string, limit uint) (int, listActionsResponse, error) {
	response, err := http.Get(fmt.Sprintf(
		"http://127.0.0.1:%d/api/v1/actions?queue=%s&limit=%d",
		port,
		queue,
		limit))
	if err != nil {
		return 0, listActionsResponse{}, err
	}

	respData := listActionsResponse{}
	decoder := json.NewDecoder(response.Body)
	decodeErr := decoder.Decode(&respData)

	response.Body.Close()

	if decodeErr != nil {
		return 0, listActionsResponse{}, decodeErr
	}

	return response.StatusCode, respData, nil
}

func TestListActionsWithValidRequest(t *testing.T) {
}

func setup() (testState, error) {
	return testState{}, nil
}

func teardown(state testState) error {
	return nil
}
<commit_msg>Finished writing an integration test for action retrieval<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor: Zack Mullaly zmullaly@mozilla.com [:zack]

package api

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"net/http"
	"path/filepath"
	"strings"
	"testing"
	"time"

	_ "github.com/lib/pq"
	"gopkg.in/gcfg.v1"

	"github.com/mozilla/mig"
	migdb "github.com/mozilla/mig/database"
)

const migAPIPort uint16 = 12345
const testActionName string = "testaction"

type config struct {
	Postgres struct {
		Host     string
		User     string
		Password string
		DBName   string
		SSLMode  string
		Port     int
		MaxConn  int
	}
}

type testState struct {
	AgentID  float64
	ActionID float64
}

type operation struct {
	Module     string                 `json:"module"`
	Parameters map[string]interface{} `json:"parameters"`
}

type TestConfig struct {
	Postgres struct {
		Host     string
		User     string
		Password string
		DBName   string
		SSLMode  string
		Port     int
		MaxConn  int
	}
}

type action struct {
	Name          string      `json:"name"`
	Target        string      `json:"target"`
	ValidFrom     time.Time   `json:"validFrom"`
	ExpireAfter   time.Time   `json:"expireAfter"`
	Operations    []operation `json:"operations"`
	Signatures    []string    `json:"signatures"`
	Status        string      `json:"status"`
	SyntaxVersion uint        `json:"syntaxVersion"`
}

type listActionsResponse struct {
	Error   *string `json:"error"`
	Actions []action
}

func listActions(port uint16, agent AgentID) (int, listActionsResponse, error) {
	response, err := http.Get(fmt.Sprintf(
		"http://127.0.0.1:%d/api/v1/actions?agent=%f",
		port,
		agent))
	if err != nil {
		return 0, listActionsResponse{}, err
	}

	respData := listActionsResponse{}
	decoder := json.NewDecoder(response.Body)
	decodeErr := decoder.Decode(&respData)

	response.Body.Close()

	if decodeErr != nil {
		return 0, listActionsResponse{}, decodeErr
	}

	return response.StatusCode, respData, nil
}

func TestListActionsWithValidRequest(t *testing.T) {
	cfg := config{}
	path, err := filepath.Abs("../../api.cfg")
	if err != nil {
		t.Fatal(err)
	}
	if err := gcfg.FatalOnly(gcfg.ReadFileInto(&cfg, path)); err != nil {
		t.Fatal(err)
	}
	url := fmt.Sprintf(
		"postgres://%s:%s@%s:%d/%s?sslmode=%s",
		cfg.Postgres.User,
		cfg.Postgres.Password,
		cfg.Postgres.Host,
		cfg.Postgres.Port,
		cfg.Postgres.DBName,
		cfg.Postgres.SSLMode)
	db, err := sql.Open("postgres", url)
	if err != nil {
		t.Fatal(err)
	}
	defer db.Close()

	state, err := setup(db)
	if err != nil {
		t.Fatal(err)
	}
	defer teardown(db, state)

	statusCode, actions, err := listactions(migAPIPort, AgentID(state.Agent))
	if err != nil {
		t.Fatal(err)
	}

	if statusCode != http.StatusOK {
		t.Errorf("Expected to get status code %d but got %d", http.StatusOK, statusCode)
	}

	if len(actions) != 1 {
		t.Errorf("Expected to get one action, but got %d", len(actions))
	}

	if actions[0].Name != testActionName {
		t.Errorf("Expected action retrieved to have name '%s' but it is '%s'", testActionName, actions[0].Name)
	}
}

func setup(db *migdb.DB) (testState, error) {
	testAgent := mig.Agent{
		ID:       mig.GenID(),
		Name:     "testagent",
		QueueLoc: "doesntmatter",
		Mode:     "daemon",
	}

	testAction := mig.Action{
		ID:          mig.GenID(),
		Name:        testActionName,
		Target:      "name = 'testagent'",
		ValidFrom:   time.Now(),
		ExpireAfter: time.Now().Add(5 * time.Minute),
		Status:      "pending",
	}

	err := db.InsertAgent(testAgent, nil)
	if err != nil {
		return testState{}, err
	}

	err = db.InsertAction(testaction)
	if err != nil {
		return testState{}, err
	}

	state = testState{
		AgentID:  testAgent.ID,
		ActionID: testAction.ID,
	}
	return state, nil
}

func teardown(db *migdb.DB, state testState) error {
	return nil
}
<|endoftext|>


<commit_before>package mysql

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestPrepareExecer(t *testing.T) {
	db, err := Open("root", "", "fizzbuzz_twitterbot_test")
	assert.NoError(t, err)

	pe := prepareExecer{db: db}

	res, err := pe.Exec("prepare error")
	assert.Error(t, err)
	assert.Nil(t, res)
}
<commit_msg>Fix mysql test.<commit_after>package mysql

import (
	"testing"

	"github.com/178inaba/fizzbuzz-twitterbot/model"
	sq "github.com/Masterminds/squirrel"
	"github.com/stretchr/testify/assert"
)

func TestPrepareExecer(t *testing.T) {
	db, err := Open("root", "", "fizzbuzz_twitterbot_test")
	assert.NoError(t, err)

	pe := prepareExecer{db: db}

	res, err := pe.Exec("prepare error")
	assert.Error(t, err)
	assert.Nil(t, res)

	sql, _, err := sq.Insert(model.FizzbuzzTweetTableName).
		Columns("number").Values(struct{}{}).ToSql()
	assert.NoError(t, err)

	res, err = pe.Exec(sql)
	assert.Error(t, err)
	assert.Nil(t, res)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"strings"

	"github.com/golang/glog"
	"github.com/kubernetes-incubator/external-storage/lib/controller"
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/kubernetes/pkg/apis/core/v1/helper"
)

const (
	provisionerName    = "ceph.com/cephfs"
	provisionCmd       = "/usr/local/bin/cephfs_provisioner"
	provisionerIDAnn   = "cephFSProvisionerIdentity"
	cephShareAnn       = "cephShare"
	provisionerNameKey = "PROVISIONER_NAME"
	secretNamespaceKey = "PROVISIONER_SECRET_NAMESPACE"
)

type provisionOutput struct {
	Path   string `json:"path"`
	User   string `json:"user"`
	Secret string `json:"auth"`
}

type cephFSProvisioner struct {
	// Kubernetes Client. Use to retrieve Ceph admin secret
	client kubernetes.Interface
	// Identity of this cephFSProvisioner, generated. Used to identify "this"
	// provisioner's PVs.
	identity string
	// Namespace secrets will be created in. If empty, secrets will be created in each PVC's namespace.
	secretNamespace string
}

func newCephFSProvisioner(client kubernetes.Interface, id string, secretNamespace string) controller.Provisioner {
	return &cephFSProvisioner{
		client:          client,
		identity:        id,
		secretNamespace: secretNamespace,
	}
}

var _ controller.Provisioner = &cephFSProvisioner{}

func generateSecretName(user string) string {
	return "ceph-" + user + "-secret"
}

func getClaimRefNamespace(pv *v1.PersistentVolume) string {
	if pv.Spec.ClaimRef != nil {
		return pv.Spec.ClaimRef.Namespace
	}
	return ""
}

// getSecretFromCephFSPersistentVolume gets secret reference from CephFS PersistentVolume.
// It fallbacks to use ClaimRef.Namespace if SecretRef.Namespace is
// empty. See https://github.com/kubernetes/kubernetes/pull/49502.
func getSecretFromCephFSPersistentVolume(pv *v1.PersistentVolume) (*v1.SecretReference, error) {
	source := &pv.Spec.PersistentVolumeSource
	if source.CephFS == nil {
		return nil, errors.New("pv.Spec.PersistentVolumeSource.CephFS is nil")
	}
	if source.CephFS.SecretRef == nil {
		return nil, errors.New("pv.Spec.PersistentVolumeSource.CephFS.SecretRef is nil")
	}
	if len(source.CephFS.SecretRef.Namespace) > 0 {
		return source.CephFS.SecretRef, nil
	}
	ns := getClaimRefNamespace(pv)
	if len(ns) <= 0 {
		return nil, errors.New("both pv.Spec.SecretRef.Namespace and pv.Spec.ClaimRef.Namespace are empty")
	}
	return &v1.SecretReference{
		Name:      source.CephFS.SecretRef.Name,
		Namespace: ns,
	}, nil
}

// Provision creates a storage asset and returns a PV object representing it.
func (p *cephFSProvisioner) Provision(options controller.VolumeOptions) (*v1.PersistentVolume, error) {
	if options.PVC.Spec.Selector != nil {
		return nil, fmt.Errorf("claim Selector is not supported")
	}
	cluster, adminID, adminSecret, pvcRoot, mon, err := p.parseParameters(options.Parameters)
	if err != nil {
		return nil, err
	}
	// create random share name
	share := fmt.Sprintf("kubernetes-dynamic-pvc-%s", uuid.NewUUID())
	// create random user id
	user := fmt.Sprintf("kubernetes-dynamic-user-%s", uuid.NewUUID())
	// provision share
	// create cmd
	cmd := exec.Command(provisionCmd, "-n", share, "-u", user)
	// set env
	cmd.Env = []string{
		"CEPH_CLUSTER_NAME=" + cluster,
		"CEPH_MON=" + strings.Join(mon[:], ","),
		"CEPH_AUTH_ID=" + adminID,
		"CEPH_AUTH_KEY=" + adminSecret,
		"CEPH_VOLUME_ROOT=" + pvcRoot}

	output, cmdErr := cmd.CombinedOutput()
	if cmdErr != nil {
		glog.Errorf("failed to provision share %q for %q, err: %v, output: %v", share, user, cmdErr, string(output))
		return nil, cmdErr
	}
	// validate output
	res := &provisionOutput{}
	json.Unmarshal([]byte(output), &res)
	if res.User == "" || res.Secret == "" || res.Path == "" {
		return nil, fmt.Errorf("invalid provisioner output")
	}
	nameSpace := p.secretNamespace
	if nameSpace == "" {
		// if empty, create secret in PVC's namespace
		nameSpace = options.PVC.Namespace
	}
	secretName := generateSecretName(user)
	secret := &v1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: nameSpace,
			Name:      secretName,
		},
		Data: map[string][]byte{
			"key": []byte(res.Secret),
		},
		Type: "Opaque",
	}

	_, err = p.client.CoreV1().Secrets(nameSpace).Create(secret)
	if err != nil {
		glog.Errorf("Cephfs Provisioner: create volume failed, err: %v", err)
		return nil, fmt.Errorf("failed to create secret")
	}

	pv := &v1.PersistentVolume{
		ObjectMeta: metav1.ObjectMeta{
			Name: options.PVName,
			Annotations: map[string]string{
				provisionerIDAnn: p.identity,
				cephShareAnn:     share,
			},
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeReclaimPolicy: options.PersistentVolumeReclaimPolicy,
			AccessModes:                   options.PVC.Spec.AccessModes,
			Capacity: v1.ResourceList{ //FIXME: kernel cephfs doesn't enforce quota, capacity is not meaningless here.
				v1.ResourceName(v1.ResourceStorage): options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)],
			},
			PersistentVolumeSource: v1.PersistentVolumeSource{
				CephFS: &v1.CephFSPersistentVolumeSource{
					Monitors: mon,
					Path:     res.Path[strings.Index(res.Path, "/"):],
					SecretRef: &v1.SecretReference{
						Name: secretName,
						// TODO https://github.com/kubernetes-incubator/external-storage/issues/309
						Namespace: nameSpace,
					},
					User: user,
				},
			},
		},
	}

	glog.Infof("successfully created CephFS share %+v", pv.Spec.PersistentVolumeSource.CephFS)

	return pv, nil
}

// Delete removes the storage asset that was created by Provision represented
// by the given PV.
func (p *cephFSProvisioner) Delete(volume *v1.PersistentVolume) error {
	ann, ok := volume.Annotations[provisionerIDAnn]
	if !ok {
		return errors.New("identity annotation not found on PV")
	}
	if ann != p.identity {
		return &controller.IgnoredError{Reason: "identity annotation on PV does not match ours"}
	}
	share, ok := volume.Annotations[cephShareAnn]
	if !ok {
		return errors.New("ceph share annotation not found on PV")
	}
	// delete CephFS
	// TODO when beta is removed, have to check kube version and pick v1/beta
	// accordingly: maybe the controller lib should offer a function for that
	class, err := p.client.StorageV1beta1().StorageClasses().Get(helper.GetPersistentVolumeClass(volume), metav1.GetOptions{})
	if err != nil {
		return err
	}
	cluster, adminID, adminSecret, pvcRoot, mon, err := p.parseParameters(class.Parameters)
	if err != nil {
		return err
	}
	user := volume.Spec.PersistentVolumeSource.CephFS.User
	// create cmd
	cmd := exec.Command(provisionCmd, "-r", "-n", share, "-u", user)
	// set env
	cmd.Env = []string{
		"CEPH_CLUSTER_NAME=" + cluster,
		"CEPH_MON=" + strings.Join(mon[:], ","),
		"CEPH_AUTH_ID=" + adminID,
		"CEPH_AUTH_KEY=" + adminSecret,
		"CEPH_VOLUME_ROOT=" + pvcRoot}

	output, cmdErr := cmd.CombinedOutput()
	if cmdErr != nil {
		glog.Errorf("failed to delete share %q for %q, err: %v, output: %v", share, user, cmdErr, string(output))
		return cmdErr
	}

	// Remove dynamic user secret
	secretRef, err := getSecretFromCephFSPersistentVolume(volume)
	if err != nil {
		glog.Errorf("failed to get secret references, err: %v", err)
		return err
	}
	err = p.client.CoreV1().Secrets(secretRef.Namespace).Delete(secretRef.Name, &metav1.DeleteOptions{})
	if err != nil {
		glog.Errorf("Cephfs Provisioner: delete secret failed, err: %v", err)
		return fmt.Errorf("failed to delete secret")
	}

	return nil
}

func (p *cephFSProvisioner) parseParameters(parameters map[string]string) (string, string, string, string, []string, error) {
	var (
		err                                                                           error
		mon                                                                           []string
		cluster, adminID, adminSecretName, adminSecretNamespace, adminSecret, pvcRoot string
	)

	adminSecretNamespace = "default"
	adminID = "admin"
	cluster = "ceph"
	pvcRoot = "/volumes/kubernetes"

	for k, v := range parameters {
		switch strings.ToLower(k) {
		case "cluster":
			cluster = v
		case "monitors":
			arr := strings.Split(v, ",")
			for _, m := range arr {
				mon = append(mon, m)
			}
		case "adminid":
			adminID = v
		case "adminsecretname":
			adminSecretName = v
		case "adminsecretnamespace":
			adminSecretNamespace = v
		case "claimroot":
			pvcRoot = v
		default:
			return "", "", "", "", nil, fmt.Errorf("invalid option %q", k)
		}
	}
	// sanity check
	if adminSecretName == "" {
		return "", "", "", "", nil, fmt.Errorf("missing Ceph admin secret name")
	}
	if adminSecret, err = p.parsePVSecret(adminSecretNamespace, adminSecretName); err != nil {
		return "", "", "", "", nil, fmt.Errorf("failed to get admin secret from [%q/%q]: %v", adminSecretNamespace, adminSecretName, err)
	}
	if len(mon) < 1 {
		return "", "", "", "", nil, fmt.Errorf("missing Ceph monitors")
	}
	return cluster, adminID, adminSecret, pvcRoot, mon, nil
}

func (p *cephFSProvisioner) parsePVSecret(namespace, secretName string) (string, error) {
	if p.client == nil {
		return "", fmt.Errorf("Cannot get kube client")
	}
	secrets, err := p.client.CoreV1().Secrets(namespace).Get(secretName, metav1.GetOptions{})
	if err != nil {
		return "", err
	}
	for _, data := range secrets.Data {
		return string(data), nil
	}

	// If not found, the last secret in the map wins as done before
	return "", fmt.Errorf("no secret found")
}

var (
	master          = flag.String("master", "", "Master URL")
	kubeconfig      = flag.String("kubeconfig", "", "Absolute path to the kubeconfig")
	id              = flag.String("id", "", "Unique provisioner identity")
	secretNamespace = flag.String("secret-namespace", "", "Namespace secrets will be created in (default: '', created in each PVC's namespace)")
)

func main() {
	flag.Parse()
	flag.Set("logtostderr", "true")

	var config *rest.Config
	var err error
	if *master != "" || *kubeconfig != "" {
		config, err = clientcmd.BuildConfigFromFlags(*master, *kubeconfig)
	} else {
		config, err = rest.InClusterConfig()
	}
	if err != nil {
		glog.Fatalf("Failed to create config: %v", err)
	}
	clientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		glog.Fatalf("Failed to create client: %v", err)
	}

	prName := provisionerName
	prNameFromEnv := os.Getenv(provisionerNameKey)
	if prNameFromEnv != "" {
		prName = prNameFromEnv
	}

	// By default, we use provisioner name as provisioner identity.
	// User may specify their own identity with `-id` flag to distinguish each
	// others, if they deploy more than one CephFS provisioners under same provisioner name.
	prID := prName
	if *id != "" {
		prID = *id
	}

	secretNamespaceFromEnv := os.Getenv(secretNamespaceKey)
	if secretNamespaceFromEnv != "" {
		*secretNamespace = secretNamespaceFromEnv
	}

	// The controller needs to know what the server version is because out-of-tree
	// provisioners aren't officially supported until 1.5
	serverVersion, err := clientset.Discovery().ServerVersion()
	if err != nil {
		glog.Fatalf("Error getting server version: %v", err)
	}

	// Create the provisioner: it implements the Provisioner interface expected by
	// the controller
	glog.Infof("Creating CephFS provisioner %s with identity: %s, secret namespace: %s", prName, prID, *secretNamespace)
	cephFSProvisioner := newCephFSProvisioner(clientset, prID, *secretNamespace)

	// Start the provision controller which will dynamically provision cephFS
	// PVs
	pc := controller.NewProvisionController(
		clientset,
		prName,
		cephFSProvisioner,
		serverVersion.GitVersion,
	)

	pc.Run(wait.NeverStop)
}
<commit_msg>ceph/cephfs/cephfs-provisioner.go:  Added handling for deterministicNames option; when set, the PVC directory name will be set to the PVC name (rather than a derivation of the randomly-generated PV name), and the Ceph user name will be set to the PVC name with "k8s." prepended to it (rather than, again, a derivation of the randomly-generated PV name)<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"strings"

	"github.com/golang/glog"
	"github.com/kubernetes-incubator/external-storage/lib/controller"
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/kubernetes/pkg/apis/core/v1/helper"
)

const (
	provisionerName    = "ceph.com/cephfs"
	provisionCmd       = "/usr/local/bin/cephfs_provisioner"
	provisionerIDAnn   = "cephFSProvisionerIdentity"
	cephShareAnn       = "cephShare"
	provisionerNameKey = "PROVISIONER_NAME"
	secretNamespaceKey = "PROVISIONER_SECRET_NAMESPACE"
)

type provisionOutput struct {
	Path   string `json:"path"`
	User   string `json:"user"`
	Secret string `json:"auth"`
}

type cephFSProvisioner struct {
	// Kubernetes Client. Use to retrieve Ceph admin secret
	client kubernetes.Interface
	// Identity of this cephFSProvisioner, generated. Used to identify "this"
	// provisioner's PVs.
	identity string
	// Namespace secrets will be created in. If empty, secrets will be created in each PVC's namespace.
	secretNamespace string
}

func newCephFSProvisioner(client kubernetes.Interface, id string, secretNamespace string) controller.Provisioner {
	return &cephFSProvisioner{
		client:          client,
		identity:        id,
		secretNamespace: secretNamespace,
	}
}

var _ controller.Provisioner = &cephFSProvisioner{}

func generateSecretName(user string) string {
	return "ceph-" + user + "-secret"
}

func getClaimRefNamespace(pv *v1.PersistentVolume) string {
	if pv.Spec.ClaimRef != nil {
		return pv.Spec.ClaimRef.Namespace
	}
	return ""
}

// getSecretFromCephFSPersistentVolume gets secret reference from CephFS PersistentVolume.
// It fallbacks to use ClaimRef.Namespace if SecretRef.Namespace is
// empty. See https://github.com/kubernetes/kubernetes/pull/49502.
func getSecretFromCephFSPersistentVolume(pv *v1.PersistentVolume) (*v1.SecretReference, error) {
	source := &pv.Spec.PersistentVolumeSource
	if source.CephFS == nil {
		return nil, errors.New("pv.Spec.PersistentVolumeSource.CephFS is nil")
	}
	if source.CephFS.SecretRef == nil {
		return nil, errors.New("pv.Spec.PersistentVolumeSource.CephFS.SecretRef is nil")
	}
	if len(source.CephFS.SecretRef.Namespace) > 0 {
		return source.CephFS.SecretRef, nil
	}
	ns := getClaimRefNamespace(pv)
	if len(ns) <= 0 {
		return nil, errors.New("both pv.Spec.SecretRef.Namespace and pv.Spec.ClaimRef.Namespace are empty")
	}
	return &v1.SecretReference{
		Name:      source.CephFS.SecretRef.Name,
		Namespace: ns,
	}, nil
}

// Provision creates a storage asset and returns a PV object representing it.
func (p *cephFSProvisioner) Provision(options controller.VolumeOptions) (*v1.PersistentVolume, error) {
	if options.PVC.Spec.Selector != nil {
		return nil, fmt.Errorf("claim Selector is not supported")
	}
	cluster, adminID, adminSecret, pvcRoot, mon, deterministicNames, err := p.parseParameters(options.Parameters)
	if err != nil {
		return nil, err
	}
	var share, user string
	if deterministicNames {
		share = fmt.Sprintf(options.PVC.Name)
		user = fmt.Sprintf("k8s.%s", options.PVC.Name)
	} else {
		// create random share name
		share = fmt.Sprintf("kubernetes-dynamic-pvc-%s", uuid.NewUUID())
		// create random user id
		user = fmt.Sprintf("kubernetes-dynamic-user-%s", uuid.NewUUID())
	}
	// provision share
	// create cmd
	cmd := exec.Command(provisionCmd, "-n", share, "-u", user)
	// set env
	cmd.Env = []string{
		"CEPH_CLUSTER_NAME=" + cluster,
		"CEPH_MON=" + strings.Join(mon[:], ","),
		"CEPH_AUTH_ID=" + adminID,
		"CEPH_AUTH_KEY=" + adminSecret,
		"CEPH_VOLUME_ROOT=" + pvcRoot}

	output, cmdErr := cmd.CombinedOutput()
	if cmdErr != nil {
		glog.Errorf("failed to provision share %q for %q, err: %v, output: %v", share, user, cmdErr, string(output))
		return nil, cmdErr
	}
	// validate output
	res := &provisionOutput{}
	json.Unmarshal([]byte(output), &res)
	if res.User == "" || res.Secret == "" || res.Path == "" {
		return nil, fmt.Errorf("invalid provisioner output")
	}
	nameSpace := p.secretNamespace
	if nameSpace == "" {
		// if empty, create secret in PVC's namespace
		nameSpace = options.PVC.Namespace
	}
	secretName := generateSecretName(user)
	secret := &v1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: nameSpace,
			Name:      secretName,
		},
		Data: map[string][]byte{
			"key": []byte(res.Secret),
		},
		Type: "Opaque",
	}

	_, err = p.client.CoreV1().Secrets(nameSpace).Create(secret)
	if err != nil {
		glog.Errorf("Cephfs Provisioner: create volume failed, err: %v", err)
		return nil, fmt.Errorf("failed to create secret")
	}

	pv := &v1.PersistentVolume{
		ObjectMeta: metav1.ObjectMeta{
			Name: options.PVName,
			Annotations: map[string]string{
				provisionerIDAnn: p.identity,
				cephShareAnn:     share,
			},
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeReclaimPolicy: options.PersistentVolumeReclaimPolicy,
			AccessModes:                   options.PVC.Spec.AccessModes,
			Capacity: v1.ResourceList{ //FIXME: kernel cephfs doesn't enforce quota, capacity is not meaningless here.
				v1.ResourceName(v1.ResourceStorage): options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)],
			},
			PersistentVolumeSource: v1.PersistentVolumeSource{
				CephFS: &v1.CephFSPersistentVolumeSource{
					Monitors: mon,
					Path:     res.Path[strings.Index(res.Path, "/"):],
					SecretRef: &v1.SecretReference{
						Name: secretName,
						// TODO https://github.com/kubernetes-incubator/external-storage/issues/309
						Namespace: nameSpace,
					},
					User: user,
				},
			},
		},
	}

	glog.Infof("successfully created CephFS share %+v", pv.Spec.PersistentVolumeSource.CephFS)

	return pv, nil
}

// Delete removes the storage asset that was created by Provision represented
// by the given PV.
func (p *cephFSProvisioner) Delete(volume *v1.PersistentVolume) error {
	ann, ok := volume.Annotations[provisionerIDAnn]
	if !ok {
		return errors.New("identity annotation not found on PV")
	}
	if ann != p.identity {
		return &controller.IgnoredError{Reason: "identity annotation on PV does not match ours"}
	}
	share, ok := volume.Annotations[cephShareAnn]
	if !ok {
		return errors.New("ceph share annotation not found on PV")
	}
	// delete CephFS
	// TODO when beta is removed, have to check kube version and pick v1/beta
	// accordingly: maybe the controller lib should offer a function for that
	class, err := p.client.StorageV1beta1().StorageClasses().Get(helper.GetPersistentVolumeClass(volume), metav1.GetOptions{})
	if err != nil {
		return err
	}
	cluster, adminID, adminSecret, pvcRoot, mon, _, err := p.parseParameters(class.Parameters)
	if err != nil {
		return err
	}
	user := volume.Spec.PersistentVolumeSource.CephFS.User
	// create cmd
	cmd := exec.Command(provisionCmd, "-r", "-n", share, "-u", user)
	// set env
	cmd.Env = []string{
		"CEPH_CLUSTER_NAME=" + cluster,
		"CEPH_MON=" + strings.Join(mon[:], ","),
		"CEPH_AUTH_ID=" + adminID,
		"CEPH_AUTH_KEY=" + adminSecret,
		"CEPH_VOLUME_ROOT=" + pvcRoot}

	output, cmdErr := cmd.CombinedOutput()
	if cmdErr != nil {
		glog.Errorf("failed to delete share %q for %q, err: %v, output: %v", share, user, cmdErr, string(output))
		return cmdErr
	}

	// Remove dynamic user secret
	secretRef, err := getSecretFromCephFSPersistentVolume(volume)
	if err != nil {
		glog.Errorf("failed to get secret references, err: %v", err)
		return err
	}
	err = p.client.CoreV1().Secrets(secretRef.Namespace).Delete(secretRef.Name, &metav1.DeleteOptions{})
	if err != nil {
		glog.Errorf("Cephfs Provisioner: delete secret failed, err: %v", err)
		return fmt.Errorf("failed to delete secret")
	}

	return nil
}

func (p *cephFSProvisioner) parseParameters(parameters map[string]string) (string, string, string, string, []string, bool, error) {
	var (
		err                                                                           error
		mon                                                                           []string
		cluster, adminID, adminSecretName, adminSecretNamespace, adminSecret, pvcRoot string
		deterministicNames                                                            bool
	)

	adminSecretNamespace = "default"
	adminID = "admin"
	cluster = "ceph"
	pvcRoot = "/volumes/kubernetes"
	deterministicNames = false

	for k, v := range parameters {
		switch strings.ToLower(k) {
		case "cluster":
			cluster = v
		case "monitors":
			arr := strings.Split(v, ",")
			for _, m := range arr {
				mon = append(mon, m)
			}
		case "adminid":
			adminID = v
		case "adminsecretname":
			adminSecretName = v
		case "adminsecretnamespace":
			adminSecretNamespace = v
		case "claimroot":
			pvcRoot = v
		case "deterministicnames":
			// On error, strconv.ParseBool() returns false; leave that, as it is a perfectly fine default
			deterministicNames, _ = strconv.ParseBool(v)
		default:
			return "", "", "", "", nil, false, fmt.Errorf("invalid option %q", k)
		}
	}
	// sanity check
	if adminSecretName == "" {
		return "", "", "", "", nil, false, fmt.Errorf("missing Ceph admin secret name")
	}
	if adminSecret, err = p.parsePVSecret(adminSecretNamespace, adminSecretName); err != nil {
		return "", "", "", "", nil, false, fmt.Errorf("failed to get admin secret from [%q/%q]: %v", adminSecretNamespace, adminSecretName, err)
	}
	if len(mon) < 1 {
		return "", "", "", "", nil, false, fmt.Errorf("missing Ceph monitors")
	}
	return cluster, adminID, adminSecret, pvcRoot, mon, deterministicNames, nil
}

func (p *cephFSProvisioner) parsePVSecret(namespace, secretName string) (string, error) {
	if p.client == nil {
		return "", fmt.Errorf("Cannot get kube client")
	}
	secrets, err := p.client.CoreV1().Secrets(namespace).Get(secretName, metav1.GetOptions{})
	if err != nil {
		return "", err
	}
	for _, data := range secrets.Data {
		return string(data), nil
	}

	// If not found, the last secret in the map wins as done before
	return "", fmt.Errorf("no secret found")
}

var (
	master          = flag.String("master", "", "Master URL")
	kubeconfig      = flag.String("kubeconfig", "", "Absolute path to the kubeconfig")
	id              = flag.String("id", "", "Unique provisioner identity")
	secretNamespace = flag.String("secret-namespace", "", "Namespace secrets will be created in (default: '', created in each PVC's namespace)")
)

func main() {
	flag.Parse()
	flag.Set("logtostderr", "true")

	var config *rest.Config
	var err error
	if *master != "" || *kubeconfig != "" {
		config, err = clientcmd.BuildConfigFromFlags(*master, *kubeconfig)
	} else {
		config, err = rest.InClusterConfig()
	}
	if err != nil {
		glog.Fatalf("Failed to create config: %v", err)
	}
	clientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		glog.Fatalf("Failed to create client: %v", err)
	}

	prName := provisionerName
	prNameFromEnv := os.Getenv(provisionerNameKey)
	if prNameFromEnv != "" {
		prName = prNameFromEnv
	}

	// By default, we use provisioner name as provisioner identity.
	// User may specify their own identity with `-id` flag to distinguish each
	// others, if they deploy more than one CephFS provisioners under same provisioner name.
	prID := prName
	if *id != "" {
		prID = *id
	}

	secretNamespaceFromEnv := os.Getenv(secretNamespaceKey)
	if secretNamespaceFromEnv != "" {
		*secretNamespace = secretNamespaceFromEnv
	}

	// The controller needs to know what the server version is because out-of-tree
	// provisioners aren't officially supported until 1.5
	serverVersion, err := clientset.Discovery().ServerVersion()
	if err != nil {
		glog.Fatalf("Error getting server version: %v", err)
	}

	// Create the provisioner: it implements the Provisioner interface expected by
	// the controller
	glog.Infof("Creating CephFS provisioner %s with identity: %s, secret namespace: %s", prName, prID, *secretNamespace)
	cephFSProvisioner := newCephFSProvisioner(clientset, prID, *secretNamespace)

	// Start the provision controller which will dynamically provision cephFS
	// PVs
	pc := controller.NewProvisionController(
		clientset,
		prName,
		cephFSProvisioner,
		serverVersion.GitVersion,
	)

	pc.Run(wait.NeverStop)
}
<|endoftext|>


<commit_before>package command_registry

import (
	"fmt"
	"os"
	"time"

	"github.com/cloudfoundry/cli/cf/api"
	"github.com/cloudfoundry/cli/cf/configuration/config_helpers"
	"github.com/cloudfoundry/cli/cf/configuration/core_config"
	"github.com/cloudfoundry/cli/cf/configuration/plugin_config"
	"github.com/cloudfoundry/cli/cf/i18n/detection"
	"github.com/cloudfoundry/cli/cf/manifest"
	"github.com/cloudfoundry/cli/cf/net"
	"github.com/cloudfoundry/cli/cf/terminal"
	"github.com/cloudfoundry/cli/cf/trace"
	"github.com/cloudfoundry/cli/plugin/models"
)

type Dependency struct {
	Ui           terminal.UI
	Config       core_config.Repository
	RepoLocator  api.RepositoryLocator
	Detector     detection.Detector
	PluginConfig plugin_config.PluginConfiguration
	ManifestRepo manifest.ManifestRepository
	Gateways     map[string]net.Gateway
	TeePrinter   *terminal.TeePrinter
	PluginModels *pluginModels
}

type pluginModels struct {
	Application   *plugin_models.GetAppModel
	AppsSummary   *[]plugin_models.GetAppsModel
	Organizations *[]plugin_models.GetOrgs_Model
	Organization  *plugin_models.GetOrg_Model
	Spaces        *[]plugin_models.GetSpaces_Model
	Space         *plugin_models.GetSpace_Model
	OrgUsers      *[]plugin_models.GetOrgUsers_Model
	SpaceUsers    *[]plugin_models.GetSpaceUsers_Model
	Services      *[]plugin_models.GetServices_Model
	Service       *plugin_models.GetService_Model
}

func NewDependency() Dependency {
	deps := Dependency{}
	deps.TeePrinter = terminal.NewTeePrinter()
	deps.Ui = terminal.NewUI(os.Stdin, deps.TeePrinter)
	deps.ManifestRepo = manifest.NewManifestDiskRepository()

	errorHandler := func(err error) {
		if err != nil {
			deps.Ui.Failed(fmt.Sprintf("Config error: %s", err))
		}
	}
	deps.Config = core_config.NewRepositoryFromFilepath(config_helpers.DefaultFilePath(), errorHandler)
	deps.PluginConfig = plugin_config.NewPluginConfig(errorHandler)
	deps.Detector = &detection.JibberJabberDetector{}

	terminal.UserAskedForColors = deps.Config.ColorEnabled()
	terminal.InitColorSupport()

	if os.Getenv("CF_TRACE") != "" {
		trace.Logger = trace.NewLogger(os.Getenv("CF_TRACE"))
	} else {
		trace.Logger = trace.NewLogger(deps.Config.Trace())
	}

	deps.Gateways = map[string]net.Gateway{
		"auth":             net.NewUAAGateway(deps.Config, deps.Ui),
		"cloud-controller": net.NewCloudControllerGateway(deps.Config, time.Now, deps.Ui),
		"uaa":              net.NewUAAGateway(deps.Config, deps.Ui),
	}
	deps.RepoLocator = api.NewRepositoryLocator(deps.Config, deps.Gateways)

	deps.PluginModels = &pluginModels{Application: nil}

	return deps
}
<commit_msg>add AppManifest to dependency object<commit_after>package command_registry

import (
	"fmt"
	"os"
	"time"

	"github.com/cloudfoundry/cli/cf/api"
	"github.com/cloudfoundry/cli/cf/configuration/config_helpers"
	"github.com/cloudfoundry/cli/cf/configuration/core_config"
	"github.com/cloudfoundry/cli/cf/configuration/plugin_config"
	"github.com/cloudfoundry/cli/cf/i18n/detection"
	"github.com/cloudfoundry/cli/cf/manifest"
	"github.com/cloudfoundry/cli/cf/net"
	"github.com/cloudfoundry/cli/cf/terminal"
	"github.com/cloudfoundry/cli/cf/trace"
	"github.com/cloudfoundry/cli/plugin/models"
)

type Dependency struct {
	Ui           terminal.UI
	Config       core_config.Repository
	RepoLocator  api.RepositoryLocator
	Detector     detection.Detector
	PluginConfig plugin_config.PluginConfiguration
	ManifestRepo manifest.ManifestRepository
	AppManifest  manifest.AppManifest
	Gateways     map[string]net.Gateway
	TeePrinter   *terminal.TeePrinter
	PluginModels *pluginModels
}

type pluginModels struct {
	Application   *plugin_models.GetAppModel
	AppsSummary   *[]plugin_models.GetAppsModel
	Organizations *[]plugin_models.GetOrgs_Model
	Organization  *plugin_models.GetOrg_Model
	Spaces        *[]plugin_models.GetSpaces_Model
	Space         *plugin_models.GetSpace_Model
	OrgUsers      *[]plugin_models.GetOrgUsers_Model
	SpaceUsers    *[]plugin_models.GetSpaceUsers_Model
	Services      *[]plugin_models.GetServices_Model
	Service       *plugin_models.GetService_Model
}

func NewDependency() Dependency {
	deps := Dependency{}
	deps.TeePrinter = terminal.NewTeePrinter()
	deps.Ui = terminal.NewUI(os.Stdin, deps.TeePrinter)
	deps.ManifestRepo = manifest.NewManifestDiskRepository()

	errorHandler := func(err error) {
		if err != nil {
			deps.Ui.Failed(fmt.Sprintf("Config error: %s", err))
		}
	}
	deps.Config = core_config.NewRepositoryFromFilepath(config_helpers.DefaultFilePath(), errorHandler)
	deps.PluginConfig = plugin_config.NewPluginConfig(errorHandler)
	deps.Detector = &detection.JibberJabberDetector{}

	terminal.UserAskedForColors = deps.Config.ColorEnabled()
	terminal.InitColorSupport()

	if os.Getenv("CF_TRACE") != "" {
		trace.Logger = trace.NewLogger(os.Getenv("CF_TRACE"))
	} else {
		trace.Logger = trace.NewLogger(deps.Config.Trace())
	}

	deps.Gateways = map[string]net.Gateway{
		"auth":             net.NewUAAGateway(deps.Config, deps.Ui),
		"cloud-controller": net.NewCloudControllerGateway(deps.Config, time.Now, deps.Ui),
		"uaa":              net.NewUAAGateway(deps.Config, deps.Ui),
	}
	deps.RepoLocator = api.NewRepositoryLocator(deps.Config, deps.Gateways)

	deps.PluginModels = &pluginModels{Application: nil}

	return deps
}
<|endoftext|>