<commit_before>package main

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"encoding/json"
	"expvar"
	"flag"
	"fmt"
	"github.com/jmhodges/howsmyssl/tls"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	_ "net/http/pprof"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const (
	resp500Format = `HTTP/1.%d 500 Internal Server Error
Content-Length: 26
Connection: close
Content-Type: text/plain; charset="utf-8"
Strict-Transport-Security: max-age=631138519; includeSubdomains
Date: %s

500 Internal Server Error
`
	hstsHeaderValue = "max-age=631138519; includeSubdomains"
	xForwardedProto = "X-Forwarded-Proto"
)

var (
	httpsAddr = flag.String("httpsAddr", "localhost:10443", "address to boot the HTTPS server on")
	httpAddr  = flag.String("httpAddr", "localhost:10080", "address to boot the HTTP server on")
	vhost     = flag.String("vhost", "localhost:10443", "public domain to use in redirects and templates")
	certPath  = flag.String("cert", "./config/development.crt", "file path to the TLS certificate to serve with")
	keyPath   = flag.String("key", "./config/development.key", "file path to the TLS key to serve with")
	staticDir = flag.String("staticDir", "./static", "file path to the directory of static files to serve")
	tmplDir   = flag.String("templateDir", "./templates", "file path to the directory of templates")
	adminPort = flag.String("adminPort", "4567", "localhost port to boot the admin server on")

	apiVars         = expvar.NewMap("api")
	staticVars      = expvar.NewMap("static")
	webVars         = expvar.NewMap("web")
	apiRequests     = new(expvar.Int)
	staticRequests  = new(expvar.Int)
	webRequests     = new(expvar.Int)
	apiStatuses     = NewStatusStats(apiVars)
	staticStatuses  = NewStatusStats(staticVars)
	webStatuses     = NewStatusStats(webVars)
	commonRedirects = expvar.NewInt("common_redirects")

	nonAlphaNumeric, _ = regexp.Compile("[^[:alnum:]]")

	index *template.Template
)

func main() {
	flag.Parse()

	host := *vhost
	if strings.Contains(*vhost, ":") {
		var err error
		shost, port, err := net.SplitHostPort(*vhost)
		if err != nil {
			log.Fatalf("unable to parse httpsAddr: %s", err)
		}
		host = shost
		if port != "443" {
			host = *vhost
		}
	}

	apiVars.Set("requests", apiRequests)
	staticVars.Set("requests", staticRequests)
	webVars.Set("requests", webRequests)

	index = loadIndex()
	tlsConf := makeTLSConfig(*certPath, *keyPath)

	tlsListener, err := tls.Listen("tcp", *httpsAddr, tlsConf)
	if err != nil {
		log.Fatalf("unable to listen for the HTTPS server on %s: %s", *httpsAddr, err)
	}
	plaintextListener, err := net.Listen("tcp", *httpAddr)
	if err != nil {
		log.Fatalf("unable to listen for the HTTP server on %s: %s", *httpAddr, err)
	}
	l := &listener{tlsListener}

	m := tlsMux(
		host,
		makeStaticHandler())

	adminAddr := net.JoinHostPort("localhost", *adminPort)
	go func() {
		err := http.ListenAndServe(adminAddr, nil)
		if err != nil {
			log.Fatalf("unable to open admin server: %s", err)
		}
	}()

	log.Printf("Booting HTTPS on %s and HTTP on %s", *httpsAddr, *httpAddr)
	go func() {
		err := http.Serve(l, m)
		if err != nil {
			log.Fatalf("https server error: %s", err)
		}
	}()
	err = http.Serve(plaintextListener, plaintextMux(host))
	if err != nil {
		log.Fatalf("http server error: %s", err)
	}
}

func tlsMux(vhost string, staticHandler http.Handler) http.Handler {
	m := http.NewServeMux()
	m.Handle(vhost+"/s/", staticHandler)
	m.HandleFunc(vhost+"/a/check", handleAPI)
	m.HandleFunc(vhost+"/", handleWeb)
	m.HandleFunc(vhost+"/healthcheck", healthcheck)
	m.HandleFunc("/healthcheck", healthcheck)
	m.Handle("/", commonRedirect(vhost))
	return protoHandler{logHandler{m}, "https"}
}

func plaintextMux(vhost string) http.Handler {
	m := http.NewServeMux()
	m.HandleFunc("/healthcheck", healthcheck)
	m.Handle("/", commonRedirect(vhost))
	return protoHandler{logHandler{m}, "http"}
}

func renderHTML(r *http.Request, data *clientInfo) ([]byte, error) {
	b := new(bytes.Buffer)
	err := index.Execute(b, data)
	if err != nil {
		return nil, err
	}
	return b.Bytes(), nil
}

func renderJSON(r *http.Request, data *clientInfo) ([]byte, error) {
	marshalled, err := json.Marshal(data)
	if err != nil {
		return marshalled, err
	}
	callback := r.FormValue("callback")
	callback = nonAlphaNumeric.ReplaceAllString(callback, "")

	if callback != "" {
		return []byte(fmt.Sprintf("%s(%s)", callback, marshalled)), nil
	} else {
		return marshalled, nil
	}
}

func handleWeb(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path != "/" {
		http.Error(w, "404 Not Found", http.StatusNotFound)
		return
	}
	webRequests.Add(1)
	hijackHandle(w, r, "text/html;charset=utf-8", webStatuses, renderHTML)
}

func handleAPI(w http.ResponseWriter, r *http.Request) {
	apiRequests.Add(1)
	hijackHandle(w, r, "application/json", apiStatuses, renderJSON)
}

func hijackHandle(w http.ResponseWriter, r *http.Request, contentType string, statuses *statusStats, render func(*http.Request, *clientInfo) ([]byte, error)) {
	hj, ok := w.(http.Hijacker)
	if !ok {
		log.Printf("server not hijackable\n")
		http.Error(w, "500 Internal Server Error", http.StatusInternalServerError)
		return
	}

	c, brw, err := hj.Hijack()
	if err != nil {
		log.Printf("server errored during hijack: %s\n", err)
		return
	}
	defer c.Close()
	tc, ok := c.(*conn)
	if !ok {
		log.Printf("Unable to convert net.Conn to *conn: %s\n", err)
		hijacked500(brw, r.ProtoMinor, statuses)
	}
	data := ClientInfo(tc)
	bs, err := render(r, data)
	if err != nil {
		log.Printf("Unable to excute index template: %s\n", err)
		hijacked500(brw, r.ProtoMinor, statuses)
		return
	}
	contentLength := int64(len(bs))
	h := make(http.Header)
	h.Set("Date", time.Now().Format(http.TimeFormat))
	h.Set("Content-Type", contentType)
	if r.ProtoMinor == 1 { // Assumes HTTP/1.x
		h.Set("Connection", "close")
	}
	h.Set("Strict-Transport-Security", hstsHeaderValue)
	h.Set("Content-Length", strconv.FormatInt(contentLength, 10))
	// Allow CORS requests from any domain, for easy API access
	h.Set("Access-Control-Allow-Origin", "*")
	resp := &http.Response{
		StatusCode:    200,
		ContentLength: contentLength,
		Header:        h,
		Body:          ioutil.NopCloser(bytes.NewBuffer(bs)),
		ProtoMajor:    1, // Assumes HTTP/1.x
		ProtoMinor:    r.ProtoMinor,
	}
	bs, err = httputil.DumpResponse(resp, true)
	if err != nil {
		log.Printf("unable to write response: %s\n", err)
		hijacked500(brw, r.ProtoMinor, statuses)
		return
	}
	statuses.status2xx.Add(1)
	brw.Write(bs)
	brw.Flush()
}

func hijacked500(brw *bufio.ReadWriter, protoMinor int, statuses *statusStats) {
	statuses.status5xx.Add(1)
	// Assumes HTTP/1.x
	s := fmt.Sprintf(resp500Format, protoMinor, time.Now().Format(http.TimeFormat))
	brw.WriteString(s)
	brw.Flush()
}

func healthcheck(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(200)
	w.Write([]byte("ok"))
}

func commonRedirect(vhost string) http.Handler {
	hf := func(w http.ResponseWriter, r *http.Request) {
		commonRedirects.Add(1)
		if r.Header.Get(xForwardedProto) == "https" {
			w.Header().Set("Strict-Transport-Security", hstsHeaderValue)
		}
		u := r.URL
		// Never set by the Go HTTP library.
		u.Scheme = "https"
		u.Host = vhost
		http.Redirect(w, r, u.String(), http.StatusMovedPermanently)
	}
	return http.HandlerFunc(hf)
}

func loadIndex() *template.Template {
	return template.Must(template.New("index.html").
		Funcs(template.FuncMap{"sentence": sentence, "ratingSpan": ratingSpan}).
		ParseFiles(*tmplDir + "/index.html"))
}

func makeTLSConfig(certPath, keyPath string) *tls.Config {
	cert, err := tls.LoadX509KeyPair(certPath, keyPath)
	if err != nil {
		log.Fatalf("unable to load TLS key cert pair %s: %s", certPath, err)
	}

	tlsConf := &tls.Config{
		Certificates:             []tls.Certificate{cert},
		NextProtos:               []string{"https"},
		PreferServerCipherSuites: true,
	}
	tlsConf.BuildNameToCertificate()
	return tlsConf
}

func makeStaticHandler() http.HandlerFunc {
	stats := NewStatusStats(staticVars)
	h := http.StripPrefix("/s/", http.FileServer(http.Dir(*staticDir)))
	h = makeGzipHandler(h)
	return func(w http.ResponseWriter, r *http.Request) {
		staticRequests.Add(1)
		w = &statWriter{w: w, stats: stats}
		h.ServeHTTP(w, r)
	}
}

func ratingSpan(rating Rating) template.HTML {
	class := ""
	switch rating {
	case okay:
		class = "okay"
	case improvable:
		class = "improvable"
	case bad:
		class = "bad"
	}
	return template.HTML(class)
}

func sentence(parts []string) string {
	if len(parts) == 1 {
		return parts[0] + "."
	}
	commaed := parts[:len(parts)-1]
	return strings.Join(commaed, ", ") + ", and " + parts[len(parts)-1] + "."
}

type logHandler struct {
	inner http.Handler
}

// Since we have a Hijack in our code, this simple writer will suffice for
// now.
func (h logHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	host, _, err := net.SplitHostPort(r.RemoteAddr)
	if err != nil {
		host = "0.0.0.0"
	}
	proto := r.Header.Get(xForwardedProto)
	if proto == "" {
		proto = "unknown"
	}
	fmt.Printf("%s %s %s\n", host, proto, r.URL)
	h.inner.ServeHTTP(w, r)
}

type protoHandler struct {
	inner http.Handler
	proto string
}

func (h protoHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	r.Header.Set(xForwardedProto, h.proto)
	h.inner.ServeHTTP(w, r)
}

type gzipWriter struct {
	io.Writer
	http.ResponseWriter
}

func (w gzipWriter) Write(b []byte) (int, error) {
	return w.Writer.Write(b)
}

func makeGzipHandler(inner http.Handler) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
			inner.ServeHTTP(w, r)
			return
		}
		w.Header().Set("Content-Encoding", "gzip")
		gz := gzip.NewWriter(w)
		defer gz.Close()
		gzr := gzipWriter{Writer: gz, ResponseWriter: w}
		inner.ServeHTTP(gzr, r)
	}
}
<commit_msg>Switch to regexp.MustCompile, use append rather than fmt.<commit_after>package main

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"encoding/json"
	"expvar"
	"flag"
	"fmt"
	"github.com/jmhodges/howsmyssl/tls"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	_ "net/http/pprof"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const (
	resp500Format = `HTTP/1.%d 500 Internal Server Error
Content-Length: 26
Connection: close
Content-Type: text/plain; charset="utf-8"
Strict-Transport-Security: max-age=631138519; includeSubdomains
Date: %s

500 Internal Server Error
`
	hstsHeaderValue = "max-age=631138519; includeSubdomains"
	xForwardedProto = "X-Forwarded-Proto"
)

var (
	httpsAddr = flag.String("httpsAddr", "localhost:10443", "address to boot the HTTPS server on")
	httpAddr  = flag.String("httpAddr", "localhost:10080", "address to boot the HTTP server on")
	vhost     = flag.String("vhost", "localhost:10443", "public domain to use in redirects and templates")
	certPath  = flag.String("cert", "./config/development.crt", "file path to the TLS certificate to serve with")
	keyPath   = flag.String("key", "./config/development.key", "file path to the TLS key to serve with")
	staticDir = flag.String("staticDir", "./static", "file path to the directory of static files to serve")
	tmplDir   = flag.String("templateDir", "./templates", "file path to the directory of templates")
	adminPort = flag.String("adminPort", "4567", "localhost port to boot the admin server on")

	apiVars         = expvar.NewMap("api")
	staticVars      = expvar.NewMap("static")
	webVars         = expvar.NewMap("web")
	apiRequests     = new(expvar.Int)
	staticRequests  = new(expvar.Int)
	webRequests     = new(expvar.Int)
	apiStatuses     = NewStatusStats(apiVars)
	staticStatuses  = NewStatusStats(staticVars)
	webStatuses     = NewStatusStats(webVars)
	commonRedirects = expvar.NewInt("common_redirects")

	nonAlphaNumeric = regexp.MustCompile("[^[:alnum:]]")

	index *template.Template
)

func main() {
	flag.Parse()

	host := *vhost
	if strings.Contains(*vhost, ":") {
		var err error
		shost, port, err := net.SplitHostPort(*vhost)
		if err != nil {
			log.Fatalf("unable to parse httpsAddr: %s", err)
		}
		host = shost
		if port != "443" {
			host = *vhost
		}
	}

	apiVars.Set("requests", apiRequests)
	staticVars.Set("requests", staticRequests)
	webVars.Set("requests", webRequests)

	index = loadIndex()
	tlsConf := makeTLSConfig(*certPath, *keyPath)

	tlsListener, err := tls.Listen("tcp", *httpsAddr, tlsConf)
	if err != nil {
		log.Fatalf("unable to listen for the HTTPS server on %s: %s", *httpsAddr, err)
	}
	plaintextListener, err := net.Listen("tcp", *httpAddr)
	if err != nil {
		log.Fatalf("unable to listen for the HTTP server on %s: %s", *httpAddr, err)
	}
	l := &listener{tlsListener}

	m := tlsMux(
		host,
		makeStaticHandler())

	adminAddr := net.JoinHostPort("localhost", *adminPort)
	go func() {
		err := http.ListenAndServe(adminAddr, nil)
		if err != nil {
			log.Fatalf("unable to open admin server: %s", err)
		}
	}()

	log.Printf("Booting HTTPS on %s and HTTP on %s", *httpsAddr, *httpAddr)
	go func() {
		err := http.Serve(l, m)
		if err != nil {
			log.Fatalf("https server error: %s", err)
		}
	}()
	err = http.Serve(plaintextListener, plaintextMux(host))
	if err != nil {
		log.Fatalf("http server error: %s", err)
	}
}

func tlsMux(vhost string, staticHandler http.Handler) http.Handler {
	m := http.NewServeMux()
	m.Handle(vhost+"/s/", staticHandler)
	m.HandleFunc(vhost+"/a/check", handleAPI)
	m.HandleFunc(vhost+"/", handleWeb)
	m.HandleFunc(vhost+"/healthcheck", healthcheck)
	m.HandleFunc("/healthcheck", healthcheck)
	m.Handle("/", commonRedirect(vhost))
	return protoHandler{logHandler{m}, "https"}
}

func plaintextMux(vhost string) http.Handler {
	m := http.NewServeMux()
	m.HandleFunc("/healthcheck", healthcheck)
	m.Handle("/", commonRedirect(vhost))
	return protoHandler{logHandler{m}, "http"}
}

func renderHTML(r *http.Request, data *clientInfo) ([]byte, error) {
	b := new(bytes.Buffer)
	err := index.Execute(b, data)
	if err != nil {
		return nil, err
	}
	return b.Bytes(), nil
}

func renderJSON(r *http.Request, data *clientInfo) ([]byte, error) {
	marshalled, err := json.Marshal(data)
	if err != nil {
		return nil, err
	}
	callback := r.FormValue("callback")
	sanitizedCallback := nonAlphaNumeric.ReplaceAll([]byte(callback), []byte(""))

	if callback != "" {
		result := append(sanitizedCallback, "("...)
		result = append(result, marshalled...)
		result = append(result, ")"...)
		return result, nil
	} else {
		return marshalled, nil
	}
}

func handleWeb(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path != "/" {
		http.Error(w, "404 Not Found", http.StatusNotFound)
		return
	}
	webRequests.Add(1)
	hijackHandle(w, r, "text/html;charset=utf-8", webStatuses, renderHTML)
}

func handleAPI(w http.ResponseWriter, r *http.Request) {
	apiRequests.Add(1)
	hijackHandle(w, r, "application/json", apiStatuses, renderJSON)
}

func hijackHandle(w http.ResponseWriter, r *http.Request, contentType string, statuses *statusStats, render func(*http.Request, *clientInfo) ([]byte, error)) {
	hj, ok := w.(http.Hijacker)
	if !ok {
		log.Printf("server not hijackable\n")
		http.Error(w, "500 Internal Server Error", http.StatusInternalServerError)
		return
	}

	c, brw, err := hj.Hijack()
	if err != nil {
		log.Printf("server errored during hijack: %s\n", err)
		return
	}
	defer c.Close()
	tc, ok := c.(*conn)
	if !ok {
		log.Printf("Unable to convert net.Conn to *conn: %s\n", err)
		hijacked500(brw, r.ProtoMinor, statuses)
	}
	data := ClientInfo(tc)
	bs, err := render(r, data)
	if err != nil {
		log.Printf("Unable to excute index template: %s\n", err)
		hijacked500(brw, r.ProtoMinor, statuses)
		return
	}
	contentLength := int64(len(bs))
	h := make(http.Header)
	h.Set("Date", time.Now().Format(http.TimeFormat))
	h.Set("Content-Type", contentType)
	if r.ProtoMinor == 1 { // Assumes HTTP/1.x
		h.Set("Connection", "close")
	}
	h.Set("Strict-Transport-Security", hstsHeaderValue)
	h.Set("Content-Length", strconv.FormatInt(contentLength, 10))
	// Allow CORS requests from any domain, for easy API access
	h.Set("Access-Control-Allow-Origin", "*")
	resp := &http.Response{
		StatusCode:    200,
		ContentLength: contentLength,
		Header:        h,
		Body:          ioutil.NopCloser(bytes.NewBuffer(bs)),
		ProtoMajor:    1, // Assumes HTTP/1.x
		ProtoMinor:    r.ProtoMinor,
	}
	bs, err = httputil.DumpResponse(resp, true)
	if err != nil {
		log.Printf("unable to write response: %s\n", err)
		hijacked500(brw, r.ProtoMinor, statuses)
		return
	}
	statuses.status2xx.Add(1)
	brw.Write(bs)
	brw.Flush()
}

func hijacked500(brw *bufio.ReadWriter, protoMinor int, statuses *statusStats) {
	statuses.status5xx.Add(1)
	// Assumes HTTP/1.x
	s := fmt.Sprintf(resp500Format, protoMinor, time.Now().Format(http.TimeFormat))
	brw.WriteString(s)
	brw.Flush()
}

func healthcheck(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(200)
	w.Write([]byte("ok"))
}

func commonRedirect(vhost string) http.Handler {
	hf := func(w http.ResponseWriter, r *http.Request) {
		commonRedirects.Add(1)
		if r.Header.Get(xForwardedProto) == "https" {
			w.Header().Set("Strict-Transport-Security", hstsHeaderValue)
		}
		u := r.URL
		// Never set by the Go HTTP library.
		u.Scheme = "https"
		u.Host = vhost
		http.Redirect(w, r, u.String(), http.StatusMovedPermanently)
	}
	return http.HandlerFunc(hf)
}

func loadIndex() *template.Template {
	return template.Must(template.New("index.html").
		Funcs(template.FuncMap{"sentence": sentence, "ratingSpan": ratingSpan}).
		ParseFiles(*tmplDir + "/index.html"))
}

func makeTLSConfig(certPath, keyPath string) *tls.Config {
	cert, err := tls.LoadX509KeyPair(certPath, keyPath)
	if err != nil {
		log.Fatalf("unable to load TLS key cert pair %s: %s", certPath, err)
	}

	tlsConf := &tls.Config{
		Certificates:             []tls.Certificate{cert},
		NextProtos:               []string{"https"},
		PreferServerCipherSuites: true,
	}
	tlsConf.BuildNameToCertificate()
	return tlsConf
}

func makeStaticHandler() http.HandlerFunc {
	stats := NewStatusStats(staticVars)
	h := http.StripPrefix("/s/", http.FileServer(http.Dir(*staticDir)))
	h = makeGzipHandler(h)
	return func(w http.ResponseWriter, r *http.Request) {
		staticRequests.Add(1)
		w = &statWriter{w: w, stats: stats}
		h.ServeHTTP(w, r)
	}
}

func ratingSpan(rating Rating) template.HTML {
	class := ""
	switch rating {
	case okay:
		class = "okay"
	case improvable:
		class = "improvable"
	case bad:
		class = "bad"
	}
	return template.HTML(class)
}

func sentence(parts []string) string {
	if len(parts) == 1 {
		return parts[0] + "."
	}
	commaed := parts[:len(parts)-1]
	return strings.Join(commaed, ", ") + ", and " + parts[len(parts)-1] + "."
}

type logHandler struct {
	inner http.Handler
}

// Since we have a Hijack in our code, this simple writer will suffice for
// now.
func (h logHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	host, _, err := net.SplitHostPort(r.RemoteAddr)
	if err != nil {
		host = "0.0.0.0"
	}
	proto := r.Header.Get(xForwardedProto)
	if proto == "" {
		proto = "unknown"
	}
	fmt.Printf("%s %s %s\n", host, proto, r.URL)
	h.inner.ServeHTTP(w, r)
}

type protoHandler struct {
	inner http.Handler
	proto string
}

func (h protoHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	r.Header.Set(xForwardedProto, h.proto)
	h.inner.ServeHTTP(w, r)
}

type gzipWriter struct {
	io.Writer
	http.ResponseWriter
}

func (w gzipWriter) Write(b []byte) (int, error) {
	return w.Writer.Write(b)
}

func makeGzipHandler(inner http.Handler) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
			inner.ServeHTTP(w, r)
			return
		}
		w.Header().Set("Content-Encoding", "gzip")
		gz := gzip.NewWriter(w)
		defer gz.Close()
		gzr := gzipWriter{Writer: gz, ResponseWriter: w}
		inner.ServeHTTP(gzr, r)
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package container defines types for sub-container events and also
// defines an interface for container operation handlers.
package raw

import (
	"fmt"
	"io/ioutil"
	"path"
	"strings"

	"github.com/google/cadvisor/container/common"
	"github.com/google/cadvisor/container/libcontainer"
	"github.com/google/cadvisor/manager/watcher"

	"github.com/golang/glog"
	"golang.org/x/exp/inotify"
)

type rawContainerWatcher struct {
	// Absolute path to the root of the cgroup hierarchies
	cgroupPaths map[string]string

	cgroupSubsystems *libcontainer.CgroupSubsystems

	// Inotify event watcher.
	watcher *common.InotifyWatcher

	// Signal for watcher thread to stop.
	stopWatcher chan error
}

func NewRawContainerWatcher() (watcher.ContainerWatcher, error) {
	cgroupSubsystems, err := libcontainer.GetCgroupSubsystems()
	if err != nil {
		return nil, fmt.Errorf("failed to get cgroup subsystems: %v", err)
	}
	if len(cgroupSubsystems.Mounts) == 0 {
		return nil, fmt.Errorf("failed to find supported cgroup mounts for the raw factory")
	}

	watcher, err := common.NewInotifyWatcher()
	if err != nil {
		return nil, err
	}

	rawWatcher := &rawContainerWatcher{
		cgroupPaths:      common.MakeCgroupPaths(cgroupSubsystems.MountPoints, "/"),
		cgroupSubsystems: &cgroupSubsystems,
		watcher:          watcher,
		stopWatcher:      make(chan error),
	}

	return rawWatcher, nil
}

func (self *rawContainerWatcher) Start(events chan watcher.ContainerEvent) error {
	// Watch this container (all its cgroups) and all subdirectories.
	for _, cgroupPath := range self.cgroupPaths {
		_, err := self.watchDirectory(cgroupPath, "/")
		if err != nil {
			return err
		}
	}

	// Process the events received from the kernel.
	go func() {
		for {
			select {
			case event := <-self.watcher.Event():
				err := self.processEvent(event, events)
				if err != nil {
					glog.Warningf("Error while processing event (%+v): %v", event, err)
				}
			case err := <-self.watcher.Error():
				glog.Warningf("Error while watching %q:", "/", err)
			case <-self.stopWatcher:
				err := self.watcher.Close()
				if err == nil {
					self.stopWatcher <- err
					return
				}
			}
		}
	}()

	return nil
}

func (self *rawContainerWatcher) Stop() error {
	// Rendezvous with the watcher thread.
	self.stopWatcher <- nil
	return <-self.stopWatcher
}

// Watches the specified directory and all subdirectories. Returns whether the path was
// already being watched and an error (if any).
func (self *rawContainerWatcher) watchDirectory(dir string, containerName string) (bool, error) {
	alreadyWatching, err := self.watcher.AddWatch(containerName, dir)
	if err != nil {
		return alreadyWatching, err
	}

	// Remove the watch if further operations failed.
	cleanup := true
	defer func() {
		if cleanup {
			_, err := self.watcher.RemoveWatch(containerName, dir)
			if err != nil {
				glog.Warningf("Failed to remove inotify watch for %q: %v", dir, err)
			}
		}
	}()

	// TODO(vmarmol): We should re-do this once we're done to ensure directories were not added in the meantime.
	// Watch subdirectories as well.
	entries, err := ioutil.ReadDir(dir)
	if err != nil {
		return alreadyWatching, err
	}
	for _, entry := range entries {
		if entry.IsDir() {
			// TODO(vmarmol): We don't have to fail here, maybe we can recover and try to get as many registrations as we can.
			_, err = self.watchDirectory(path.Join(dir, entry.Name()), path.Join(containerName, entry.Name()))
			if err != nil {
				return alreadyWatching, err
			}
		}
	}

	cleanup = false
	return alreadyWatching, nil
}

func (self *rawContainerWatcher) processEvent(event *inotify.Event, events chan watcher.ContainerEvent) error {
	// Convert the inotify event type to a container create or delete.
	var eventType watcher.ContainerEventType
	switch {
	case (event.Mask & inotify.IN_CREATE) > 0:
		eventType = watcher.ContainerAdd
	case (event.Mask & inotify.IN_DELETE) > 0:
		eventType = watcher.ContainerDelete
	case (event.Mask & inotify.IN_MOVED_FROM) > 0:
		eventType = watcher.ContainerDelete
	case (event.Mask & inotify.IN_MOVED_TO) > 0:
		eventType = watcher.ContainerAdd
	default:
		// Ignore other events.
		return nil
	}

	// Derive the container name from the path name.
	var containerName string
	for _, mount := range self.cgroupSubsystems.Mounts {
		mountLocation := path.Clean(mount.Mountpoint) + "/"
		if strings.HasPrefix(event.Name, mountLocation) {
			containerName = event.Name[len(mountLocation)-1:]
			break
		}
	}
	if containerName == "" {
		return fmt.Errorf("unable to detect container from watch event on directory %q", event.Name)
	}

	// Maintain the watch for the new or deleted container.
	switch eventType {
	case watcher.ContainerAdd:
		// New container was created, watch it.
		alreadyWatched, err := self.watchDirectory(event.Name, containerName)
		if err != nil {
			return err
		}

		// Only report container creation once.
		if alreadyWatched {
			return nil
		}
	case watcher.ContainerDelete:
		// Container was deleted, stop watching for it.
		lastWatched, err := self.watcher.RemoveWatch(containerName, event.Name)
		if err != nil {
			return err
		}

		// Only report container deletion once.
		if !lastWatched {
			return nil
		}
	default:
		return fmt.Errorf("unknown event type %v", eventType)
	}

	// Deliver the event.
	events <- watcher.ContainerEvent{
		EventType:   eventType,
		Name:        containerName,
		WatchSource: watcher.Raw,
	}

	return nil
}
<commit_msg>Continue watching other directories when there is watch error.<commit_after>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package container defines types for sub-container events and also
// defines an interface for container operation handlers.
package raw

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"

	"github.com/google/cadvisor/container/common"
	"github.com/google/cadvisor/container/libcontainer"
	"github.com/google/cadvisor/manager/watcher"

	"github.com/golang/glog"
	"golang.org/x/exp/inotify"
)

type rawContainerWatcher struct {
	// Absolute path to the root of the cgroup hierarchies
	cgroupPaths map[string]string

	cgroupSubsystems *libcontainer.CgroupSubsystems

	// Inotify event watcher.
	watcher *common.InotifyWatcher

	// Signal for watcher thread to stop.
	stopWatcher chan error
}

func NewRawContainerWatcher() (watcher.ContainerWatcher, error) {
	cgroupSubsystems, err := libcontainer.GetCgroupSubsystems()
	if err != nil {
		return nil, fmt.Errorf("failed to get cgroup subsystems: %v", err)
	}
	if len(cgroupSubsystems.Mounts) == 0 {
		return nil, fmt.Errorf("failed to find supported cgroup mounts for the raw factory")
	}

	watcher, err := common.NewInotifyWatcher()
	if err != nil {
		return nil, err
	}

	rawWatcher := &rawContainerWatcher{
		cgroupPaths:      common.MakeCgroupPaths(cgroupSubsystems.MountPoints, "/"),
		cgroupSubsystems: &cgroupSubsystems,
		watcher:          watcher,
		stopWatcher:      make(chan error),
	}

	return rawWatcher, nil
}

func (self *rawContainerWatcher) Start(events chan watcher.ContainerEvent) error {
	// Watch this container (all its cgroups) and all subdirectories.
	for _, cgroupPath := range self.cgroupPaths {
		_, err := self.watchDirectory(cgroupPath, "/")
		if err != nil {
			return err
		}
	}

	// Process the events received from the kernel.
	go func() {
		for {
			select {
			case event := <-self.watcher.Event():
				err := self.processEvent(event, events)
				if err != nil {
					glog.Warningf("Error while processing event (%+v): %v", event, err)
				}
			case err := <-self.watcher.Error():
				glog.Warningf("Error while watching %q:", "/", err)
			case <-self.stopWatcher:
				err := self.watcher.Close()
				if err == nil {
					self.stopWatcher <- err
					return
				}
			}
		}
	}()

	return nil
}

func (self *rawContainerWatcher) Stop() error {
	// Rendezvous with the watcher thread.
	self.stopWatcher <- nil
	return <-self.stopWatcher
}

// Watches the specified directory and all subdirectories. Returns whether the path was
// already being watched and an error (if any).
func (self *rawContainerWatcher) watchDirectory(dir string, containerName string) (bool, error) {
	alreadyWatching, err := self.watcher.AddWatch(containerName, dir)
	if err != nil {
		return alreadyWatching, err
	}

	// Remove the watch if further operations failed.
	cleanup := true
	defer func() {
		if cleanup {
			_, err := self.watcher.RemoveWatch(containerName, dir)
			if err != nil {
				glog.Warningf("Failed to remove inotify watch for %q: %v", dir, err)
			}
		}
	}()

	// TODO(vmarmol): We should re-do this once we're done to ensure directories were not added in the meantime.
	// Watch subdirectories as well.
	entries, err := ioutil.ReadDir(dir)
	if err != nil {
		return alreadyWatching, err
	}
	for _, entry := range entries {
		if entry.IsDir() {
			entryPath := path.Join(dir, entry.Name())
			_, err = self.watchDirectory(entryPath, path.Join(containerName, entry.Name()))
			if err != nil {
				glog.Errorf("Failed to watch directory %q: %v", entryPath, err)
				if os.IsNotExist(err) {
					// The directory may have been removed before watching. Try to watch the other
					// subdirectories. (https://github.com/kubernetes/kubernetes/issues/28997)
					continue
				}
				return alreadyWatching, err
			}
		}
	}

	cleanup = false
	return alreadyWatching, nil
}

func (self *rawContainerWatcher) processEvent(event *inotify.Event, events chan watcher.ContainerEvent) error {
	// Convert the inotify event type to a container create or delete.
	var eventType watcher.ContainerEventType
	switch {
	case (event.Mask & inotify.IN_CREATE) > 0:
		eventType = watcher.ContainerAdd
	case (event.Mask & inotify.IN_DELETE) > 0:
		eventType = watcher.ContainerDelete
	case (event.Mask & inotify.IN_MOVED_FROM) > 0:
		eventType = watcher.ContainerDelete
	case (event.Mask & inotify.IN_MOVED_TO) > 0:
		eventType = watcher.ContainerAdd
	default:
		// Ignore other events.
		return nil
	}

	// Derive the container name from the path name.
	var containerName string
	for _, mount := range self.cgroupSubsystems.Mounts {
		mountLocation := path.Clean(mount.Mountpoint) + "/"
		if strings.HasPrefix(event.Name, mountLocation) {
			containerName = event.Name[len(mountLocation)-1:]
			break
		}
	}
	if containerName == "" {
		return fmt.Errorf("unable to detect container from watch event on directory %q", event.Name)
	}

	// Maintain the watch for the new or deleted container.
	switch eventType {
	case watcher.ContainerAdd:
		// New container was created, watch it.
		alreadyWatched, err := self.watchDirectory(event.Name, containerName)
		if err != nil {
			return err
		}

		// Only report container creation once.
		if alreadyWatched {
			return nil
		}
	case watcher.ContainerDelete:
		// Container was deleted, stop watching for it.
		lastWatched, err := self.watcher.RemoveWatch(containerName, event.Name)
		if err != nil {
			return err
		}

		// Only report container deletion once.
		if !lastWatched {
			return nil
		}
	default:
		return fmt.Errorf("unknown event type %v", eventType)
	}

	// Deliver the event.
	events <- watcher.ContainerEvent{
		EventType:   eventType,
		Name:        containerName,
		WatchSource: watcher.Raw,
	}

	return nil
}
<|endoftext|>


<commit_before>//  Copyright (c) 2012 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package base

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/couchbaselabs/walrus"
	"github.com/dustin/gomemcached"
)

// Simple error implementation wrapping an HTTP response status.
type HTTPError struct {
	Status  int
	Message string
}

func (err *HTTPError) Error() string {
	return fmt.Sprintf("%d %s", err.Status, err.Message)
}

func HTTPErrorf(status int, format string, args ...interface{}) *HTTPError {
	return &HTTPError{status, fmt.Sprintf(format, args...)}
}

// Attempts to map an error to an HTTP status code and message.
// Defaults to 500 if it doesn't recognize the error. Returns 200 for a nil error.
func ErrorAsHTTPStatus(err error) (int, string) {
	if err == nil {
		return 200, "OK"
	}
	switch err := err.(type) {
	case *HTTPError:
		return err.Status, err.Message
	case *gomemcached.MCResponse:
		switch err.Status {
		case gomemcached.KEY_ENOENT:
			return http.StatusNotFound, "missing"
		case gomemcached.KEY_EEXISTS:
			return http.StatusConflict, "Conflict"
		case gomemcached.E2BIG:
			return http.StatusRequestEntityTooLarge, "Too Large: " + string(err.Body)
		default:
			return http.StatusBadGateway, fmt.Sprintf("%s (%s)",
				string(err.Body), err.Status.String())
		}
	case walrus.MissingError:
		return http.StatusNotFound, "missing"
	case *json.SyntaxError, *json.UnmarshalTypeError:
		return http.StatusBadRequest, fmt.Sprintf("Invalid JSON: \"%v\"", err)
	}
	Warn("Couldn't interpret error type %T, value %v", err, err)
	return http.StatusInternalServerError, fmt.Sprintf("Internal error: %v", err)
}

// Returns the standard CouchDB error string for an HTTP error status.
// These are important for compatibility, as some REST APIs don't show numeric statuses,
// only these strings.
func CouchHTTPErrorName(status int) string {
	switch status {
	case 400:
		return "bad_request"
	case 401:
		return "unauthorized"
	case 404:
		return "not_found"
	case 403:
		return "forbidden"
	case 406:
		return "not_acceptable"
	case 409:
		return "conflict"
	case 412:
		return "file_exists"
	case 415:
		return "bad_content_type"
	}
	return fmt.Sprintf("%d", status)
}

// Returns true if an error is a doc-not-found error
func IsDocNotFoundError(err error) bool {
	switch err := err.(type) {
	case *gomemcached.MCResponse:
		return err.Status == gomemcached.KEY_ENOENT || err.Status == gomemcached.NOT_STORED
	case walrus.MissingError:
		return true
	case *HTTPError:
		return err.Status == http.StatusNotFound
	default:
		return false
	}
}
<commit_msg>Handle TMPFAIL error from db server by returning HTTP 503 status<commit_after>//  Copyright (c) 2012 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package base

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/couchbaselabs/walrus"
	"github.com/dustin/gomemcached"
)

// Simple error implementation wrapping an HTTP response status.
type HTTPError struct {
	Status  int
	Message string
}

func (err *HTTPError) Error() string {
	return fmt.Sprintf("%d %s", err.Status, err.Message)
}

func HTTPErrorf(status int, format string, args ...interface{}) *HTTPError {
	return &HTTPError{status, fmt.Sprintf(format, args...)}
}

// Attempts to map an error to an HTTP status code and message.
// Defaults to 500 if it doesn't recognize the error. Returns 200 for a nil error.
func ErrorAsHTTPStatus(err error) (int, string) {
	if err == nil {
		return 200, "OK"
	}
	switch err := err.(type) {
	case *HTTPError:
		return err.Status, err.Message
	case *gomemcached.MCResponse:
		switch err.Status {
		case gomemcached.KEY_ENOENT:
			return http.StatusNotFound, "missing"
		case gomemcached.KEY_EEXISTS:
			return http.StatusConflict, "Conflict"
		case gomemcached.E2BIG:
			return http.StatusRequestEntityTooLarge, "Too Large: " + string(err.Body)
		case gomemcached.TMPFAIL:
			return http.StatusServiceUnavailable, "Database server is over capacity"
		default:
			return http.StatusBadGateway, fmt.Sprintf("%s (%s)",
				string(err.Body), err.Status.String())
		}
	case walrus.MissingError:
		return http.StatusNotFound, "missing"
	case *json.SyntaxError, *json.UnmarshalTypeError:
		return http.StatusBadRequest, fmt.Sprintf("Invalid JSON: \"%v\"", err)
	}
	Warn("Couldn't interpret error type %T, value %v", err, err)
	return http.StatusInternalServerError, fmt.Sprintf("Internal error: %v", err)
}

// Returns the standard CouchDB error string for an HTTP error status.
// These are important for compatibility, as some REST APIs don't show numeric statuses,
// only these strings.
func CouchHTTPErrorName(status int) string {
	switch status {
	case 400:
		return "bad_request"
	case 401:
		return "unauthorized"
	case 404:
		return "not_found"
	case 403:
		return "forbidden"
	case 406:
		return "not_acceptable"
	case 409:
		return "conflict"
	case 412:
		return "file_exists"
	case 415:
		return "bad_content_type"
	}
	return fmt.Sprintf("%d", status)
}

// Returns true if an error is a doc-not-found error
func IsDocNotFoundError(err error) bool {
	switch err := err.(type) {
	case *gomemcached.MCResponse:
		return err.Status == gomemcached.KEY_ENOENT || err.Status == gomemcached.NOT_STORED
	case walrus.MissingError:
		return true
	case *HTTPError:
		return err.Status == http.StatusNotFound
	default:
		return false
	}
}
<|endoftext|>


<commit_before>package logging

import (
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"sync"
	"time"

	"github.com/getlantern/rotator"

	"github.com/getlantern/appdir"
	"github.com/getlantern/flashlight/config"
	"github.com/getlantern/flashlight/globals"
	"github.com/getlantern/flashlight/util"
	"github.com/getlantern/go-loggly"
	"github.com/getlantern/golog"
	"github.com/getlantern/jibber_jabber"
	"github.com/getlantern/waitforserver"
	"github.com/getlantern/wfilter"
)

const (
	logTimestampFormat = "Jan 02 15:04:05.000"
)

var (
	log = golog.LoggerFor("flashlight")

	logFile  *rotator.SizeRotator
	cfgMutex sync.Mutex

	// logglyToken is populated at build time by crosscompile.bash. During
	// development time, logglyToken will be empty and we won't log to Loggly.
	logglyToken string

	errorOut io.Writer
	debugOut io.Writer

	lastAddr string
)

func Init() error {
	logdir := appdir.Logs("Lantern")
	log.Debugf("Placing logs in %v", logdir)
	if _, err := os.Stat(logdir); err != nil {
		if os.IsNotExist(err) {
			// Create log dir
			if err := os.MkdirAll(logdir, 0755); err != nil {
				return fmt.Errorf("Unable to create logdir at %s: %s", logdir, err)
			}
		}
	}
	logFile = rotator.NewSizeRotator(filepath.Join(logdir, "lantern.log"))
	// Set log files to 1 MB
	logFile.RotationSize = 1 * 1024 * 1024
	// Keep up to 20 log files
	logFile.MaxRotation = 20

	// Loggly has its own timestamp so don't bother adding it in message,
	// moreover, golog always write each line in whole, so we need not to care about line breaks.
	errorOut = timestamped(NonStopWriter(os.Stderr, logFile))
	debugOut = timestamped(NonStopWriter(os.Stdout, logFile))
	golog.SetOutputs(errorOut, debugOut)

	return nil
}

func Configure(cfg *config.Config, version string, buildDate string) {
	if logglyToken == "" {
		log.Debugf("No logglyToken, not sending error logs to Loggly")
		return
	}

	if version == "" {
		log.Error("No version configured, Loggly won't include version information")
		return
	}

	if buildDate == "" {
		log.Error("No build date configured, Loggly won't include build date information")
		return
	}

	cfgMutex.Lock()
	if cfg.Addr == lastAddr {
		cfgMutex.Unlock()
		log.Debug("Logging configuration unchanged")
		return
	}

	// Using a goroutine because we'll be using waitforserver and at this time
	// the proxy is not yet ready.
	go func() {
		lastAddr = cfg.Addr
		enableLoggly(cfg, version, buildDate)
		cfgMutex.Unlock()
	}()
}

func Close() error {
	golog.ResetOutputs()
	return logFile.Close()
}

// timestamped adds a timestamp to the beginning of log lines
func timestamped(orig io.Writer) io.Writer {
	return wfilter.LinePrepender(orig, func(w io.Writer) (int, error) {
		return fmt.Fprintf(w, "%s - ", time.Now().In(time.UTC).Format(logTimestampFormat))
	})
}

func enableLoggly(cfg *config.Config, version string, buildDate string) {
	if cfg.Addr == "" {
		log.Error("No known proxy, won't report to Loggly")
		removeLoggly()
		return
	}

	err := waitforserver.WaitForServer("tcp", cfg.Addr, 10*time.Second)
	if err != nil {
		log.Errorf("Proxy never came online at %v, not logging to Loggly", cfg.Addr)
		removeLoggly()
		return
	}

	var client *http.Client
	client, err = util.HTTPClient(cfg.CloudConfigCA, cfg.Addr)
	if err != nil {
		log.Errorf("Could not create proxied HTTP client, not logging to Loggly: %v", err)
		removeLoggly()
		return
	}

	log.Debugf("Sending error logs to Loggly via proxy at %v", cfg.Addr)

	lang, _ := jibber_jabber.DetectLanguage()
	logglyWriter := &logglyErrorWriter{
		lang:            lang,
		tz:              time.Now().Format("MST"),
		versionToLoggly: fmt.Sprintf("%v (%v)", version, buildDate),
		client:          loggly.New(logglyToken),
	}
	logglyWriter.client.SetHTTPClient(client)
	addLoggly(logglyWriter)
}

func addLoggly(logglyWriter io.Writer) {
	golog.SetOutputs(NonStopWriter(errorOut, logglyWriter), debugOut)
}

func removeLoggly() {
	golog.SetOutputs(errorOut, debugOut)
}

type logglyErrorWriter struct {
	lang            string
	tz              string
	versionToLoggly string
	client          *loggly.Client
}

func (w logglyErrorWriter) Write(b []byte) (int, error) {
	extra := map[string]string{
		"logLevel":  "ERROR",
		"osName":    runtime.GOOS,
		"osArch":    runtime.GOARCH,
		"osVersion": "",
		"language":  w.lang,
		"country":   globals.GetCountry(),
		"timeZone":  w.tz,
		"version":   w.versionToLoggly,
	}

	m := loggly.Message{
		"extra":   extra,
		"message": string(b),
	}

	err := w.client.Send(m)
	if err != nil {
		return 0, err
	}
	return len(b), nil
}

type nonStopWriter struct {
	writers []io.Writer
}

// NonStopWriter creates a writer that duplicates its writes to all the
// provided writers, even if errors encountered while writting.
func NonStopWriter(writers ...io.Writer) io.Writer {
	w := make([]io.Writer, len(writers))
	copy(w, writers)
	return &nonStopWriter{w}
}

// Write implements the method from io.Writer. It returns the smallest number
// of bytes written to any of the writers and the first error encountered in
// writing to any of the writers.
func (t *nonStopWriter) Write(p []byte) (int, error) {
	var fn int
	var ferr error
	first := true
	for _, w := range t.writers {
		n, err := w.Write(p)
		if first {
			fn, ferr = n, err
			first = false
		} else {
			// Use the smallest written n
			if n < fn {
				fn = n
			}
			// Use the first error encountered
			if ferr == nil && err != nil {
				ferr = err
			}
		}
	}

	if ferr == nil && fn < len(p) {
		ferr = io.ErrShortWrite
	}

	return fn, ferr
}
<commit_msg>extract prefix and truncate messages to 100 chars<commit_after>package logging

import (
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/getlantern/appdir"
	"github.com/getlantern/flashlight/config"
	"github.com/getlantern/flashlight/globals"
	"github.com/getlantern/flashlight/util"
	"github.com/getlantern/go-loggly"
	"github.com/getlantern/golog"
	"github.com/getlantern/jibber_jabber"
	"github.com/getlantern/rotator"
	"github.com/getlantern/waitforserver"
	"github.com/getlantern/wfilter"
)

const (
	logTimestampFormat = "Jan 02 15:04:05.000"
)

var (
	log = golog.LoggerFor("flashlight")

	logFile  *rotator.SizeRotator
	cfgMutex sync.Mutex

	// logglyToken is populated at build time by crosscompile.bash. During
	// development time, logglyToken will be empty and we won't log to Loggly.
	logglyToken string

	errorOut io.Writer
	debugOut io.Writer

	lastAddr string
)

func Init() error {
	logdir := appdir.Logs("Lantern")
	log.Debugf("Placing logs in %v", logdir)
	if _, err := os.Stat(logdir); err != nil {
		if os.IsNotExist(err) {
			// Create log dir
			if err := os.MkdirAll(logdir, 0755); err != nil {
				return fmt.Errorf("Unable to create logdir at %s: %s", logdir, err)
			}
		}
	}
	logFile = rotator.NewSizeRotator(filepath.Join(logdir, "lantern.log"))
	// Set log files to 1 MB
	logFile.RotationSize = 1 * 1024 * 1024
	// Keep up to 20 log files
	logFile.MaxRotation = 20

	// Loggly has its own timestamp so don't bother adding it in message,
	// moreover, golog always write each line in whole, so we need not to care about line breaks.
	errorOut = timestamped(NonStopWriter(os.Stderr, logFile))
	debugOut = timestamped(NonStopWriter(os.Stdout, logFile))
	golog.SetOutputs(errorOut, debugOut)

	return nil
}

func Configure(cfg *config.Config, version string, buildDate string) {
	if logglyToken == "" {
		log.Debugf("No logglyToken, not sending error logs to Loggly")
		return
	}

	if version == "" {
		log.Error("No version configured, Loggly won't include version information")
		return
	}

	if buildDate == "" {
		log.Error("No build date configured, Loggly won't include build date information")
		return
	}

	cfgMutex.Lock()
	if cfg.Addr == lastAddr {
		cfgMutex.Unlock()
		log.Debug("Logging configuration unchanged")
		return
	}

	// Using a goroutine because we'll be using waitforserver and at this time
	// the proxy is not yet ready.
	go func() {
		lastAddr = cfg.Addr
		enableLoggly(cfg, version, buildDate)
		cfgMutex.Unlock()
	}()
}

func Close() error {
	golog.ResetOutputs()
	return logFile.Close()
}

// timestamped adds a timestamp to the beginning of log lines
func timestamped(orig io.Writer) io.Writer {
	return wfilter.LinePrepender(orig, func(w io.Writer) (int, error) {
		return fmt.Fprintf(w, "%s - ", time.Now().In(time.UTC).Format(logTimestampFormat))
	})
}

func enableLoggly(cfg *config.Config, version string, buildDate string) {
	if cfg.Addr == "" {
		log.Error("No known proxy, won't report to Loggly")
		removeLoggly()
		return
	}

	err := waitforserver.WaitForServer("tcp", cfg.Addr, 10*time.Second)
	if err != nil {
		log.Errorf("Proxy never came online at %v, not logging to Loggly", cfg.Addr)
		removeLoggly()
		return
	}

	var client *http.Client
	client, err = util.HTTPClient(cfg.CloudConfigCA, cfg.Addr)
	if err != nil {
		log.Errorf("Could not create proxied HTTP client, not logging to Loggly: %v", err)
		removeLoggly()
		return
	}

	log.Debugf("Sending error logs to Loggly via proxy at %v", cfg.Addr)

	lang, _ := jibber_jabber.DetectLanguage()
	logglyWriter := &logglyErrorWriter{
		lang:            lang,
		tz:              time.Now().Format("MST"),
		versionToLoggly: fmt.Sprintf("%v (%v)", version, buildDate),
		client:          loggly.New(logglyToken),
	}
	logglyWriter.client.SetHTTPClient(client)
	addLoggly(logglyWriter)
}

func addLoggly(logglyWriter io.Writer) {
	golog.SetOutputs(NonStopWriter(errorOut, logglyWriter), debugOut)
}

func removeLoggly() {
	golog.SetOutputs(errorOut, debugOut)
}

type logglyErrorWriter struct {
	lang            string
	tz              string
	versionToLoggly string
	client          *loggly.Client
}

func (w logglyErrorWriter) Write(b []byte) (int, error) {
	extra := map[string]string{
		"logLevel":  "ERROR",
		"osName":    runtime.GOOS,
		"osArch":    runtime.GOARCH,
		"osVersion": "",
		"language":  w.lang,
		"country":   globals.GetCountry(),
		"timeZone":  w.tz,
		"version":   w.versionToLoggly,
	}
	fullMessage := string(b)
	message := fullMessage
	// Loggly can't index fields with more than 100 characters
	if len(fullMessage) > 100 {
		message = fullMessage[0:100]
	}
	pos := strings.IndexRune(message, ':')
	if pos == -1 {
		pos = 0
	}
	prefix := message[0:pos]
	m := loggly.Message{
		"extra":        extra,
		"locationInfo": prefix,
		"message":      message,
		"fullMessage":  fullMessage,
	}

	err := w.client.Send(m)
	if err != nil {
		return 0, err
	}
	return len(b), nil
}

type nonStopWriter struct {
	writers []io.Writer
}

// NonStopWriter creates a writer that duplicates its writes to all the
// provided writers, even if errors encountered while writting.
func NonStopWriter(writers ...io.Writer) io.Writer {
	w := make([]io.Writer, len(writers))
	copy(w, writers)
	return &nonStopWriter{w}
}

// Write implements the method from io.Writer. It returns the smallest number
// of bytes written to any of the writers and the first error encountered in
// writing to any of the writers.
func (t *nonStopWriter) Write(p []byte) (int, error) {
	var fn int
	var ferr error
	first := true
	for _, w := range t.writers {
		n, err := w.Write(p)
		if first {
			fn, ferr = n, err
			first = false
		} else {
			// Use the smallest written n
			if n < fn {
				fn = n
			}
			// Use the first error encountered
			if ferr == nil && err != nil {
				ferr = err
			}
		}
	}

	if ferr == nil && fn < len(p) {
		ferr = io.ErrShortWrite
	}

	return fn, ferr
}
<|endoftext|>


<commit_before>package main

import (
    "log"
    "math/rand"
    "time"
    "strconv"
    "flag"
    "os"
    . "os"
    "encoding/csv"
    "fmt"
)

func randomFloat() (float64){
    // seed
    r := rand.New(rand.NewSource(time.Now().UnixNano()))
    randomFloat := r.Float64()
    return randomFloat
}

func randomFloats(num int) [] string {
    var res [] string
    for i := 0; i < num; i++{
        f := strconv.FormatFloat(randomFloat()*1000, 'f', -1, 64)
        res = append(res, f)
    }
    return res
}

func main(){
    var numPairs = flag.Int("numpairs", 77, "number of cc pairs")
    var filename = flag.String("filename", "data.csv", "where to store data")
    var dsfile = flag.String("dsfile", "/Users/bjorn/stallo/data/exprs.csv",
                                "original dataset. where to get gene names")
    flag.Parse()

    // Read header from original dataset
    exprsfile, err := os.Open(*dsfile)

    if err != nil {
        log.Panic(err)
    }

    defer exprsfile.Close()

    reader := csv.NewReader(exprsfile)
    header, err := reader.Read()

    numGenes := len(header) - 1

    file, err := os.OpenFile(*filename, O_RDWR | O_CREATE, 0666)

    if err != nil {
        log.Println("Could not open or create file", err)
        return
    }

    log.Println("Writing dataset with ",numGenes,"genes and",*numPairs,
                    "cc pairs to",*filename)

    writer := csv.NewWriter(file)

    // write header to file
    writer.Write(header)

    var np int
    np = *numPairs
    for i := 0; i < *numPairs; i++ {
        p := strconv.Itoa(i)

        record := make([]string,0)
        record = append(record,p)
        record = append(record, randomFloats(numGenes)...)

        writer.Write(record)
        if err := writer.Error(); err != nil {
            log.Println(err)
        }

        p = p+"_1"
        record = make([]string,0)
        record = append(record,p)
        record = append(record, randomFloats(numGenes)...)
        writer.Write(record)

        if err := writer.Error(); err != nil {
            log.Println(err)
        }

        fmt.Print(i," of ",np," iterations done \r")
    }
    fmt.Print("\n")

    writer.Flush()

    if err := writer.Error(); err != nil {
        log.Println(err)
    }
}

// id, gene, gene, gene
// integer, gene, gene, gene
// integer_1, gene, gene, gene
// integer2, gene, gene, gene
// integer2_1, gene, gene, gene

<commit_msg>now with support for generating background dataset<commit_after>package main

import (
	"encoding/csv"
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	. "os"
	"strconv"
	"time"
)

func randomFloat() float64 {
	// seed
	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	randomFloat := r.Float64()
	return randomFloat
}

func randomInt(n int) string {
	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	num := r.Intn(n)
	res := strconv.Itoa(num)
	return res
}

func randomFloats(num int) []string {
	var res []string
	for i := 0; i < num; i++ {
		f := strconv.FormatFloat(randomFloat()*1000, 'f', -1, 64)
		res = append(res, f)
	}
	return res
}

func main() {
	var numPairs = flag.Int("numpairs", 77, "number of cc pairs")

	var exprsFilename = flag.String("exprsFilename", "exprs.csv",
		"output filename for expression data ")
	var bgFilename = flag.String("bgFilename", "background.csv",
		"output filename for background data")

	var dsfile = flag.String("dsfile", "/Users/bjorn/stallo/data/exprs.csv",
		"original dataset. where to get gene names")
	flag.Parse()

	// Read header from original dataset
	exprsfile, err := os.Open(*dsfile)

	if err != nil {
		log.Panic(err)
	}

	defer exprsfile.Close()

	reader := csv.NewReader(exprsfile)
	header, err := reader.Read()

	numGenes := len(header) - 1

	exprsFile, err := os.OpenFile(*exprsFilename, O_RDWR|O_CREATE, 0666)

	if err != nil {
		log.Println("Could not open or create file", err)
		return
	}

	bgFile, err := os.OpenFile(*bgFilename, O_RDWR|O_CREATE, 0666)

	if err != nil {
		log.Println("Could not open or create file", err)
		return
	}

	log.Println("Writing dataset with ", numGenes, "genes and", *numPairs,
		"cc pairs to", *exprsFilename, "and background information to",
		*bgFilename)

	exprsWriter := csv.NewWriter(exprsFile)
	bgWriter := csv.NewWriter(bgFile)

	// write header to file
	exprsWriter.Write(header)

	//write bg header to bg file
	bgHeader := []string{"labnr", "id", "Case_ctrl", "ET", "EN", "stage"}
	bgWriter.Write(bgHeader)

	var np int
	np = *numPairs
	for i := 0; i < *numPairs; i++ {
		p := strconv.Itoa(i)

		record := make([]string, 0)
		record = append(record, p)
		record = append(record, randomFloats(numGenes)...)

		exprsWriter.Write(record)
		if err := exprsWriter.Error(); err != nil {
			log.Println(err)
		}

		labnr := randomInt(200)
		id := p
		cc := "\"case\""
		// these numbers below are randomly picked
		et := randomInt(5)
		en := randomInt(5)
		stage := randomInt(3)

		bgrec := []string{labnr, id, cc, et, en, stage}
		bgWriter.Write(bgrec)

		p = p + "_1"
		record = make([]string, 0)
		record = append(record, p)
		record = append(record, randomFloats(numGenes)...)
		exprsWriter.Write(record)

		if err := exprsWriter.Error(); err != nil {
			log.Println(err)
		}

		labnr = randomInt(200)
		id = p
		cc = "ctrl"
		// NA for ctrl
		et = "NA"
		en = "NA"
		stage = "NA"

		bgrec = []string{labnr, id, cc, et, en, stage}
		bgWriter.Write(bgrec)

		fmt.Print(i, " of ", np, " iterations done \r")
	}
	fmt.Print("\n")

	exprsWriter.Flush()
	bgWriter.Flush()

	if err := exprsWriter.Error(); err != nil {
		log.Println(err)
	}
	if err := bgWriter.Error(); err != nil {
		log.Println(err)
	}
}

// id, gene, gene, gene
// integer, gene, gene, gene
// integer_1, gene, gene, gene
// integer2, gene, gene, gene
// integer2_1, gene, gene, gene
<|endoftext|>


<commit_before>package client

import (
	"errors"

	imap "github.com/emersion/imap/common"
	"github.com/emersion/imap/commands"
)

// Request a listing of capabilities that the server supports. Capabilities are
// often returned by the server with the greeting or with the STARTTLS and LOGIN
// responses, so usally explicitely requesting capabilities isn't needed.
func (c *Client) Capability() (caps map[string]bool, err error) {
	cmd := &commands.Capability{}

	status, err = c.execute(cmd, nil)
	if err != nil {
		return
	}

	caps = c.Caps
	err = status.Err()
	return
}

// This command always succeeds. It does nothing.
// Can be used as a periodic poll for new messages or message status updates
// during a period of inactivity. Can also be used to reset any inactivity
// autologout timer on the server.
func (c *Client) Noop() (err error) {
	cmd := &commands.Noop{}

	status, err = c.execute(cmd, nil)
	if err != nil {
		return
	}

	err = status.Err()
	return
}

// Close the connection.
func (c *Client) Logout() (err error) {
	if c.State == imap.LogoutState {
		err = errors.New("Already logged out")
		return
	}

	cmd := &commands.Logout{}

	status, err := c.execute(cmd, nil)
	if err != nil {
		return
	}

	if status != nil {
		err = status.Err()
	}
	return
}
<commit_msg>Fixes build errors<commit_after>package client

import (
	"errors"

	imap "github.com/emersion/imap/common"
	"github.com/emersion/imap/commands"
)

// Request a listing of capabilities that the server supports. Capabilities are
// often returned by the server with the greeting or with the STARTTLS and LOGIN
// responses, so usally explicitely requesting capabilities isn't needed.
func (c *Client) Capability() (caps map[string]bool, err error) {
	cmd := &commands.Capability{}

	status, err := c.execute(cmd, nil)
	if err != nil {
		return
	}

	caps = c.Caps
	err = status.Err()
	return
}

// This command always succeeds. It does nothing.
// Can be used as a periodic poll for new messages or message status updates
// during a period of inactivity. Can also be used to reset any inactivity
// autologout timer on the server.
func (c *Client) Noop() (err error) {
	cmd := &commands.Noop{}

	status, err := c.execute(cmd, nil)
	if err != nil {
		return
	}

	err = status.Err()
	return
}

// Close the connection.
func (c *Client) Logout() (err error) {
	if c.State == imap.LogoutState {
		err = errors.New("Already logged out")
		return
	}

	cmd := &commands.Logout{}

	status, err := c.execute(cmd, nil)
	if err != nil {
		return
	}

	if status != nil {
		err = status.Err()
	}
	return
}
<|endoftext|>


<commit_before>package kubo_deployment_tests_test

import (
	"fmt"
	"os/exec"
	"path/filepath"
	"strings"

	. "github.com/jhvhs/gob-mock"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
)

var _ = Describe("Generate cloud config", func() {

	var kuboEnv = filepath.Join(testEnvironmentPath, "test_gcp")

	BeforeEach(func() {
		bash.Source(pathToScript("generate_cloud_config"), nil)
		bash.Source("", func(string) ([]byte, error) {
			return repoDirectoryFunction, nil
		})
	})

	It("calls bosh-cli with appropriate arguments", func() {
		boshMock := SpyAndCallThrough("bosh-cli")
		ApplyMocks(bash, []Gob{boshMock})
		status, err := bash.Run("main", []string{kuboEnv})

		Expect(err).NotTo(HaveOccurred())
		Expect(status).To(Equal(0))
		cloudConfig := pathFromRoot("configurations/gcp/cloud-config.yml")
		boshCmd := fmt.Sprintf("bosh-cli int %s --vars-file %s/director.yml", cloudConfig, kuboEnv)
		Expect(stderr).To(gbytes.Say(boshCmd))
	})

	It("Does not include load balancer config for cf-based environment", func() {
		bash.Run("main", []string{filepath.Join(testEnvironmentPath, "test_vsphere")})

		Expect(stdout).NotTo(gbytes.Say("    target_pool: \\(\\(master_target_pool\\)\\)"))
	})

	It("includes load balancer configuration for iaas-based environment", func() {
		bash.Run("main", []string{kuboEnv})

		Expect(stdout).To(gbytes.Say("    target_pool: \\(\\(master_target_pool\\)\\)"))
	})

	It("fails with no arguments", func() {
		status, err := bash.Run("main", []string{})

		Expect(err).NotTo(HaveOccurred())
		Expect(status).To(Equal(1))
	})

	It("expands the bosh environment path to absolute value", func() {
		command := exec.Command("./generate_cloud_config", "../src/kubo-deployment-tests/resources/environments/test_gcp")
		command.Stdout = bash.Stdout
		command.Stderr = bash.Stderr
		command.Dir = pathToScript("")
		Expect(command.Run()).To(Succeed())
	})

	It("applies extra cloud config ops files when CLOUD_CONFIG_OPS_FILES variable is set", func() {
		opsFiles := []string{
			filepath.Join(resourcesPath, "ops-files", "cloud-config-plus.yml"),
			filepath.Join(resourcesPath, "ops-files", "cloud-config-plus-plus.yml"),
		}

		bash.Export("CLOUD_CONFIG_OPS_FILES", strings.Join(opsFiles, ":"))
		status, err := bash.Run("main", []string{kuboEnv})
		Expect(err).NotTo(HaveOccurred())
		Expect(status).To(Equal(0))

		Expect(stdout).To(gbytes.Say("machine_type: foo"))
		Expect(stdout).To(gbytes.Say("tags: supertest"))
	})

	It("applies extra cloud config ops files when CLOUD_CONFIG_OPS_FILES variable is set to one file", func() {
		opsFiles := []string{
			filepath.Join(resourcesPath, "ops-files", "cloud-config-plus.yml"),
		}

		bash.Export("CLOUD_CONFIG_OPS_FILES", strings.Join(opsFiles, ":"))
		status, err := bash.Run("main", []string{kuboEnv})
		Expect(err).NotTo(HaveOccurred())
		Expect(status).To(Equal(0))

		Expect(stdout).To(gbytes.Say("machine_type: foo"))
	})

	Context("On GCP", func() {
		Context("With IAAS Routing", func() {
			Context("When service_accounts for master and worker are provided", func() {
				It("adds service account to master and worker", func() {
					command := exec.Command("./generate_cloud_config", "../src/kubo-deployment-tests/resources/environments/test_gcp")
					command.Stdout = bash.Stdout
					command.Stderr = bash.Stderr
					command.Dir = pathToScript("")
					Expect(command.Run()).To(Succeed())

					masterServiceAccount, err := propertyFromYaml("/vm_types/name=master/cloud_properties/service_account", stdout.Contents())
					Expect(err).ToNot(HaveOccurred())
					Expect(masterServiceAccount).To(Equal("master-service-account@google.com"))

					workerServiceAccount, err := propertyFromYaml("/vm_types/name=worker/cloud_properties/service_account", stdout.Contents())
					Expect(err).ToNot(HaveOccurred())
					Expect(workerServiceAccount).To(Equal("worker-service-account@google.com"))
				})
			})

			Context("When service_accounts for master and worker are not provided", func() {
				It("doesn't add service account to master and worker", func() {
					command := exec.Command("./generate_cloud_config", "../src/kubo-deployment-tests/resources/environments/test_gcp_with_service_key")
					command.Stdout = bash.Stdout
					command.Stderr = bash.Stderr
					command.Dir = pathToScript("")
					Expect(command.Run()).To(Succeed())

					_, err := propertyFromYaml("/vm_types/name=master/cloud_properties/service_account", stdout.Contents())
					Expect(err).To(HaveOccurred(), "The master service account should have been removed")
					_, err = propertyFromYaml("/vm_types/name=worker/cloud_properties/service_account", stdout.Contents())
					Expect(err).To(HaveOccurred(), "The worker service account should have been removed")
				})
			})
		})
		Context("With CF Routing", func() {
			Context("When service_accounts for master and worker are provided", func() {
				It("adds service account to master and worker", func() {
					command := exec.Command("./generate_cloud_config", "../src/kubo-deployment-tests/resources/environments/test_gcp_with_cf")
					command.Stdout = bash.Stdout
					command.Stderr = bash.Stderr
					command.Dir = pathToScript("")
					Expect(command.Run()).To(Succeed())

					masterServiceAccount, err := propertyFromYaml("/vm_types/name=master/cloud_properties/service_account", stdout.Contents())
					Expect(err).ToNot(HaveOccurred())
					Expect(masterServiceAccount).To(Equal("master-service-account@google.com"))

					workerServiceAccount, err := propertyFromYaml("/vm_types/name=worker/cloud_properties/service_account", stdout.Contents())
					Expect(err).ToNot(HaveOccurred())
					Expect(workerServiceAccount).To(Equal("worker-service-account@google.com"))
				})
			})

			Context("When service_accounts for master and worker are not provided", func() {
				It("doesn't add service account to master and worker", func() {
					command := exec.Command("./generate_cloud_config", "../src/kubo-deployment-tests/resources/environments/test_gcp_with_cf_and_service_key")
					command.Stdout = bash.Stdout
					command.Stderr = bash.Stderr
					command.Dir = pathToScript("")
					Expect(command.Run()).To(Succeed())

					_, err := propertyFromYaml("/vm_types/name=master/cloud_properties/service_account", stdout.Contents())
					Expect(err).To(HaveOccurred(), "The master service account should have been removed")
					_, err = propertyFromYaml("/vm_types/name=worker/cloud_properties/service_account", stdout.Contents())
					Expect(err).To(HaveOccurred(), "The worker service account should have been removed")
				})
			})
		})
	})
})
<commit_msg>Test generate_cloud_config does not leak secrets<commit_after>package kubo_deployment_tests_test

import (
	"fmt"
	"os/exec"
	"path/filepath"
	"strings"

	. "github.com/jhvhs/gob-mock"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
)

var _ = Describe("Generate cloud config", func() {

	var kuboEnv = filepath.Join(testEnvironmentPath, "test_gcp")

	BeforeEach(func() {
		bash.Source(pathToScript("generate_cloud_config"), nil)
		bash.Source("", func(string) ([]byte, error) {
			return repoDirectoryFunction, nil
		})
	})

	It("calls bosh-cli with appropriate arguments", func() {
		boshMock := SpyAndCallThrough("bosh-cli")
		ApplyMocks(bash, []Gob{boshMock})
		status, err := bash.Run("main", []string{kuboEnv})

		Expect(err).NotTo(HaveOccurred())
		Expect(status).To(Equal(0))
		cloudConfig := pathFromRoot("configurations/gcp/cloud-config.yml")
		boshCmd := fmt.Sprintf("bosh-cli int %s --vars-file %s/director.yml", cloudConfig, kuboEnv)
		Expect(stderr).To(gbytes.Say(boshCmd))
	})

	It("Does not include load balancer config for cf-based environment", func() {
		bash.Run("main", []string{filepath.Join(testEnvironmentPath, "test_vsphere")})

		Expect(stdout).NotTo(gbytes.Say("    target_pool: \\(\\(master_target_pool\\)\\)"))
	})

	It("includes load balancer configuration for iaas-based environment", func() {
		bash.Run("main", []string{kuboEnv})

		Expect(stdout).To(gbytes.Say("    target_pool: \\(\\(master_target_pool\\)\\)"))
	})

	It("fails with no arguments", func() {
		status, err := bash.Run("main", []string{})

		Expect(err).NotTo(HaveOccurred())
		Expect(status).To(Equal(1))
	})

	It("expands the bosh environment path to absolute value", func() {
		command := exec.Command("./generate_cloud_config", "../src/kubo-deployment-tests/resources/environments/test_gcp")
		command.Stdout = bash.Stdout
		command.Stderr = bash.Stderr
		command.Dir = pathToScript("")
		Expect(command.Run()).To(Succeed())
	})

	It("applies extra cloud config ops files when CLOUD_CONFIG_OPS_FILES variable is set", func() {
		opsFiles := []string{
			filepath.Join(resourcesPath, "ops-files", "cloud-config-plus.yml"),
			filepath.Join(resourcesPath, "ops-files", "cloud-config-plus-plus.yml"),
		}

		bash.Export("CLOUD_CONFIG_OPS_FILES", strings.Join(opsFiles, ":"))
		status, err := bash.Run("main", []string{kuboEnv})
		Expect(err).NotTo(HaveOccurred())
		Expect(status).To(Equal(0))

		Expect(stdout).To(gbytes.Say("machine_type: foo"))
		Expect(stdout).To(gbytes.Say("tags: supertest"))
	})

	It("applies extra cloud config ops files when CLOUD_CONFIG_OPS_FILES variable is set to one file", func() {
		opsFiles := []string{
			filepath.Join(resourcesPath, "ops-files", "cloud-config-plus.yml"),
		}

		bash.Export("CLOUD_CONFIG_OPS_FILES", strings.Join(opsFiles, ":"))
		status, err := bash.Run("main", []string{kuboEnv})
		Expect(err).NotTo(HaveOccurred())
		Expect(status).To(Equal(0))

		Expect(stdout).To(gbytes.Say("machine_type: foo"))
	})

	It("hides cloud-config from output with debug flag", func(){
		bash.Export("DEBUG", "1")
		status, err := bash.Run("main", []string{kuboEnv})
		Expect(err).NotTo(HaveOccurred())
		Expect(status).To(Equal(0))

		Expect(stderr).NotTo(gbytes.Say("azs"))
	})

	Context("On GCP", func() {
		Context("With IAAS Routing", func() {
			Context("When service_accounts for master and worker are provided", func() {
				It("adds service account to master and worker", func() {
					command := exec.Command("./generate_cloud_config", "../src/kubo-deployment-tests/resources/environments/test_gcp")
					command.Stdout = bash.Stdout
					command.Stderr = bash.Stderr
					command.Dir = pathToScript("")
					Expect(command.Run()).To(Succeed())

					masterServiceAccount, err := propertyFromYaml("/vm_types/name=master/cloud_properties/service_account", stdout.Contents())
					Expect(err).ToNot(HaveOccurred())
					Expect(masterServiceAccount).To(Equal("master-service-account@google.com"))

					workerServiceAccount, err := propertyFromYaml("/vm_types/name=worker/cloud_properties/service_account", stdout.Contents())
					Expect(err).ToNot(HaveOccurred())
					Expect(workerServiceAccount).To(Equal("worker-service-account@google.com"))
				})
			})

			Context("When service_accounts for master and worker are not provided", func() {
				It("doesn't add service account to master and worker", func() {
					command := exec.Command("./generate_cloud_config", "../src/kubo-deployment-tests/resources/environments/test_gcp_with_service_key")
					command.Stdout = bash.Stdout
					command.Stderr = bash.Stderr
					command.Dir = pathToScript("")
					Expect(command.Run()).To(Succeed())

					_, err := propertyFromYaml("/vm_types/name=master/cloud_properties/service_account", stdout.Contents())
					Expect(err).To(HaveOccurred(), "The master service account should have been removed")
					_, err = propertyFromYaml("/vm_types/name=worker/cloud_properties/service_account", stdout.Contents())
					Expect(err).To(HaveOccurred(), "The worker service account should have been removed")
				})
			})
		})
		Context("With CF Routing", func() {
			Context("When service_accounts for master and worker are provided", func() {
				It("adds service account to master and worker", func() {
					command := exec.Command("./generate_cloud_config", "../src/kubo-deployment-tests/resources/environments/test_gcp_with_cf")
					command.Stdout = bash.Stdout
					command.Stderr = bash.Stderr
					command.Dir = pathToScript("")
					Expect(command.Run()).To(Succeed())

					masterServiceAccount, err := propertyFromYaml("/vm_types/name=master/cloud_properties/service_account", stdout.Contents())
					Expect(err).ToNot(HaveOccurred())
					Expect(masterServiceAccount).To(Equal("master-service-account@google.com"))

					workerServiceAccount, err := propertyFromYaml("/vm_types/name=worker/cloud_properties/service_account", stdout.Contents())
					Expect(err).ToNot(HaveOccurred())
					Expect(workerServiceAccount).To(Equal("worker-service-account@google.com"))
				})
			})

			Context("When service_accounts for master and worker are not provided", func() {
				It("doesn't add service account to master and worker", func() {
					command := exec.Command("./generate_cloud_config", "../src/kubo-deployment-tests/resources/environments/test_gcp_with_cf_and_service_key")
					command.Stdout = bash.Stdout
					command.Stderr = bash.Stderr
					command.Dir = pathToScript("")
					Expect(command.Run()).To(Succeed())

					_, err := propertyFromYaml("/vm_types/name=master/cloud_properties/service_account", stdout.Contents())
					Expect(err).To(HaveOccurred(), "The master service account should have been removed")
					_, err = propertyFromYaml("/vm_types/name=worker/cloud_properties/service_account", stdout.Contents())
					Expect(err).To(HaveOccurred(), "The worker service account should have been removed")
				})
			})
		})
	})
})
<|endoftext|>


<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package goobj

import (
	"debug/elf"
	"debug/macho"
	"debug/pe"
	"fmt"
	"internal/testenv"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"testing"
)

var (
	buildDir   string
	go1obj     string
	go2obj     string
	goarchive  string
	cgoarchive string
)

func TestMain(m *testing.M) {
	if !testenv.HasGoBuild() {
		return
	}

	if err := buildGoobj(); err != nil {
		fmt.Println(err)
		os.RemoveAll(buildDir)
		os.Exit(1)
	}

	exit := m.Run()

	os.RemoveAll(buildDir)
	os.Exit(exit)
}

func copyDir(dst, src string) error {
	err := os.MkdirAll(dst, 0777)
	if err != nil {
		return err
	}
	fis, err := ioutil.ReadDir(src)
	if err != nil {
		return err
	}
	for _, fi := range fis {
		err = copyFile(filepath.Join(dst, fi.Name()), filepath.Join(src, fi.Name()))
		if err != nil {
			return err
		}
	}
	return nil
}

func copyFile(dst, src string) (err error) {
	var s, d *os.File
	s, err = os.Open(src)
	if err != nil {
		return err
	}
	defer s.Close()
	d, err = os.Create(dst)
	if err != nil {
		return err
	}
	defer func() {
		e := d.Close()
		if err == nil {
			err = e
		}
	}()
	_, err = io.Copy(d, s)
	if err != nil {
		return err
	}
	return nil
}

func buildGoobj() error {
	var err error

	buildDir, err = ioutil.TempDir("", "TestGoobj")
	if err != nil {
		return err
	}

	go1obj = filepath.Join(buildDir, "go1.o")
	go2obj = filepath.Join(buildDir, "go2.o")
	goarchive = filepath.Join(buildDir, "go.a")

	gotool, err := testenv.GoTool()
	if err != nil {
		return err
	}

	go1src := filepath.Join("testdata", "go1.go")
	go2src := filepath.Join("testdata", "go2.go")

	out, err := exec.Command(gotool, "tool", "compile", "-o", go1obj, go1src).CombinedOutput()
	if err != nil {
		return fmt.Errorf("go tool compile -o %s %s: %v\n%s", go1obj, go1src, err, out)
	}
	out, err = exec.Command(gotool, "tool", "compile", "-o", go2obj, go2src).CombinedOutput()
	if err != nil {
		return fmt.Errorf("go tool compile -o %s %s: %v\n%s", go2obj, go2src, err, out)
	}
	out, err = exec.Command(gotool, "tool", "pack", "c", goarchive, go1obj, go2obj).CombinedOutput()
	if err != nil {
		return fmt.Errorf("go tool pack c %s %s %s: %v\n%s", goarchive, go1obj, go2obj, err, out)
	}

	if testenv.HasCGO() {
		gopath := filepath.Join(buildDir, "gopath")
		err = copyDir(filepath.Join(gopath, "src", "mycgo"), filepath.Join("testdata", "mycgo"))
		if err == nil {
			err = ioutil.WriteFile(filepath.Join(gopath, "src", "mycgo", "go.mod"), []byte("module mycgo\n"), 0666)
		}
		if err != nil {
			return err
		}
		cmd := exec.Command(gotool, "install", "-gcflags=all="+os.Getenv("GO_GCFLAGS"), "mycgo")
		cmd.Dir = filepath.Join(gopath, "src", "mycgo")
		cmd.Env = append(os.Environ(), "GOPATH="+gopath)
		out, err = cmd.CombinedOutput()
		if err != nil {
			return fmt.Errorf("go install mycgo: %v\n%s", err, out)
		}
		pat := filepath.Join(gopath, "pkg", "*", "mycgo.a")
		ms, err := filepath.Glob(pat)
		if err != nil {
			return err
		}
		if len(ms) == 0 {
			return fmt.Errorf("cannot found paths for pattern %s", pat)
		}
		cgoarchive = ms[0]
	}

	return nil
}

func TestParseGoobj(t *testing.T) {
	path := go1obj

	f, err := os.Open(path)
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()

	p, err := Parse(f, "mypkg")
	if err != nil {
		t.Fatal(err)
	}
	if p.Arch != runtime.GOARCH {
		t.Errorf("%s: got %v, want %v", path, p.Arch, runtime.GOARCH)
	}
	var found bool
	for _, s := range p.Syms {
		if s.Name == "mypkg.go1" {
			found = true
			break
		}
	}
	if !found {
		t.Errorf(`%s: symbol "mypkg.go1" not found`, path)
	}
}

func TestParseArchive(t *testing.T) {
	path := goarchive

	f, err := os.Open(path)
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()

	p, err := Parse(f, "mypkg")
	if err != nil {
		t.Fatal(err)
	}
	if p.Arch != runtime.GOARCH {
		t.Errorf("%s: got %v, want %v", path, p.Arch, runtime.GOARCH)
	}
	var found1 bool
	var found2 bool
	for _, s := range p.Syms {
		if s.Name == "mypkg.go1" {
			found1 = true
		}
		if s.Name == "mypkg.go2" {
			found2 = true
		}
	}
	if !found1 {
		t.Errorf(`%s: symbol "mypkg.go1" not found`, path)
	}
	if !found2 {
		t.Errorf(`%s: symbol "mypkg.go2" not found`, path)
	}
}

func TestParseCGOArchive(t *testing.T) {
	testenv.MustHaveCGO(t)

	path := cgoarchive

	f, err := os.Open(path)
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()

	p, err := Parse(f, "mycgo")
	if err != nil {
		t.Fatal(err)
	}
	if p.Arch != runtime.GOARCH {
		t.Errorf("%s: got %v, want %v", path, p.Arch, runtime.GOARCH)
	}
	var found1 bool
	var found2 bool
	for _, s := range p.Syms {
		if s.Name == "mycgo.go1" {
			found1 = true
		}
		if s.Name == "mycgo.go2" {
			found2 = true
		}
	}
	if !found1 {
		t.Errorf(`%s: symbol "mycgo.go1" not found`, path)
	}
	if !found2 {
		t.Errorf(`%s: symbol "mycgo.go2" not found`, path)
	}

	c1 := "c1"
	c2 := "c2"

	found1 = false
	found2 = false

	switch runtime.GOOS {
	case "darwin":
		c1 = "_" + c1
		c2 = "_" + c2
		for _, obj := range p.Native {
			mf, err := macho.NewFile(obj)
			if err != nil {
				t.Fatal(err)
			}
			if mf.Symtab == nil {
				continue
			}
			for _, s := range mf.Symtab.Syms {
				switch s.Name {
				case c1:
					found1 = true
				case c2:
					found2 = true
				}
			}
		}
	case "windows":
		if runtime.GOARCH == "386" {
			c1 = "_" + c1
			c2 = "_" + c2
		}
		for _, obj := range p.Native {
			pf, err := pe.NewFile(obj)
			if err != nil {
				t.Fatal(err)
			}
			for _, s := range pf.Symbols {
				switch s.Name {
				case c1:
					found1 = true
				case c2:
					found2 = true
				}
			}
		}
	default:
		for _, obj := range p.Native {
			ef, err := elf.NewFile(obj)
			if err != nil {
				t.Fatal(err)
			}
			syms, err := ef.Symbols()
			if err != nil {
				t.Fatal(err)
			}
			for _, s := range syms {
				switch s.Name {
				case c1:
					found1 = true
				case c2:
					found2 = true
				}
			}
		}
	}

	if !found1 {
		t.Errorf(`%s: symbol %q not found`, path, c1)
	}
	if !found2 {
		t.Errorf(`%s: symbol %q not found`, path, c2)
	}
}
<commit_msg>cmd/internal/goobj: add XCOFF support to TestParseCGOArchive<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package goobj

import (
	"debug/elf"
	"debug/macho"
	"debug/pe"
	"fmt"
	"internal/testenv"
	"internal/xcoff"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"testing"
)

var (
	buildDir   string
	go1obj     string
	go2obj     string
	goarchive  string
	cgoarchive string
)

func TestMain(m *testing.M) {
	if !testenv.HasGoBuild() {
		return
	}

	if err := buildGoobj(); err != nil {
		fmt.Println(err)
		os.RemoveAll(buildDir)
		os.Exit(1)
	}

	exit := m.Run()

	os.RemoveAll(buildDir)
	os.Exit(exit)
}

func copyDir(dst, src string) error {
	err := os.MkdirAll(dst, 0777)
	if err != nil {
		return err
	}
	fis, err := ioutil.ReadDir(src)
	if err != nil {
		return err
	}
	for _, fi := range fis {
		err = copyFile(filepath.Join(dst, fi.Name()), filepath.Join(src, fi.Name()))
		if err != nil {
			return err
		}
	}
	return nil
}

func copyFile(dst, src string) (err error) {
	var s, d *os.File
	s, err = os.Open(src)
	if err != nil {
		return err
	}
	defer s.Close()
	d, err = os.Create(dst)
	if err != nil {
		return err
	}
	defer func() {
		e := d.Close()
		if err == nil {
			err = e
		}
	}()
	_, err = io.Copy(d, s)
	if err != nil {
		return err
	}
	return nil
}

func buildGoobj() error {
	var err error

	buildDir, err = ioutil.TempDir("", "TestGoobj")
	if err != nil {
		return err
	}

	go1obj = filepath.Join(buildDir, "go1.o")
	go2obj = filepath.Join(buildDir, "go2.o")
	goarchive = filepath.Join(buildDir, "go.a")

	gotool, err := testenv.GoTool()
	if err != nil {
		return err
	}

	go1src := filepath.Join("testdata", "go1.go")
	go2src := filepath.Join("testdata", "go2.go")

	out, err := exec.Command(gotool, "tool", "compile", "-o", go1obj, go1src).CombinedOutput()
	if err != nil {
		return fmt.Errorf("go tool compile -o %s %s: %v\n%s", go1obj, go1src, err, out)
	}
	out, err = exec.Command(gotool, "tool", "compile", "-o", go2obj, go2src).CombinedOutput()
	if err != nil {
		return fmt.Errorf("go tool compile -o %s %s: %v\n%s", go2obj, go2src, err, out)
	}
	out, err = exec.Command(gotool, "tool", "pack", "c", goarchive, go1obj, go2obj).CombinedOutput()
	if err != nil {
		return fmt.Errorf("go tool pack c %s %s %s: %v\n%s", goarchive, go1obj, go2obj, err, out)
	}

	if testenv.HasCGO() {
		gopath := filepath.Join(buildDir, "gopath")
		err = copyDir(filepath.Join(gopath, "src", "mycgo"), filepath.Join("testdata", "mycgo"))
		if err == nil {
			err = ioutil.WriteFile(filepath.Join(gopath, "src", "mycgo", "go.mod"), []byte("module mycgo\n"), 0666)
		}
		if err != nil {
			return err
		}
		cmd := exec.Command(gotool, "install", "-gcflags=all="+os.Getenv("GO_GCFLAGS"), "mycgo")
		cmd.Dir = filepath.Join(gopath, "src", "mycgo")
		cmd.Env = append(os.Environ(), "GOPATH="+gopath)
		out, err = cmd.CombinedOutput()
		if err != nil {
			return fmt.Errorf("go install mycgo: %v\n%s", err, out)
		}
		pat := filepath.Join(gopath, "pkg", "*", "mycgo.a")
		ms, err := filepath.Glob(pat)
		if err != nil {
			return err
		}
		if len(ms) == 0 {
			return fmt.Errorf("cannot found paths for pattern %s", pat)
		}
		cgoarchive = ms[0]
	}

	return nil
}

func TestParseGoobj(t *testing.T) {
	path := go1obj

	f, err := os.Open(path)
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()

	p, err := Parse(f, "mypkg")
	if err != nil {
		t.Fatal(err)
	}
	if p.Arch != runtime.GOARCH {
		t.Errorf("%s: got %v, want %v", path, p.Arch, runtime.GOARCH)
	}
	var found bool
	for _, s := range p.Syms {
		if s.Name == "mypkg.go1" {
			found = true
			break
		}
	}
	if !found {
		t.Errorf(`%s: symbol "mypkg.go1" not found`, path)
	}
}

func TestParseArchive(t *testing.T) {
	path := goarchive

	f, err := os.Open(path)
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()

	p, err := Parse(f, "mypkg")
	if err != nil {
		t.Fatal(err)
	}
	if p.Arch != runtime.GOARCH {
		t.Errorf("%s: got %v, want %v", path, p.Arch, runtime.GOARCH)
	}
	var found1 bool
	var found2 bool
	for _, s := range p.Syms {
		if s.Name == "mypkg.go1" {
			found1 = true
		}
		if s.Name == "mypkg.go2" {
			found2 = true
		}
	}
	if !found1 {
		t.Errorf(`%s: symbol "mypkg.go1" not found`, path)
	}
	if !found2 {
		t.Errorf(`%s: symbol "mypkg.go2" not found`, path)
	}
}

func TestParseCGOArchive(t *testing.T) {
	testenv.MustHaveCGO(t)

	path := cgoarchive

	f, err := os.Open(path)
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()

	p, err := Parse(f, "mycgo")
	if err != nil {
		t.Fatal(err)
	}
	if p.Arch != runtime.GOARCH {
		t.Errorf("%s: got %v, want %v", path, p.Arch, runtime.GOARCH)
	}
	var found1 bool
	var found2 bool
	for _, s := range p.Syms {
		if s.Name == "mycgo.go1" {
			found1 = true
		}
		if s.Name == "mycgo.go2" {
			found2 = true
		}
	}
	if !found1 {
		t.Errorf(`%s: symbol "mycgo.go1" not found`, path)
	}
	if !found2 {
		t.Errorf(`%s: symbol "mycgo.go2" not found`, path)
	}

	c1 := "c1"
	c2 := "c2"

	found1 = false
	found2 = false

	switch runtime.GOOS {
	case "darwin":
		c1 = "_" + c1
		c2 = "_" + c2
		for _, obj := range p.Native {
			mf, err := macho.NewFile(obj)
			if err != nil {
				t.Fatal(err)
			}
			if mf.Symtab == nil {
				continue
			}
			for _, s := range mf.Symtab.Syms {
				switch s.Name {
				case c1:
					found1 = true
				case c2:
					found2 = true
				}
			}
		}
	case "windows":
		if runtime.GOARCH == "386" {
			c1 = "_" + c1
			c2 = "_" + c2
		}
		for _, obj := range p.Native {
			pf, err := pe.NewFile(obj)
			if err != nil {
				t.Fatal(err)
			}
			for _, s := range pf.Symbols {
				switch s.Name {
				case c1:
					found1 = true
				case c2:
					found2 = true
				}
			}
		}
	case "aix":
		c1 = "." + c1
		c2 = "." + c2
		for _, obj := range p.Native {
			xf, err := xcoff.NewFile(obj)
			if err != nil {
				t.Fatal(err)
			}
			for _, s := range xf.Symbols {
				switch s.Name {
				case c1:
					found1 = true
				case c2:
					found2 = true
				}
			}
		}

	default:
		for _, obj := range p.Native {
			ef, err := elf.NewFile(obj)
			if err != nil {
				t.Fatal(err)
			}
			syms, err := ef.Symbols()
			if err != nil {
				t.Fatal(err)
			}
			for _, s := range syms {
				switch s.Name {
				case c1:
					found1 = true
				case c2:
					found2 = true
				}
			}
		}
	}

	if !found1 {
		t.Errorf(`%s: symbol %q not found`, path, c1)
	}
	if !found2 {
		t.Errorf(`%s: symbol %q not found`, path, c2)
	}
}
<|endoftext|>