<commit_before>// Copyright 2021 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package security_test

import (
	"crypto/x509"
	"os"
	"path/filepath"
	"testing"

	seclib "v.io/x/ref/lib/security"
	"v.io/x/ref/lib/security/internal"
)

func customCertPool(t *testing.T, cafile string) *x509.CertPool {
	rf, err := os.Open(cafile)
	if err != nil {
		t.Fatal(err)
	}
	defer rf.Close()
	rootCert, err := internal.LoadCertificate(rf)
	if err != nil {
		t.Fatal(err)
	}
	certPool := x509.NewCertPool()
	certPool.AddCert(rootCert)
	return certPool
}

func TestSSLKeys(t *testing.T) {
	certPool := customCertPool(t, filepath.Join("testdata", "root-ca.pem"))
	purpose, message := []byte("testing"), []byte("a message")
	for _, tc := range []struct {
		prefix string
		pk     string
	}{
		{"ec256.vanadium.io", "7e:1f:4d:6d:99:3b:6c:51:16:90:83:cf:07:a9:a3:fc"},
		{"ed25519.vanadium.io", "c4:33:17:69:02:42:8c:19:5d:69:77:02:71:c5:1d:7e"},
		{"rsa2048.vanadium.io", "53:fb:b2:07:10:fd:9c:89:16:f5:76:4b:e8:5c:17:30"},
		{"rsa4096.vanadium.io", "10:d6:7b:2f:7d:a2:6b:96:c1:27:50:05:ce:d6:d5:26"},
	} {

		privKeyFile := tc.prefix + ".key"
		cpriv, err := seclib.ParsePEMPrivateKeyFile(filepath.Join("testdata", privKeyFile),
			nil)
		if err != nil {
			t.Errorf("failed to load %v: %v", privKeyFile, err)
		}
		signer, err := seclib.NewInMemorySigner(cpriv)
		if err != nil {
			t.Errorf("failed to create signer for %v: %v", privKeyFile, err)
		}
		sig, err := signer.Sign(purpose, message)
		if err != nil {
			t.Errorf("failed to sign using %v: %v", privKeyFile, err)
		}
		if !sig.Verify(signer.PublicKey(), message) {
			t.Errorf("failed to verify signature using %v: %v", privKeyFile, err)
		}

		opts := x509.VerifyOptions{
			Roots: certPool,
		}
		crtFile := tc.prefix + ".crt"
		cert, err := seclib.ParseOpenSSLCertificateFile(filepath.Join("testdata", crtFile), opts)
		if err != nil {
			t.Errorf("failed to load %v: %v", crtFile, err)
			continue
		}

		if got, want := cert.PublicKey.String(), tc.pk; got != want {
			t.Errorf("%v: got %v, want %v", crtFile, got, want)
		}
	}
}

func TestLetsEncryptKeys(t *testing.T) {
	filename := filepath.Join("testdata", "www.labdrive.io.letsencrypt")
	rf, err := os.Open(filename)
	if err != nil {
		t.Fatal(err)
	}
	defer rf.Close()

	purpose, message := []byte("testing"), []byte("another message")

	cpriv, err := seclib.ParsePEMPrivateKeyFile(filename, nil)
	if err != nil {
		t.Errorf("failed to load %v: %v", filename, err)
	}
	signer, err := seclib.NewInMemorySigner(cpriv)
	if err != nil {
		t.Errorf("failed to create signer for %v: %v", filename, err)
	}
	sig, err := signer.Sign(purpose, message)
	if err != nil {
		t.Errorf("failed to sign using %v: %v", filename, err)
	}
	if !sig.Verify(signer.PublicKey(), message) {
		t.Errorf("failed to verify signature using %v: %v", filename, err)
	}

	opts := x509.VerifyOptions{
		Roots: customCertPool(t, filepath.Join("testdata", "letsencrypt-stg-int-e1.pem")),
	}
	cert, err := seclib.ParseOpenSSLCertificateFile(filename, opts)
	if err != nil {
		t.Fatalf("failed to load %v: %v", filename, err)
	}

	// openssl x509 -in testdata/lwww.labdrive.io.letsencrypt --pubkey --noout |
	// openssl ec --pubin --inform PEM --outform DER |openssl md5 -c
	if got, want := cert.PublicKey.String(), "b4:1c:fc:66:5a:60:66:ea:e1:c5:46:76:59:8c:fc:6a"; got != want {
		t.Errorf("%v: got %v, want %v", filename, got, want)
	}

	// Now parse the root certificate also.
	cert, err = seclib.ParseOpenSSLCertificateFile(
		filepath.Join("testdata", "letsencrypt-stg-int-e1.pem"), opts)
	if err != nil {
		t.Fatalf("failed to load %v: %v", filename, err)
	}
	// openssl x509 -in testdata/letsencrypt-stg-int-e1.pem --pubkey --noout |
	// openssl ec --pubin --inform PEM --outform DER |openssl md5 -c
	if got, want := cert.PublicKey.String(), "8d:49:53:4b:8c:e3:7a:d5:e0:69:95:18:49:1f:7b:bf"; got != want {
		t.Errorf("%v: got %v, want %v", filename, got, want)
	}

}
<commit_msg>x/ref/lib/security: fix out of date SSL cert test (#237)<commit_after>// Copyright 2021 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package security_test

import (
	"crypto/x509"
	"os"
	"path/filepath"
	"testing"
	"time"

	seclib "v.io/x/ref/lib/security"
	"v.io/x/ref/lib/security/internal"
)

func customCertPool(t *testing.T, cafile string) *x509.CertPool {
	rf, err := os.Open(cafile)
	if err != nil {
		t.Fatal(err)
	}
	defer rf.Close()
	rootCert, err := internal.LoadCertificate(rf)
	if err != nil {
		t.Fatal(err)
	}
	certPool := x509.NewCertPool()
	certPool.AddCert(rootCert)
	return certPool
}

func TestSSLKeys(t *testing.T) {
	certPool := customCertPool(t, filepath.Join("testdata", "root-ca.pem"))
	purpose, message := []byte("testing"), []byte("a message")
	for _, tc := range []struct {
		prefix string
		pk     string
	}{
		{"ec256.vanadium.io", "7e:1f:4d:6d:99:3b:6c:51:16:90:83:cf:07:a9:a3:fc"},
		{"ed25519.vanadium.io", "c4:33:17:69:02:42:8c:19:5d:69:77:02:71:c5:1d:7e"},
		{"rsa2048.vanadium.io", "53:fb:b2:07:10:fd:9c:89:16:f5:76:4b:e8:5c:17:30"},
		{"rsa4096.vanadium.io", "10:d6:7b:2f:7d:a2:6b:96:c1:27:50:05:ce:d6:d5:26"},
	} {

		privKeyFile := tc.prefix + ".key"
		cpriv, err := seclib.ParsePEMPrivateKeyFile(filepath.Join("testdata", privKeyFile),
			nil)
		if err != nil {
			t.Errorf("failed to load %v: %v", privKeyFile, err)
		}
		signer, err := seclib.NewInMemorySigner(cpriv)
		if err != nil {
			t.Errorf("failed to create signer for %v: %v", privKeyFile, err)
		}
		sig, err := signer.Sign(purpose, message)
		if err != nil {
			t.Errorf("failed to sign using %v: %v", privKeyFile, err)
		}
		if !sig.Verify(signer.PublicKey(), message) {
			t.Errorf("failed to verify signature using %v: %v", privKeyFile, err)
		}

		opts := x509.VerifyOptions{
			Roots: certPool,
		}
		crtFile := tc.prefix + ".crt"
		cert, err := seclib.ParseOpenSSLCertificateFile(filepath.Join("testdata", crtFile), opts)
		if err != nil {
			t.Errorf("failed to load %v: %v", crtFile, err)
			continue
		}

		if got, want := cert.PublicKey.String(), tc.pk; got != want {
			t.Errorf("%v: got %v, want %v", crtFile, got, want)
		}
	}
}

func TestLetsEncryptKeys(t *testing.T) {
	filename := filepath.Join("testdata", "www.labdrive.io.letsencrypt")
	rf, err := os.Open(filename)
	if err != nil {
		t.Fatal(err)
	}
	defer rf.Close()

	purpose, message := []byte("testing"), []byte("another message")

	cpriv, err := seclib.ParsePEMPrivateKeyFile(filename, nil)
	if err != nil {
		t.Errorf("failed to load %v: %v", filename, err)
	}
	signer, err := seclib.NewInMemorySigner(cpriv)
	if err != nil {
		t.Errorf("failed to create signer for %v: %v", filename, err)
	}
	sig, err := signer.Sign(purpose, message)
	if err != nil {
		t.Errorf("failed to sign using %v: %v", filename, err)
	}
	if !sig.Verify(signer.PublicKey(), message) {
		t.Errorf("failed to verify signature using %v: %v", filename, err)
	}

	pastTime, _ := time.Parse("2006-Jan-02", "2021-Nov-02")
	opts := x509.VerifyOptions{
		Roots:       customCertPool(t, filepath.Join("testdata", "letsencrypt-stg-int-e1.pem")),
		CurrentTime: pastTime,
	}
	cert, err := seclib.ParseOpenSSLCertificateFile(filename, opts)
	if err != nil {
		t.Fatalf("failed to load %v: %v", filename, err)
	}

	// openssl x509 -in testdata/lwww.labdrive.io.letsencrypt --pubkey --noout |
	// openssl ec --pubin --inform PEM --outform DER |openssl md5 -c
	if got, want := cert.PublicKey.String(), "b4:1c:fc:66:5a:60:66:ea:e1:c5:46:76:59:8c:fc:6a"; got != want {
		t.Errorf("%v: got %v, want %v", filename, got, want)
	}

	// Now parse the root certificate also.
	cert, err = seclib.ParseOpenSSLCertificateFile(
		filepath.Join("testdata", "letsencrypt-stg-int-e1.pem"), opts)
	if err != nil {
		t.Fatalf("failed to load %v: %v", filename, err)
	}
	// openssl x509 -in testdata/letsencrypt-stg-int-e1.pem --pubkey --noout |
	// openssl ec --pubin --inform PEM --outform DER |openssl md5 -c
	if got, want := cert.PublicKey.String(), "8d:49:53:4b:8c:e3:7a:d5:e0:69:95:18:49:1f:7b:bf"; got != want {
		t.Errorf("%v: got %v, want %v", filename, got, want)
	}

}
<|endoftext|>


<commit_before>package api

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"github.com/julienschmidt/httprouter"
	"github.com/proshik/githubstatbot/telegram"
	"io"
	"log"
	"net/http"
	"strings"
)

type AccessTokenReq struct {
	ClientId     string `json:"client_id"`
	ClientSecret string `json:"client_secret"`
	Code         string `json:"code"`
}

type AccessTokenResp struct {
	AccessToken string `json:"access_token"`
	TokenType   string `json:"token_type"`
	Scope       string `json:"scope"`
}

var client = &http.Client{}

func (h *Handler) Index(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	http.ServeFile(w, r, h.staticPath+"/index.html")
}

func (h *Handler) Version(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	if checkAuth(w, r, h.basicAuth) {
		io.WriteString(w, "Version: 0.5.1")
		return
	}

	w.Header().Set("WWW-Authenticate", `Basic realm="MY REALM"`)
	w.WriteHeader(401)
	w.Write([]byte("401 Unauthorized\n"))
}

func checkAuth(w http.ResponseWriter, r *http.Request, ba *BasicAuth) bool {
	s := strings.SplitN(r.Header.Get("Authorization"), " ", 2)
	if len(s) != 2 {
		return false
	}

	b, err := base64.StdEncoding.DecodeString(s[1])
	if err != nil {
		return false
	}

	pair := strings.SplitN(string(b), ":", 2)
	if len(pair) != 2 {
		return false
	}

	return pair[0] == ba.Username && pair[1] == ba.Password
}

func (h *Handler) GitHubRedirect(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	code := r.URL.Query().Get("code")
	if code == "" {
		log.Printf("Code is empty in response from github")
		http.Redirect(w, r, telegram.RedirectBotAddress, http.StatusMovedPermanently)
		return
	}
	state := r.URL.Query().Get("state")
	if state == "" {
		log.Printf("State is empty in response from github")
		http.Redirect(w, r, telegram.RedirectBotAddress, http.StatusMovedPermanently)
		return
	}

	//check state on valid and get chatId by state
	chatId, err := h.stateStore.Get(state)
	if err != nil {
		log.Printf("Not found chatId by state. Error: %v\n", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	//delete chatId value state store
	h.stateStore.Delete(state)

	//Build request for get accessToken
	bodyReq := AccessTokenReq{h.oAuth.ClientId, h.oAuth.ClientSecret, code}
	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(bodyReq)

	req, err := http.NewRequest("POST", "https://github.com/login/oauth/access_token", b)
	req.Header.Add("Accept", "application/json")
	req.Header.Add("Content-Type", "application/json")

	resp, err := client.Do(req)
	if err != nil {
		log.Printf("Erorr on build request object. Error: %v\n", err)
		h.bot.InformAuth(chatId, false)
		http.Redirect(w, r, telegram.RedirectBotAddress, http.StatusMovedPermanently)
		return
	}

	defer resp.Body.Close()
	//decode response with accessToken
	var bodyResp AccessTokenResp
	json.NewDecoder(resp.Body).Decode(&bodyResp)

	//save token in storage
	err = h.tokenStore.Add(int64(chatId), bodyResp.AccessToken)
	if err != nil {
		log.Printf("Error on add GitHub user token in db, %v\n", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	//inform user in bot about success auth
	h.bot.InformAuth(chatId, true)
	//redirect user to bot page in telegram
	//log.Printf("Was authentication user with chatId=%d", int64(chatId))
	http.Redirect(w, r, telegram.RedirectBotAddress, http.StatusMovedPermanently)
}
<commit_msg>Fix compile errors.<commit_after>package api

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"github.com/julienschmidt/httprouter"
	"github.com/proshik/githubstatbot/telegram"
	"io"
	"log"
	"net/http"
	"strings"
)

type AccessTokenReq struct {
	ClientId     string `json:"client_id"`
	ClientSecret string `json:"client_secret"`
	Code         string `json:"code"`
}

type AccessTokenResp struct {
	AccessToken string `json:"access_token"`
	TokenType   string `json:"token_type"`
	Scope       string `json:"scope"`
}

var client = &http.Client{}

func (h *Handler) Index(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	http.ServeFile(w, r, h.staticPath+"/index.html")
}

func (h *Handler) Version(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	if checkAuth(w, r, h.basicAuth) {
		io.WriteString(w, "Version: 0.5.1")
		return
	}

	w.Header().Set("WWW-Authenticate", `Basic realm="MY REALM"`)
	w.WriteHeader(401)
	w.Write([]byte("401 Unauthorized\n"))
}

func checkAuth(w http.ResponseWriter, r *http.Request, ba *BasicAuth) bool {
	s := strings.SplitN(r.Header.Get("Authorization"), " ", 2)
	if len(s) != 2 {
		return false
	}

	b, err := base64.StdEncoding.DecodeString(s[1])
	if err != nil {
		return false
	}

	pair := strings.SplitN(string(b), ":", 2)
	if len(pair) != 2 {
		return false
	}

	return pair[0] == ba.Username && pair[1] == ba.Password
}

func (h *Handler) GitHubRedirect(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	code := r.URL.Query().Get("code")
	if code == "" {
		log.Printf("Code is empty in response from github")
		http.Redirect(w, r, telegram.RedirectBotAddress, http.StatusMovedPermanently)
		return
	}
	state := r.URL.Query().Get("state")
	if state == "" {
		log.Printf("State is empty in response from github")
		http.Redirect(w, r, telegram.RedirectBotAddress, http.StatusMovedPermanently)
		return
	}

	//check state on valid and get chatId by state
	chatId, err := h.stateStore.Get(state)
	if err != nil {
		log.Printf("Not found chatId by state. Error: %v\n", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	//delete chatId value state store
	h.stateStore.Delete(state)

	//Build request for get accessToken
	bodyReq := AccessTokenReq{h.oAuth.ClientId, h.oAuth.ClientSecret, code}
	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(bodyReq)

	req, err := http.NewRequest("POST", "https://github.com/login/oauth/access_token", b)
	req.Header.Add("Accept", "application/json")
	req.Header.Add("Content-Type", "application/json")

	resp, err := client.Do(req)
	if err != nil {
		log.Printf("Erorr on build request object. Error: %v\n", err)
		h.bot.InformAuth(chatId, false)
		http.Redirect(w, r, telegram.RedirectBotAddress, http.StatusMovedPermanently)
		return
	}

	defer resp.Body.Close()
	//decode response with accessToken
	var bodyResp AccessTokenResp
	json.NewDecoder(resp.Body).Decode(&bodyResp)

	//save token in storage
	err = h.tokenStore.Add(int64(chatId), bodyResp.AccessToken)
	if err != nil {
		log.Printf("Error on add GitHub user token in db, %v\n", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	//inform user in bot about success auth
	h.bot.InformAuth(chatId, true)
	//redirect user to bot page in telegram
	log.Printf("Was authentication user with chatId=%d", int64(chatId))
	//http.Redirect(w, r, telegram.RedirectBotAddress, http.StatusMovedPermanently)
}
<|endoftext|>


<commit_before>package api

import (
	"encoding/json"
	"net/http"
	"strings"

	"github.com/FederationOfFathers/dashboard/bridge"
	"github.com/FederationOfFathers/dashboard/streams"
	"github.com/gorilla/mux"
	"github.com/jinzhu/gorm"
	"github.com/uber-go/zap"
)

func init() {
	Router.Path("/api/v0/streams").Methods("GET").Handler(jwtHandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set("Content-Type", "application/json")
			enc := json.NewEncoder(w)
			enc.Encode(streams.Streams)
		},
	))

	Router.Path("/api/v0/streams/{memberID}/{type}").Methods("DELETE").Handler(jwtHandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set("Content-Type", "application/json")
			id := getSlackUserID(r)
			admin, _ := bridge.Data.Slack.IsUserIDAdmin(id)
			member, err := DB.MemberBySlackID(mux.Vars(r)["memberID"])
			if err != nil {
				http.NotFound(w, r)
				return
			}
			if strings.ToLower(member.Slack) != strings.ToLower(id) && !admin {
				http.NotFound(w, r)
				return
			}
			stream, err := DB.StreamByMemberID(member.ID)
			if err != nil {
				http.NotFound(w, r)
				return
			}
			if err := streams.Remove(stream.ID, mux.Vars(r)["type"]); err != nil {
				logger.Error("Error removing stream", zap.String("uri", r.URL.RawPath), zap.Error(err))
				w.WriteHeader(http.StatusInternalServerError)
			}
		},
	))

	Router.Path("/api/v0/streams/{memberID}").Methods("GET").Handler(jwtHandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set("Content-Type", "application/json")
			member, err := DB.MemberByAny(mux.Vars(r)["memberID"])
			if err == gorm.ErrRecordNotFound {
				http.NotFound(w, r)
				return
			}
			if err != nil {
				w.WriteHeader(http.StatusInternalServerError)
				return
			}
			stream, err := DB.StreamByMemberID(member.ID)
			if err == gorm.ErrRecordNotFound {
				http.NotFound(w, r)
				return
			}
			if err != nil {
				w.WriteHeader(http.StatusInternalServerError)
				return
			}
			json.NewEncoder(w).Encode(stream)
		},
	))

	Router.Path("/api/v0/streams").Methods("POST", "PUT").Handler(jwtHandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set("Content-Type", "application/json")
			kind := r.FormValue("kind")
			id := r.FormValue("id")
			userID := r.FormValue("userID")
			if userID == "" {
				userID = getSlackUserID(r)
			}
			err := streams.Add(kind, id, userID)
			if err != nil {
				logger.Error("Error adding stream", zap.String("uri", r.URL.RawPath), zap.Error(err))
				w.WriteHeader(http.StatusInternalServerError)
			}
		},
	))
}
<commit_msg>add more information to this error for debugging...<commit_after>package api

import (
	"encoding/json"
	"net/http"
	"strings"

	"github.com/FederationOfFathers/dashboard/bridge"
	"github.com/FederationOfFathers/dashboard/streams"
	"github.com/gorilla/mux"
	"github.com/jinzhu/gorm"
	"github.com/uber-go/zap"
)

func init() {
	Router.Path("/api/v0/streams").Methods("GET").Handler(jwtHandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set("Content-Type", "application/json")
			enc := json.NewEncoder(w)
			enc.Encode(streams.Streams)
		},
	))

	Router.Path("/api/v0/streams/{memberID}/{type}").Methods("DELETE").Handler(jwtHandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set("Content-Type", "application/json")
			id := getSlackUserID(r)
			admin, _ := bridge.Data.Slack.IsUserIDAdmin(id)
			member, err := DB.MemberBySlackID(mux.Vars(r)["memberID"])
			if err != nil {
				http.NotFound(w, r)
				return
			}
			if strings.ToLower(member.Slack) != strings.ToLower(id) && !admin {
				http.NotFound(w, r)
				return
			}
			stream, err := DB.StreamByMemberID(member.ID)
			if err != nil {
				http.NotFound(w, r)
				return
			}
			if err := streams.Remove(stream.ID, mux.Vars(r)["type"]); err != nil {
				logger.Error("Error removing stream", zap.String("uri", r.URL.RawPath), zap.Error(err))
				w.WriteHeader(http.StatusInternalServerError)
			}
		},
	))

	Router.Path("/api/v0/streams/{memberID}").Methods("GET").Handler(jwtHandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set("Content-Type", "application/json")
			member, err := DB.MemberByAny(mux.Vars(r)["memberID"])
			if err == gorm.ErrRecordNotFound {
				http.NotFound(w, r)
				return
			}
			if err != nil {
				w.WriteHeader(http.StatusInternalServerError)
				return
			}
			stream, err := DB.StreamByMemberID(member.ID)
			if err == gorm.ErrRecordNotFound {
				http.NotFound(w, r)
				return
			}
			if err != nil {
				w.WriteHeader(http.StatusInternalServerError)
				return
			}
			json.NewEncoder(w).Encode(stream)
		},
	))

	Router.Path("/api/v0/streams").Methods("POST", "PUT").Handler(jwtHandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set("Content-Type", "application/json")
			kind := r.FormValue("kind")
			id := r.FormValue("id")
			userID := r.FormValue("userID")
			if userID == "" {
				userID = getSlackUserID(r)
			}
			err := streams.Add(kind, id, userID)
			if err != nil {
				logger.Error("Error adding stream", zap.String("uri", r.URL.RawPath), zap.String("kind", kind), zap.String("id", id), zap.String("userID", userID), zap.Error(err))
				w.WriteHeader(http.StatusInternalServerError)
			}
		},
	))
}
<|endoftext|>


<commit_before>package entities

import (
	"fmt"
	"github.com/Vladimiroff/vec2d"
	"strconv"
	"testing"
	"time"
)

func TestGeneratePlanets(t *testing.T) {
	start_time := time.Date(2012, time.November, 10, 23, 0, 0, 0, time.UTC)
	hash := "5762908447300427353060676895795336101745023746116233389596883"
	sun_position := vec2d.New(500, 300)
	expected_planets := []Planet{
		Planet{[]int{-223, -4}, 6, 3, start_time.Unix(), 10, 0, "gophie", "avatarURL"},
		Planet{[]int{1490, 300}, 8, 5, start_time.Unix(), 10, 0, "gophie", "avatarURL"},
		Planet{[]int{-578, -153}, 3, 1, start_time.Unix(), 10, 0, "gophie", "avatarURL"},
		Planet{[]int{-797, 591}, 2, 8, start_time.Unix(), 10, 0, "gophie", "avatarURL"},
		Planet{[]int{1233, -1014}, 3, 1, start_time.Unix(), 10, 0, "gophie", "avatarURL"},
		Planet{[]int{2195, 300}, 6, 8, start_time.Unix(), 10, 0, "gophie", "avatarURL"},
		Planet{[]int{2203, -507}, 9, 6, start_time.Unix(), 10, 0, "gophie", "avatarURL"},
		Planet{[]int{-90, 2294}, 5, 4, start_time.Unix(), 10, 0, "gophie", "avatarURL"},
		Planet{[]int{1234, 2431}, 1, 1, start_time.Unix(), 10, 0, "gophie", "avatarURL"},
		Planet{[]int{-1730, -638}, 4, 6, start_time.Unix(), 10, 0, "gophie", "avatarURL"},
	}
	generated_planets, _ := GeneratePlanets(hash, sun_position)

	if len(generated_planets) != 10 {
		t.Error("Wrong planets count")
	}
	for i := 0; i < 10; i++ {
		if generated_planets[i].coords[0] != expected_planets[i].coords[0] {
			t.Error("X coordinate missmatch on Planet[" + strconv.Itoa(i) + "]")
		}
		if generated_planets[i].coords[1] != expected_planets[i].coords[1] {
			t.Error("Y coordinate missmatch on Planet[" + strconv.Itoa(i) + "]")
		}
		if generated_planets[i].Texture != expected_planets[i].Texture {
			t.Error("Ring offset missmatch on Planet[" + strconv.Itoa(i) + "]")
		}
		if generated_planets[i].Size != expected_planets[i].Size {
			t.Error("Size missmatch on Planet[" + strconv.Itoa(i) + "]")
		}
	}
}

func TestDatabasePreparationsWithoutAnOwner(t *testing.T) {
	start_time := time.Now()
	planet := Planet{[]int{271, 203}, 3, 1, start_time.Unix(), 0, 0, "", ""}
	json_base := "{\"Texture\":3,\"Size\":1,\"LastShipCountUpdate\":%v,\"ShipCount\":0,\"MaxShipCount\":0,\"Owner\":\"\",\"OwnerAvatarURL\":\"\"}"
	expected_json := fmt.Sprintf(json_base, start_time.Unix())
	expected_key := "planet.271_203"

	key, json, err := planet.Serialize()
	if key != expected_key || string(json) != expected_json {
		t.Error(string(json))
		t.Error("Planet JSON formatting gone wrong!")
	}

	if err != nil {
		t.Error("Error during serialization: ", err)
	}
}

func TestDatabasePreparationsWithAnOwner(t *testing.T) {
	start_time := time.Now()
	planet := Planet{[]int{271, 203}, 3, 1, start_time.Unix(), 0, 0, "gophie", "avatarURL"}
	json_base := "{\"Texture\":3,\"Size\":1,\"LastShipCountUpdate\":%v,\"ShipCount\":0,\"MaxShipCount\":0,\"Owner\":\"gophie\",\"OwnerAvatarURL\":\"avatarURL\"}"
	expected_json := fmt.Sprintf(json_base, start_time.Unix())
	expected_key := "planet.271_203"

	key, json, err := planet.Serialize()
	if key != expected_key || string(json) != expected_json {
		t.Error(string(json))
		t.Error(string(expected_json))
		t.Error("Planet JSON formatting gone wrong!")
	}

	if err != nil {
		t.Error("Error during serialization: ", err)
	}
}

func TestDeserializePlanet(t *testing.T) {
	var planet *Planet
	serialized_planet := []byte("{\"Texture\":3,\"Size\":1,\"LastShipCountUpdate\":1352588400,\"ShipCount\":10,\"MaxShipCount\":15,\"Owner\":\"gophie\",\"OwnerAvatarURL\":\"pngURL\"}")
	planet = Construct("planet.10_12", serialized_planet).(*Planet)

	if planet.Texture != 3 {
		t.Error("Planet's texture is ", planet.Texture)
	}

	if planet.Size != 1 {
		t.Error("Planet's tize is ", planet.Size)
	}

	if planet.ShipCount != 10 {
		t.Error("Planet's ship count is ", planet.ShipCount)
	}

	if planet.MaxShipCount != 15 {
		t.Error("Planet's max ship count is ", planet.MaxShipCount)
	}

	if planet.Owner != "gophie" {
		t.Error("Planet's owner is ", planet.Owner)
	}

	if planet.coords[0] != 10 && planet.coords[1] != 12 {
		t.Error("Planet's coords are ", planet.coords)
	}
}
<commit_msg>added test case for deserialization<commit_after>package entities

import (
	"fmt"
	"github.com/Vladimiroff/vec2d"
	"strconv"
	"testing"
	"time"
)

func TestGeneratePlanets(t *testing.T) {
	start_time := time.Date(2012, time.November, 10, 23, 0, 0, 0, time.UTC)
	hash := "5762908447300427353060676895795336101745023746116233389596883"
	sun_position := vec2d.New(500, 300)
	expected_planets := []Planet{
		Planet{[]int{-223, -4}, 6, 3, start_time.Unix(), 10, 0, "gophie", "avatarURL"},
		Planet{[]int{1490, 300}, 8, 5, start_time.Unix(), 10, 0, "gophie", "avatarURL"},
		Planet{[]int{-578, -153}, 3, 1, start_time.Unix(), 10, 0, "gophie", "avatarURL"},
		Planet{[]int{-797, 591}, 2, 8, start_time.Unix(), 10, 0, "gophie", "avatarURL"},
		Planet{[]int{1233, -1014}, 3, 1, start_time.Unix(), 10, 0, "gophie", "avatarURL"},
		Planet{[]int{2195, 300}, 6, 8, start_time.Unix(), 10, 0, "gophie", "avatarURL"},
		Planet{[]int{2203, -507}, 9, 6, start_time.Unix(), 10, 0, "gophie", "avatarURL"},
		Planet{[]int{-90, 2294}, 5, 4, start_time.Unix(), 10, 0, "gophie", "avatarURL"},
		Planet{[]int{1234, 2431}, 1, 1, start_time.Unix(), 10, 0, "gophie", "avatarURL"},
		Planet{[]int{-1730, -638}, 4, 6, start_time.Unix(), 10, 0, "gophie", "avatarURL"},
	}
	generated_planets, _ := GeneratePlanets(hash, sun_position)

	if len(generated_planets) != 10 {
		t.Error("Wrong planets count")
	}
	for i := 0; i < 10; i++ {
		if generated_planets[i].coords[0] != expected_planets[i].coords[0] {
			t.Error("X coordinate missmatch on Planet[" + strconv.Itoa(i) + "]")
		}
		if generated_planets[i].coords[1] != expected_planets[i].coords[1] {
			t.Error("Y coordinate missmatch on Planet[" + strconv.Itoa(i) + "]")
		}
		if generated_planets[i].Texture != expected_planets[i].Texture {
			t.Error("Ring offset missmatch on Planet[" + strconv.Itoa(i) + "]")
		}
		if generated_planets[i].Size != expected_planets[i].Size {
			t.Error("Size missmatch on Planet[" + strconv.Itoa(i) + "]")
		}
	}
}

func TestDatabasePreparationsWithoutAnOwner(t *testing.T) {
	start_time := time.Now()
	planet := Planet{[]int{271, 203}, 3, 1, start_time.Unix(), 0, 0, "", ""}
	json_base := "{\"Texture\":3,\"Size\":1,\"LastShipCountUpdate\":%v,\"ShipCount\":0,\"MaxShipCount\":0,\"Owner\":\"\",\"OwnerAvatarURL\":\"\"}"
	expected_json := fmt.Sprintf(json_base, start_time.Unix())
	expected_key := "planet.271_203"

	key, json, err := planet.Serialize()
	if key != expected_key || string(json) != expected_json {
		t.Error(string(json))
		t.Error("Planet JSON formatting gone wrong!")
	}

	if err != nil {
		t.Error("Error during serialization: ", err)
	}
}

func TestDatabasePreparationsWithAnOwner(t *testing.T) {
	start_time := time.Now()
	planet := Planet{[]int{271, 203}, 3, 1, start_time.Unix(), 0, 0, "gophie", "avatarURL"}
	json_base := "{\"Texture\":3,\"Size\":1,\"LastShipCountUpdate\":%v,\"ShipCount\":0,\"MaxShipCount\":0,\"Owner\":\"gophie\",\"OwnerAvatarURL\":\"avatarURL\"}"
	expected_json := fmt.Sprintf(json_base, start_time.Unix())
	expected_key := "planet.271_203"

	key, json, err := planet.Serialize()
	if key != expected_key || string(json) != expected_json {
		t.Error(string(json))
		t.Error(string(expected_json))
		t.Error("Planet JSON formatting gone wrong!")
	}

	if err != nil {
		t.Error("Error during serialization: ", err)
	}
}

func TestDeserializePlanet(t *testing.T) {
	var planet *Planet
	serialized_planet := []byte("{\"Texture\":3,\"Size\":1,\"LastShipCountUpdate\":1352588400,\"ShipCount\":10,\"MaxShipCount\":15,\"Owner\":\"gophie\",\"OwnerAvatarURL\":\"pngURL\"}")
	planet = Construct("planet.10_12", serialized_planet).(*Planet)

	if planet.Texture != 3 {
		t.Error("Planet's texture is ", planet.Texture)
	}

	if planet.Size != 1 {
		t.Error("Planet's tize is ", planet.Size)
	}

	if planet.ShipCount != 10 {
		t.Error("Planet's ship count is ", planet.ShipCount)
	}

	if planet.MaxShipCount != 15 {
		t.Error("Planet's max ship count is ", planet.MaxShipCount)
	}

	if planet.Owner != "gophie" {
		t.Error("Planet's owner is ", planet.Owner)
	}

	if planet.OwnerAvatarURL != "pngURL" {
		t.Error("Planet's owner avatar url is ", planet.OwnerAvatarURL)
	}
	if planet.coords[0] != 10 && planet.coords[1] != 12 {
		t.Error("Planet's coords are ", planet.coords)
	}
}
<|endoftext|>


<commit_before>// Virtua SmartOS Prometheus exporter
//
// Worflow :
//  - detect if launched in GZ or into a Zone
//  - retrieve useful metrics
//  - expose them to http://xxx:9100/metrics (same as node_exporter)

package main

import (
        "log"
        "net/http"
        "os/exec"
        "strconv"
        "strings"
//      "fmt"
        // Prometheus Go toolset
        "github.com/prometheus/client_golang/prometheus"
        "github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
    // Global variables
    exporterPort = ":9100"
    // Metrics definitions
    gzCpuUsage = prometheus.NewGaugeVec(prometheus.GaugeOpts{
        Name: "smartos_gz_cpu_usage_total",
        Help: "CPU usage exposed in percent.",
        },
        []string{"cpu","type"},
    )
    gzFreeMem = prometheus.NewGaugeVec(prometheus.GaugeOpts{
        Name: "smartos_gz_memory_free_bytes_total",
        Help: "Total free memory (both RAM and Swap) of the CN.",
        },
        []string{"type"},
    )
    gzMlagUsage = prometheus.NewGaugeVec(prometheus.GaugeOpts{
        Name: "smartos_gz_network_mlag_bytes_total",
        Help: "MLAG (aggr0) usage of the CN.",
        },
        []string{"device","type"},
    )
    gzZpoolList = prometheus.NewGaugeVec(prometheus.GaugeOpts{
        Name: "smartos_gz_zpool_list_total",
        Help: "ZFS zpool list summary.",
        },
        []string{"zpool","type"},
    )
    gzDiskErrors = prometheus.NewCounterVec(prometheus.CounterOpts{
        Name: "smartos_gz_disk_errors_total",
        Help: "Number of hardware disk errors.",
        },
        []string{"device","type"},
    )
)

// program start

func init() {
    // Metrics have to be registered to be exposed:
    gz := isGlobalZone()
    if gz == 1 {
        prometheus.MustRegister(gzCpuUsage)
        prometheus.MustRegister(gzFreeMem)
        prometheus.MustRegister(gzMlagUsage)
        prometheus.MustRegister(gzDiskErrors)
        prometheus.MustRegister(gzZpoolList)
    } else {
        // not yet implemented
        log.Fatal("zone statistics gathering is not yet implemented.")
    }
}

// Global Helpers

// try to determine if its executed inside the GZ or not.
// return 1 if in GZ
//        0 if in zone
func isGlobalZone() (int) {
    out, eerr := exec.Command("zonename").Output()
    if eerr != nil {
        log.Fatal(eerr)
    }
    if (strings.Contains(string(out), "global")) == false {
        return 0
    } else {
        return 1
    }
}

// SmartOS command / tool callers
func iostat() {
    out, eerr := exec.Command("iostat", "-en").Output()
    if eerr != nil {
        log.Fatal(eerr)
    }
    perr := parseIostatOutput(string(out))
    if perr != nil {
        log.Fatal(perr)
    }
}

func mpstat() {
    out, eerr := exec.Command("mpstat", "1", "1").Output()
    if eerr != nil {
        log.Fatal(eerr)
    }
    perr := parseMpstatOutput(string(out))
    if perr != nil {
        log.Fatal(perr)
    }
}

func nicstat() {
    out, eerr := exec.Command("nicstat", "-i", "aggr0").Output()
    if eerr != nil {
        log.Fatal(eerr)
    }
    perr := parseNicstatOutput(string(out))
    if perr != nil {
        log.Fatal(perr)
    }
}

func vmstat() {
    out, eerr := exec.Command("vmstat", "1", "1").Output()
    if eerr != nil {
        log.Fatal(eerr)
    }
    perr := parseVmstatOutput(string(out))
    if perr != nil {
        log.Fatal(perr)
    }
}

func zpoolList() {
    out, eerr := exec.Command("zpool", "list", "-p", "zones").Output()
    if eerr != nil {
        log.Fatal(eerr)
    }
    perr := parseZpoolStatusOutput(string(out))
    if perr != nil {
        log.Fatal(perr)
    }
}

// Parsers

func parseIostatOutput(out string) (error) {
    outlines := strings.Split(out, "\n")
    l := len(outlines)
    for _, line := range outlines[2:l-1] {
        parsedLine := strings.Fields(line)
        deviceName := parsedLine[4]
        softErr, err := strconv.ParseFloat(parsedLine[0], 64)
        if err != nil {
            return err
        }
        hardErr, err := strconv.ParseFloat(parsedLine[1], 64)
        if err != nil {
            return err
        }
        trnErr, err := strconv.ParseFloat(parsedLine[2], 64)
        if err != nil {
            return err
        }
        gzDiskErrors.With(prometheus.Labels{"device":deviceName,"type":"soft"}).Add(softErr)
        gzDiskErrors.With(prometheus.Labels{"device":deviceName,"type":"hard"}).Add(hardErr)
        gzDiskErrors.With(prometheus.Labels{"device":deviceName,"type":"trn"}).Add(trnErr)
    }
    return nil
}

func parseMpstatOutput(out string) (error) {
    outlines := strings.Split(out, "\n")
    l := len(outlines)
    for _, line := range outlines[1:l-1] {
        parsedLine := strings.Fields(line)
        cpuId := parsedLine[0]
        cpuUsr, err := strconv.ParseFloat(parsedLine[12], 64)
        if err != nil {
            return err
        }
        cpuSys, err := strconv.ParseFloat(parsedLine[13], 64)
        if err != nil {
            return err
        }
        cpuIdl, err := strconv.ParseFloat(parsedLine[15], 64)
        if err != nil {
            return err
        }
        gzCpuUsage.With(prometheus.Labels{"cpu": cpuId, "type":"user"}).Set(cpuUsr)
        gzCpuUsage.With(prometheus.Labels{"cpu": cpuId, "type":"system"}).Set(cpuSys)
        gzCpuUsage.With(prometheus.Labels{"cpu": cpuId, "type":"idle"}).Set(cpuIdl)
        //fmt.Printf("cpuId : %d, cpuUsr : %d, cpuSys : %d \n", cpuId, cpuUsr, cpuSys)
    }
    return nil
}

func parseNicstatOutput(out string) (error) {
    outlines := strings.Split(out, "\n")
    l := len(outlines)
    for _, line := range outlines[1:l-1] {
        parsedLine := strings.Fields(line)
        readKb, err := strconv.ParseFloat(parsedLine[2], 64)
        if err != nil {
            return err
        }
        writeKb, err := strconv.ParseFloat(parsedLine[3], 64)
        if err != nil {
            return err
        }
        gzMlagUsage.With(prometheus.Labels{"device":"aggr0", "type":"read"}).Set(readKb)
        gzMlagUsage.With(prometheus.Labels{"device":"aggr0", "type":"write"}).Set(writeKb)
    }
    return nil
}

func parseVmstatOutput(out string) (error) {
    outlines := strings.Split(out, "\n")
    l := len(outlines)
    for _, line := range outlines[2:l-1] {
        parsedLine := strings.Fields(line)
        freeSwap, err := strconv.ParseFloat(parsedLine[3], 64)
        if err != nil {
            return err
        }
        freeRam, err := strconv.ParseFloat(parsedLine[4], 64)
        if err != nil {
            return err
        }
        gzFreeMem.With(prometheus.Labels{"type":"swap"}).Set(freeSwap)
        gzFreeMem.With(prometheus.Labels{"type":"ram"}).Set(freeRam)
    }
    return nil
}

func parseZpoolStatusOutput(out string) (error) {
    outlines := strings.Split(out, "\n")
    l := len(outlines)
    for _, line := range outlines[1:l-1] {
        parsedLine := strings.Fields(line)
        sizeBytes, err := strconv.ParseFloat(parsedLine[1], 64)
        if err != nil {
            return err
        }
        allocBytes, err := strconv.ParseFloat(parsedLine[2], 64)
        if err != nil {
            return err
        }
        freeBytes, err := strconv.ParseFloat(parsedLine[3], 64)
        if err != nil {
            return err
        }
        fragPercent := strings.TrimSuffix(parsedLine[5], "%")
        fragPercentTrim, err := strconv.ParseFloat(fragPercent, 64)
        if err != nil {
            return err
        }
        capPercent := strings.TrimSuffix(parsedLine[6], "%")
        capPercentTrim, err := strconv.ParseFloat(capPercent, 64)
        if err != nil {
            return err
        }
        health := parsedLine[8]
        if (strings.Contains(health, "ONLINE")) == true {
          gzZpoolList.With(prometheus.Labels{"zpool":"zones", "type":"faulty"}).Set(0)
        } else {
          gzZpoolList.With(prometheus.Labels{"zpool":"zones", "type":"faulty"}).Set(1)
        }

        gzZpoolList.With(prometheus.Labels{"zpool":"zones", "type":"size"}).Set(sizeBytes)
        gzZpoolList.With(prometheus.Labels{"zpool":"zones", "type":"alloc"}).Set(allocBytes)
        gzZpoolList.With(prometheus.Labels{"zpool":"zones", "type":"free"}).Set(freeBytes)
        gzZpoolList.With(prometheus.Labels{"zpool":"zones", "type":"frag"}).Set(fragPercentTrim)
        gzZpoolList.With(prometheus.Labels{"zpool":"zones", "type":"capacity"}).Set(capPercentTrim)
    }
    return nil
}

// program starter

func init() {
    // Metrics have to be registered to be exposed:
    gz := isGlobalZone()
    if gz == 1 {
        prometheus.MustRegister(gzCpuUsage)
        prometheus.MustRegister(gzFreeMem)
        prometheus.MustRegister(gzMlagUsage)
        prometheus.MustRegister(gzDiskErrors)
        prometheus.MustRegister(gzZpoolList)
    } else {
        // not yet implemented
        // XXX
        log.Fatal("zone statistics gathering is not yet implemented.")
    }
}

func main() {
    gz := isGlobalZone()
    if gz == 1 {
        mpstat()
        vmstat()
        nicstat()
        iostat()
        zpoolList()
    } else {
        // not yet implemented
        // XXX
        log.Fatal("zone statistics gathering is not yet implemented.")
    }

    // The Handler function provides a default handler to expose metrics
    // via an HTTP server. "/metrics" is the usual endpoint for that.
    http.Handle("/metrics", promhttp.Handler())
    log.Fatal(http.ListenAndServe(exporterPort, nil))
}
<commit_msg>Fix duplicated bloc<commit_after>// Virtua SmartOS Prometheus exporter
//
// Worflow :
//  - detect if launched in GZ or into a Zone
//  - retrieve useful metrics
//  - expose them to http://xxx:9100/metrics (same as node_exporter)

package main

import (
        "log"
        "net/http"
        "os/exec"
        "strconv"
        "strings"
//      "fmt"
        // Prometheus Go toolset
        "github.com/prometheus/client_golang/prometheus"
        "github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
    // Global variables
    exporterPort = ":9100"
    // Metrics definitions
    gzCpuUsage = prometheus.NewGaugeVec(prometheus.GaugeOpts{
        Name: "smartos_gz_cpu_usage_total",
        Help: "CPU usage exposed in percent.",
        },
        []string{"cpu","type"},
    )
    gzFreeMem = prometheus.NewGaugeVec(prometheus.GaugeOpts{
        Name: "smartos_gz_memory_free_bytes_total",
        Help: "Total free memory (both RAM and Swap) of the CN.",
        },
        []string{"type"},
    )
    gzMlagUsage = prometheus.NewGaugeVec(prometheus.GaugeOpts{
        Name: "smartos_gz_network_mlag_bytes_total",
        Help: "MLAG (aggr0) usage of the CN.",
        },
        []string{"device","type"},
    )
    gzZpoolList = prometheus.NewGaugeVec(prometheus.GaugeOpts{
        Name: "smartos_gz_zpool_list_total",
        Help: "ZFS zpool list summary.",
        },
        []string{"zpool","type"},
    )
    gzDiskErrors = prometheus.NewCounterVec(prometheus.CounterOpts{
        Name: "smartos_gz_disk_errors_total",
        Help: "Number of hardware disk errors.",
        },
        []string{"device","type"},
    )
)

// Global Helpers

// try to determine if its executed inside the GZ or not.
// return 1 if in GZ
//        0 if in zone
func isGlobalZone() (int) {
    out, eerr := exec.Command("zonename").Output()
    if eerr != nil {
        log.Fatal(eerr)
    }
    if (strings.Contains(string(out), "global")) == false {
        return 0
    } else {
        return 1
    }
}

// SmartOS command / tool callers
func iostat() {
    out, eerr := exec.Command("iostat", "-en").Output()
    if eerr != nil {
        log.Fatal(eerr)
    }
    perr := parseIostatOutput(string(out))
    if perr != nil {
        log.Fatal(perr)
    }
}

func mpstat() {
    out, eerr := exec.Command("mpstat", "1", "1").Output()
    if eerr != nil {
        log.Fatal(eerr)
    }
    perr := parseMpstatOutput(string(out))
    if perr != nil {
        log.Fatal(perr)
    }
}

func nicstat() {
    out, eerr := exec.Command("nicstat", "-i", "aggr0").Output()
    if eerr != nil {
        log.Fatal(eerr)
    }
    perr := parseNicstatOutput(string(out))
    if perr != nil {
        log.Fatal(perr)
    }
}

func vmstat() {
    out, eerr := exec.Command("vmstat", "1", "1").Output()
    if eerr != nil {
        log.Fatal(eerr)
    }
    perr := parseVmstatOutput(string(out))
    if perr != nil {
        log.Fatal(perr)
    }
}

func zpoolList() {
    out, eerr := exec.Command("zpool", "list", "-p", "zones").Output()
    if eerr != nil {
        log.Fatal(eerr)
    }
    perr := parseZpoolStatusOutput(string(out))
    if perr != nil {
        log.Fatal(perr)
    }
}

// Parsers

func parseIostatOutput(out string) (error) {
    outlines := strings.Split(out, "\n")
    l := len(outlines)
    for _, line := range outlines[2:l-1] {
        parsedLine := strings.Fields(line)
        deviceName := parsedLine[4]
        softErr, err := strconv.ParseFloat(parsedLine[0], 64)
        if err != nil {
            return err
        }
        hardErr, err := strconv.ParseFloat(parsedLine[1], 64)
        if err != nil {
            return err
        }
        trnErr, err := strconv.ParseFloat(parsedLine[2], 64)
        if err != nil {
            return err
        }
        gzDiskErrors.With(prometheus.Labels{"device":deviceName,"type":"soft"}).Add(softErr)
        gzDiskErrors.With(prometheus.Labels{"device":deviceName,"type":"hard"}).Add(hardErr)
        gzDiskErrors.With(prometheus.Labels{"device":deviceName,"type":"trn"}).Add(trnErr)
    }
    return nil
}

func parseMpstatOutput(out string) (error) {
    outlines := strings.Split(out, "\n")
    l := len(outlines)
    for _, line := range outlines[1:l-1] {
        parsedLine := strings.Fields(line)
        cpuId := parsedLine[0]
        cpuUsr, err := strconv.ParseFloat(parsedLine[12], 64)
        if err != nil {
            return err
        }
        cpuSys, err := strconv.ParseFloat(parsedLine[13], 64)
        if err != nil {
            return err
        }
        cpuIdl, err := strconv.ParseFloat(parsedLine[15], 64)
        if err != nil {
            return err
        }
        gzCpuUsage.With(prometheus.Labels{"cpu": cpuId, "type":"user"}).Set(cpuUsr)
        gzCpuUsage.With(prometheus.Labels{"cpu": cpuId, "type":"system"}).Set(cpuSys)
        gzCpuUsage.With(prometheus.Labels{"cpu": cpuId, "type":"idle"}).Set(cpuIdl)
        //fmt.Printf("cpuId : %d, cpuUsr : %d, cpuSys : %d \n", cpuId, cpuUsr, cpuSys)
    }
    return nil
}

func parseNicstatOutput(out string) (error) {
    outlines := strings.Split(out, "\n")
    l := len(outlines)
    for _, line := range outlines[1:l-1] {
        parsedLine := strings.Fields(line)
        readKb, err := strconv.ParseFloat(parsedLine[2], 64)
        if err != nil {
            return err
        }
        writeKb, err := strconv.ParseFloat(parsedLine[3], 64)
        if err != nil {
            return err
        }
        gzMlagUsage.With(prometheus.Labels{"device":"aggr0", "type":"read"}).Set(readKb)
        gzMlagUsage.With(prometheus.Labels{"device":"aggr0", "type":"write"}).Set(writeKb)
    }
    return nil
}

func parseVmstatOutput(out string) (error) {
    outlines := strings.Split(out, "\n")
    l := len(outlines)
    for _, line := range outlines[2:l-1] {
        parsedLine := strings.Fields(line)
        freeSwap, err := strconv.ParseFloat(parsedLine[3], 64)
        if err != nil {
            return err
        }
        freeRam, err := strconv.ParseFloat(parsedLine[4], 64)
        if err != nil {
            return err
        }
        gzFreeMem.With(prometheus.Labels{"type":"swap"}).Set(freeSwap)
        gzFreeMem.With(prometheus.Labels{"type":"ram"}).Set(freeRam)
    }
    return nil
}

func parseZpoolStatusOutput(out string) (error) {
    outlines := strings.Split(out, "\n")
    l := len(outlines)
    for _, line := range outlines[1:l-1] {
        parsedLine := strings.Fields(line)
        sizeBytes, err := strconv.ParseFloat(parsedLine[1], 64)
        if err != nil {
            return err
        }
        allocBytes, err := strconv.ParseFloat(parsedLine[2], 64)
        if err != nil {
            return err
        }
        freeBytes, err := strconv.ParseFloat(parsedLine[3], 64)
        if err != nil {
            return err
        }
        fragPercent := strings.TrimSuffix(parsedLine[5], "%")
        fragPercentTrim, err := strconv.ParseFloat(fragPercent, 64)
        if err != nil {
            return err
        }
        capPercent := strings.TrimSuffix(parsedLine[6], "%")
        capPercentTrim, err := strconv.ParseFloat(capPercent, 64)
        if err != nil {
            return err
        }
        health := parsedLine[8]
        if (strings.Contains(health, "ONLINE")) == true {
          gzZpoolList.With(prometheus.Labels{"zpool":"zones", "type":"faulty"}).Set(0)
        } else {
          gzZpoolList.With(prometheus.Labels{"zpool":"zones", "type":"faulty"}).Set(1)
        }

        gzZpoolList.With(prometheus.Labels{"zpool":"zones", "type":"size"}).Set(sizeBytes)
        gzZpoolList.With(prometheus.Labels{"zpool":"zones", "type":"alloc"}).Set(allocBytes)
        gzZpoolList.With(prometheus.Labels{"zpool":"zones", "type":"free"}).Set(freeBytes)
        gzZpoolList.With(prometheus.Labels{"zpool":"zones", "type":"frag"}).Set(fragPercentTrim)
        gzZpoolList.With(prometheus.Labels{"zpool":"zones", "type":"capacity"}).Set(capPercentTrim)
    }
    return nil
}

// program starter

func init() {
    // Metrics have to be registered to be exposed:
    gz := isGlobalZone()
    if gz == 1 {
        prometheus.MustRegister(gzCpuUsage)
        prometheus.MustRegister(gzFreeMem)
        prometheus.MustRegister(gzMlagUsage)
        prometheus.MustRegister(gzDiskErrors)
        prometheus.MustRegister(gzZpoolList)
    } else {
        // not yet implemented
        // XXX
        log.Fatal("zone statistics gathering is not yet implemented.")
    }
}

func main() {
    gz := isGlobalZone()
    if gz == 1 {
        mpstat()
        vmstat()
        nicstat()
        iostat()
        zpoolList()
    } else {
        // not yet implemented
        // XXX
        log.Fatal("zone statistics gathering is not yet implemented.")
    }

    // The Handler function provides a default handler to expose metrics
    // via an HTTP server. "/metrics" is the usual endpoint for that.
    http.Handle("/metrics", promhttp.Handler())
    log.Fatal(http.ListenAndServe(exporterPort, nil))
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gce

import (
	"fmt"
	"sync"
	"time"

	"k8s.io/contrib/cluster-autoscaler/config"
	gceurl "k8s.io/contrib/cluster-autoscaler/utils/gce_url"

	"github.com/golang/glog"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	gce "google.golang.org/api/compute/v1"
	"k8s.io/kubernetes/pkg/util/wait"
)

type gceManager struct {
	migs       []*config.MigConfig
	service    *gce.Service
	migCache   map[config.InstanceConfig]*config.MigConfig
	cacheMutex sync.Mutex
}

// CreateGceManager constructs gceManager object.
func CreateGceManager(migs []*config.MigConfig) (*gceManager, error) {
	// Create Google Compute Engine service.
	client := oauth2.NewClient(oauth2.NoContext, google.ComputeTokenSource(""))
	gceService, err := gce.New(client)
	if err != nil {
		return nil, err
	}

	manager := &gceManager{
		migs:     migs,
		service:  gceService,
		migCache: map[config.InstanceConfig]*config.MigConfig{},
	}

	go wait.Forever(func() { manager.regenerateCacheIgnoreError() }, time.Hour)

	return manager, nil
}

func (m *gceManager) GetMigSize(migConf *config.MigConfig) (int64, error) {
	mig, err := m.service.InstanceGroupManagers.Get(migConf.Project, migConf.Zone, migConf.Name).Do()
	if err != nil {
		return -1, err
	}
	return mig.TargetSize, nil
}

func (m *gceManager) SetMigSize(migConf *config.MigConfig, size int64) error {
	op, err := m.service.InstanceGroupManagers.Resize(migConf.Project, migConf.Zone, migConf.Name, size).Do()
	if err != nil {
		return err
	}
	if err := m.waitForOp(op); err != nil {
		return err
	}
	return nil
}

func (m *gceManager) waitForOp(op *gce.Operation) error {
	return nil
}

// All instances must be controlled by the same MIG.
func (m *gceManager) DeleteInstances(instances []*config.InstanceConfig) error {
	if len(instances) == 0 {
		return nil
	}
	commonMig, err := m.GetMigForInstance(instances[0])
	if err != nil {
		return err
	}
	for _, instance := range instances {
		mig, err := m.GetMigForInstance(instance)
		if err != nil {
			return err
		}
		if mig != commonMig {
			return fmt.Errorf("Connot delete instances which don't belong to the same MIG.")
		}
	}

	req := gce.InstanceGroupManagersDeleteInstancesRequest{
		Instances: []string{},
	}
	for _, instance := range instances {
		req.Instances = append(req.Instances, gceurl.GenerateInstanceUrl(instance.Project, instance.Zone, instance.Name))
	}

	op, err := m.service.InstanceGroupManagers.DeleteInstances(commonMig.Project, commonMig.Zone, commonMig.Name, &req).Do()
	if err != nil {
		return err
	}
	if err := m.waitForOp(op); err != nil {
		return err
	}
	return nil
}

func (m *gceManager) GetMigForInstance(instance *config.InstanceConfig) (*config.MigConfig, error) {
	m.cacheMutex.Lock()
	defer m.cacheMutex.Unlock()
	if mig, found := m.migCache[*instance]; found {
		return mig, nil
	}
	if err := m.regenerateCache(); err != nil {
		return nil, fmt.Errorf("Error while looking for MIG for instance %+v, error: %v", *instance, err)
	}
	if mig, found := m.migCache[*instance]; found {
		return mig, nil
	}
	return nil, fmt.Errorf("Instance %+v does not belong to any known MIG", *instance)
}

func (m *gceManager) regenerateCacheIgnoreError() {
	m.cacheMutex.Lock()
	defer m.cacheMutex.Unlock()
	if err := m.regenerateCache(); err != nil {
		glog.Errorf("Error while regenerating Mig cache: %v", err)
	}
}

func (m *gceManager) regenerateCache() error {
	newMigCache := map[config.InstanceConfig]*config.MigConfig{}

	for _, mig := range m.migs {
		instances, err := m.service.InstanceGroupManagers.ListManagedInstances(mig.Project, mig.Zone, mig.Name).Do()
		if err != nil {
			return err
		}
		for _, instance := range instances.ManagedInstances {
			project, zone, name, err := gceurl.ParseInstanceUrl(instance.Instance)
			if err != nil {
				return err
			}
			newMigCache[config.InstanceConfig{Project: project, Zone: zone, Name: name}] = mig
		}
	}

	m.migCache = newMigCache
	return nil
}
<commit_msg>Cluster-autoscaler: Implemented waiting for operation in GCE Manager<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gce

import (
	"fmt"
	"sync"
	"time"

	"k8s.io/contrib/cluster-autoscaler/config"
	gceurl "k8s.io/contrib/cluster-autoscaler/utils/gce_url"

	"github.com/golang/glog"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	gce "google.golang.org/api/compute/v1"
	"k8s.io/kubernetes/pkg/util/wait"
)

const (
	operationWaitTimeout  = 5 * time.Second
	operationPollInterval = 100 * time.Millisecond
)

type gceManager struct {
	migs       []*config.MigConfig
	service    *gce.Service
	migCache   map[config.InstanceConfig]*config.MigConfig
	cacheMutex sync.Mutex
}

// CreateGceManager constructs gceManager object.
func CreateGceManager(migs []*config.MigConfig) (*gceManager, error) {
	// Create Google Compute Engine service.
	client := oauth2.NewClient(oauth2.NoContext, google.ComputeTokenSource(""))
	gceService, err := gce.New(client)
	if err != nil {
		return nil, err
	}

	manager := &gceManager{
		migs:     migs,
		service:  gceService,
		migCache: map[config.InstanceConfig]*config.MigConfig{},
	}

	go wait.Forever(func() { manager.regenerateCacheIgnoreError() }, time.Hour)

	return manager, nil
}

func (m *gceManager) GetMigSize(migConf *config.MigConfig) (int64, error) {
	mig, err := m.service.InstanceGroupManagers.Get(migConf.Project, migConf.Zone, migConf.Name).Do()
	if err != nil {
		return -1, err
	}
	return mig.TargetSize, nil
}

func (m *gceManager) SetMigSize(migConf *config.MigConfig, size int64) error {
	op, err := m.service.InstanceGroupManagers.Resize(migConf.Project, migConf.Zone, migConf.Name, size).Do()
	if err != nil {
		return err
	}
	if err := m.waitForOp(op, migConf.Project); err != nil {
		return err
	}
	return nil
}

func (m *gceManager) waitForOp(operation *gce.Operation, project string) error {
	for start := time.Now(); time.Since(start) < operationWaitTimeout; time.Sleep(operationPollInterval) {
		if op, err := m.service.ZoneOperations.Get(project, operation.Zone, operation.Name).Do(); err == nil {
			if op.Status == "DONE" {
				return nil
			}
		} else {
			glog.Warningf("Error while getting operation %s on %s: %v", operation.Name, operation.TargetLink, err)
		}
	}
	return fmt.Errorf("Timeout while waiting for operation %s on %s to complete.", operation.Name, operation.TargetLink)
}

// All instances must be controlled by the same MIG.
func (m *gceManager) DeleteInstances(instances []*config.InstanceConfig) error {
	if len(instances) == 0 {
		return nil
	}
	commonMig, err := m.GetMigForInstance(instances[0])
	if err != nil {
		return err
	}
	for _, instance := range instances {
		mig, err := m.GetMigForInstance(instance)
		if err != nil {
			return err
		}
		if mig != commonMig {
			return fmt.Errorf("Connot delete instances which don't belong to the same MIG.")
		}
	}

	req := gce.InstanceGroupManagersDeleteInstancesRequest{
		Instances: []string{},
	}
	for _, instance := range instances {
		req.Instances = append(req.Instances, gceurl.GenerateInstanceUrl(instance.Project, instance.Zone, instance.Name))
	}

	op, err := m.service.InstanceGroupManagers.DeleteInstances(commonMig.Project, commonMig.Zone, commonMig.Name, &req).Do()
	if err != nil {
		return err
	}
	if err := m.waitForOp(op, commonMig.Project); err != nil {
		return err
	}
	return nil
}

func (m *gceManager) GetMigForInstance(instance *config.InstanceConfig) (*config.MigConfig, error) {
	m.cacheMutex.Lock()
	defer m.cacheMutex.Unlock()
	if mig, found := m.migCache[*instance]; found {
		return mig, nil
	}
	if err := m.regenerateCache(); err != nil {
		return nil, fmt.Errorf("Error while looking for MIG for instance %+v, error: %v", *instance, err)
	}
	if mig, found := m.migCache[*instance]; found {
		return mig, nil
	}
	return nil, fmt.Errorf("Instance %+v does not belong to any known MIG", *instance)
}

func (m *gceManager) regenerateCacheIgnoreError() {
	m.cacheMutex.Lock()
	defer m.cacheMutex.Unlock()
	if err := m.regenerateCache(); err != nil {
		glog.Errorf("Error while regenerating Mig cache: %v", err)
	}
}

func (m *gceManager) regenerateCache() error {
	newMigCache := map[config.InstanceConfig]*config.MigConfig{}

	for _, mig := range m.migs {
		instances, err := m.service.InstanceGroupManagers.ListManagedInstances(mig.Project, mig.Zone, mig.Name).Do()
		if err != nil {
			return err
		}
		for _, instance := range instances.ManagedInstances {
			project, zone, name, err := gceurl.ParseInstanceUrl(instance.Instance)
			if err != nil {
				return err
			}
			newMigCache[config.InstanceConfig{Project: project, Zone: zone, Name: name}] = mig
		}
	}

	m.migCache = newMigCache
	return nil
}
<|endoftext|>


<commit_before>package squirrel

import (
	"bytes"
	"database/sql"
	"fmt"
	"strings"
)

// InsertBuilder builds SQL INSERT statements.
type InsertBuilder struct {
	StatementBuilderType

	prefixes exprs
	options  []string
	into     string
	columns  []string
	values   [][]interface{}
	suffixes exprs
}

func NewInsertBuilder(b StatementBuilderType) *InsertBuilder {
	return &InsertBuilder{StatementBuilderType: b}
}

func (b *InsertBuilder) Exec() (sql.Result, error) {
	if b.runWith == nil {
		return nil, RunnerNotSet
	}
	return ExecWith(b.runWith, b)
}

// Query builds and Querys the query with the Runner set by RunWith.
func (b *InsertBuilder) Query() (*sql.Rows, error) {
	if b.runWith == nil {
		return nil, RunnerNotSet
	}
	return QueryWith(b.runWith, b)
}

// QueryRow builds and QueryRows the query with the Runner set by RunWith.
func (b *InsertBuilder) QueryRow() RowScanner {
	if b.runWith == nil {
		return &Row{err: RunnerNotSet}
	}
	queryRower, ok := b.runWith.(QueryRower)
	if !ok {
		return &Row{err: RunnerNotQueryRunner}
	}
	return QueryRowWith(queryRower, b)
}

// Scan is a shortcut for QueryRow().Scan.
func (b *InsertBuilder) Scan(dest ...interface{}) error {
	return b.QueryRow().Scan(dest...)
}

// PlaceholderFormat sets PlaceholderFormat (e.g. Question or Dollar) for the
// query.
func (b *InsertBuilder) PlaceholderFormat(f PlaceholderFormat) *InsertBuilder {
	b.placeholderFormat = f
	return b
}

// Runner methods

// RunWith sets a Runner (like database/sql.DB) to be used with e.g. Exec.
func (b *InsertBuilder) RunWith(runner BaseRunner) *InsertBuilder {
	b.runWith = runner
	return b
}

// SQL methods

// ToSql builds the query into a SQL string and bound args.
func (b *InsertBuilder) ToSql() (sqlStr string, args []interface{}, err error) {
	if len(b.into) == 0 {
		err = fmt.Errorf("insert statements must specify a table")
		return
	}
	if len(b.values) == 0 {
		err = fmt.Errorf("insert statements must have at least one set of values")
		return
	}

	sql := &bytes.Buffer{}

	if len(b.prefixes) > 0 {
		args, _ = b.prefixes.AppendToSql(sql, " ", args)
		sql.WriteString(" ")
	}

	sql.WriteString("INSERT ")

	if len(b.options) > 0 {
		sql.WriteString(strings.Join(b.options, " "))
		sql.WriteString(" ")
	}

	sql.WriteString("INTO ")
	sql.WriteString(b.into)
	sql.WriteString(" ")

	if len(b.columns) > 0 {
		sql.WriteString("(")
		sql.WriteString(strings.Join(b.columns, ","))
		sql.WriteString(") ")
	}

	sql.WriteString("VALUES ")

	valuesStrings := make([]string, len(b.values))
	for r, row := range b.values {

		valueStrings := make([]string, len(row))

		for v, val := range row {
			switch typedVal := val.(type) {
			case Sqlizer:
				var valSql string
				var valArgs []interface{}

				valSql, valArgs, err = typedVal.ToSql()
				if err != nil {
					return
				}

				valueStrings[v] = valSql
				args = append(args, valArgs...)
			default:
				valueStrings[v] = "?"
				args = append(args, val)
			}
		}

		valuesStrings[r] = fmt.Sprintf("(%s)", strings.Join(valueStrings, ","))
	}
	sql.WriteString(strings.Join(valuesStrings, ","))

	if len(b.suffixes) > 0 {
		sql.WriteString(" ")
		args, _ = b.suffixes.AppendToSql(sql, " ", args)
	}

	sqlStr, err = b.placeholderFormat.ReplacePlaceholders(sql.String())
	return
}

// Prefix adds an expression to the beginning of the query
func (b *InsertBuilder) Prefix(sql string, args ...interface{}) *InsertBuilder {
	b.prefixes = append(b.prefixes, Expr(sql, args...))
	return b
}

// Options adds keyword options before the INTO clause of the query.
func (b *InsertBuilder) Options(options ...string) *InsertBuilder {
	b.options = append(b.options, options...)
	return b
}

// Into sets the INTO clause of the query.
func (b *InsertBuilder) Into(into string) *InsertBuilder {
	b.into = into
	return b
}

// Columns adds insert columns to the query.
func (b *InsertBuilder) Columns(columns ...string) *InsertBuilder {
	b.columns = append(b.columns, columns...)
	return b
}

// Values adds a single row's values to the query.
func (b *InsertBuilder) Values(values ...interface{}) *InsertBuilder {
	b.values = append(b.values, values)
	return b
}

// Suffix adds an expression to the end of the query
func (b *InsertBuilder) Suffix(sql string, args ...interface{}) *InsertBuilder {
	b.suffixes = append(b.suffixes, Expr(sql, args...))
	return b
}

// SetMap set columns and values for insert builder from a map of column name and value
// note that it will reset all previous columns and values was set if any
func (b *InsertBuilder) SetMap(clauses map[string]interface{}) *InsertBuilder {
	// TODO: replace resetting previous values with extending existing ones?
	b.columns = make([]string, 0, len(clauses))
	b.values = make([][]interface{}, 0, len(clauses))

	for col, val := range clauses {
		b.columns = append(b.columns, col)

		// may panic here

		b.values = append(b.values, val.([]interface{}))
	}

	return b
}
<commit_msg>fixed InsertBuilder in order, now tests pass<commit_after>package squirrel

import (
	"bytes"
	"database/sql"
	"fmt"
	"strings"
)

// InsertBuilder builds SQL INSERT statements.
type InsertBuilder struct {
	StatementBuilderType

	prefixes exprs
	options  []string
	into     string
	columns  []string
	values   [][]interface{}
	suffixes exprs
}

func NewInsertBuilder(b StatementBuilderType) *InsertBuilder {
	return &InsertBuilder{StatementBuilderType: b}
}

func (b *InsertBuilder) Exec() (sql.Result, error) {
	if b.runWith == nil {
		return nil, RunnerNotSet
	}
	return ExecWith(b.runWith, b)
}

// Query builds and Querys the query with the Runner set by RunWith.
func (b *InsertBuilder) Query() (*sql.Rows, error) {
	if b.runWith == nil {
		return nil, RunnerNotSet
	}
	return QueryWith(b.runWith, b)
}

// QueryRow builds and QueryRows the query with the Runner set by RunWith.
func (b *InsertBuilder) QueryRow() RowScanner {
	if b.runWith == nil {
		return &Row{err: RunnerNotSet}
	}
	queryRower, ok := b.runWith.(QueryRower)
	if !ok {
		return &Row{err: RunnerNotQueryRunner}
	}
	return QueryRowWith(queryRower, b)
}

// Scan is a shortcut for QueryRow().Scan.
func (b *InsertBuilder) Scan(dest ...interface{}) error {
	return b.QueryRow().Scan(dest...)
}

// PlaceholderFormat sets PlaceholderFormat (e.g. Question or Dollar) for the
// query.
func (b *InsertBuilder) PlaceholderFormat(f PlaceholderFormat) *InsertBuilder {
	b.placeholderFormat = f
	return b
}

// Runner methods

// RunWith sets a Runner (like database/sql.DB) to be used with e.g. Exec.
func (b *InsertBuilder) RunWith(runner BaseRunner) *InsertBuilder {
	b.runWith = runner
	return b
}

// SQL methods

// ToSql builds the query into a SQL string and bound args.
func (b *InsertBuilder) ToSql() (sqlStr string, args []interface{}, err error) {
	if len(b.into) == 0 {
		err = fmt.Errorf("insert statements must specify a table")
		return
	}
	if len(b.values) == 0 {
		err = fmt.Errorf("insert statements must have at least one set of values")
		return
	}

	sql := &bytes.Buffer{}

	if len(b.prefixes) > 0 {
		args, _ = b.prefixes.AppendToSql(sql, " ", args)
		sql.WriteString(" ")
	}

	sql.WriteString("INSERT ")

	if len(b.options) > 0 {
		sql.WriteString(strings.Join(b.options, " "))
		sql.WriteString(" ")
	}

	sql.WriteString("INTO ")
	sql.WriteString(b.into)
	sql.WriteString(" ")

	if len(b.columns) > 0 {
		sql.WriteString("(")
		sql.WriteString(strings.Join(b.columns, ","))
		sql.WriteString(") ")
	}

	sql.WriteString("VALUES ")

	valuesStrings := make([]string, len(b.values))
	for r, row := range b.values {

		valueStrings := make([]string, len(row))

		for v, val := range row {
			switch typedVal := val.(type) {
			case Sqlizer:
				var valSql string
				var valArgs []interface{}

				valSql, valArgs, err = typedVal.ToSql()
				if err != nil {
					return
				}

				valueStrings[v] = valSql
				args = append(args, valArgs...)
			default:
				valueStrings[v] = "?"
				args = append(args, val)
			}
		}

		valuesStrings[r] = fmt.Sprintf("(%s)", strings.Join(valueStrings, ","))
	}
	sql.WriteString(strings.Join(valuesStrings, ","))

	if len(b.suffixes) > 0 {
		sql.WriteString(" ")
		args, _ = b.suffixes.AppendToSql(sql, " ", args)
	}

	sqlStr, err = b.placeholderFormat.ReplacePlaceholders(sql.String())
	return
}

// Prefix adds an expression to the beginning of the query
func (b *InsertBuilder) Prefix(sql string, args ...interface{}) *InsertBuilder {
	b.prefixes = append(b.prefixes, Expr(sql, args...))
	return b
}

// Options adds keyword options before the INTO clause of the query.
func (b *InsertBuilder) Options(options ...string) *InsertBuilder {
	b.options = append(b.options, options...)
	return b
}

// Into sets the INTO clause of the query.
func (b *InsertBuilder) Into(into string) *InsertBuilder {
	b.into = into
	return b
}

// Columns adds insert columns to the query.
func (b *InsertBuilder) Columns(columns ...string) *InsertBuilder {
	b.columns = append(b.columns, columns...)
	return b
}

// Values adds a single row's values to the query.
func (b *InsertBuilder) Values(values ...interface{}) *InsertBuilder {
	b.values = append(b.values, values)
	return b
}

// Suffix adds an expression to the end of the query
func (b *InsertBuilder) Suffix(sql string, args ...interface{}) *InsertBuilder {
	b.suffixes = append(b.suffixes, Expr(sql, args...))
	return b
}

// SetMap set columns and values for insert builder from a map of column name and value
// note that it will reset all previous columns and values was set if any
func (b *InsertBuilder) SetMap(clauses map[string]interface{}) *InsertBuilder {
	// TODO: replace resetting previous values with extending existing ones?
	cols := make([]string, 0, len(clauses))
	vals := make([]interface{}, 0, len(clauses))

	for col, val := range clauses {
		cols = append(cols, col)
		vals = append(vals, val)
	}

	b.columns = cols
	b.values = [][]interface{}{vals}

	return b
}
<|endoftext|>


<commit_before>package kafka

import (
	l "log"
	"os"
	"sync"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/cmd/kateway/store"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/golib/color"
	"github.com/funkygao/kafka-cg/consumergroup"
	log "github.com/funkygao/log4go"
)

type subStore struct {
	shutdownCh   chan struct{}
	closedConnCh <-chan string // remote addr
	wg           *sync.WaitGroup
	hostname     string

	subManager *subManager
}

func NewSubStore(wg *sync.WaitGroup, closedConnCh <-chan string, debug bool) *subStore {
	if debug {
		sarama.Logger = l.New(os.Stdout, color.Blue("[Sarama]"),
			l.LstdFlags|l.Lshortfile)
	}

	return &subStore{
		hostname:     ctx.Hostname(),
		wg:           wg,
		shutdownCh:   make(chan struct{}),
		closedConnCh: closedConnCh,
	}
}

func (this *subStore) Name() string {
	return "kafka"
}

func (this *subStore) Start() (err error) {
	this.wg.Add(1)

	this.subManager = newSubManager()

	go func() {
		defer this.wg.Done()

		var remoteAddr string
		for {
			select {
			case <-this.shutdownCh:
				log.Trace("sub store[%s] stopped", this.Name())
				return

			case remoteAddr = <-this.closedConnCh:
				go this.subManager.killClient(remoteAddr)

			}
		}
	}()

	return
}

func (this *subStore) Stop() {
	this.subManager.Stop()
	close(this.shutdownCh)
}

func (this *subStore) Fetch(cluster, topic, group, remoteAddr, realIp,
	resetOffset string, permitStandby bool) (store.Fetcher, error) {
	cg, err := this.subManager.PickConsumerGroup(cluster, topic, group, remoteAddr, realIp, resetOffset, permitStandby)
	if err != nil {
		return nil, err
	}

	return &consumerFetcher{
		ConsumerGroup: cg,
		remoteAddr:    remoteAddr,
		store:         this,
	}, nil
}

func (this *subStore) IsSystemError(err error) bool {
	switch err {
	case consumergroup.ErrTooManyConsumers, store.ErrTooManyConsumers:
		return false

	default:
		return true
	}
}
<commit_msg>wait for all consumer groups shutdown complete<commit_after>package kafka

import (
	l "log"
	"os"
	"sync"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/cmd/kateway/store"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/golib/color"
	"github.com/funkygao/kafka-cg/consumergroup"
	log "github.com/funkygao/log4go"
)

type subStore struct {
	shutdownCh   chan struct{}
	closedConnCh <-chan string // remote addr
	wg           *sync.WaitGroup
	hostname     string

	subManager *subManager
}

func NewSubStore(wg *sync.WaitGroup, closedConnCh <-chan string, debug bool) *subStore {
	if debug {
		sarama.Logger = l.New(os.Stdout, color.Blue("[Sarama]"),
			l.LstdFlags|l.Lshortfile)
	}

	return &subStore{
		hostname:     ctx.Hostname(),
		wg:           wg,
		shutdownCh:   make(chan struct{}),
		closedConnCh: closedConnCh,
	}
}

func (this *subStore) Name() string {
	return "kafka"
}

func (this *subStore) Start() (err error) {
	this.subManager = newSubManager()

	this.wg.Add(1)
	go func() {
		defer this.wg.Done()

		var remoteAddr string
		for {
			select {
			case <-this.shutdownCh:
				log.Trace("sub store[%s] stopped", this.Name())
				return

			case remoteAddr = <-this.closedConnCh:
				this.wg.Add(1)
				go func() {
					this.subManager.killClient(remoteAddr)
					this.wg.Done()
				}()

			}
		}
	}()

	return
}

func (this *subStore) Stop() {
	this.subManager.Stop()
	close(this.shutdownCh)
	this.wg.Wait()
}

func (this *subStore) Fetch(cluster, topic, group, remoteAddr, realIp,
	resetOffset string, permitStandby bool) (store.Fetcher, error) {
	cg, err := this.subManager.PickConsumerGroup(cluster, topic, group, remoteAddr, realIp, resetOffset, permitStandby)
	if err != nil {
		return nil, err
	}

	return &consumerFetcher{
		ConsumerGroup: cg,
		remoteAddr:    remoteAddr,
		store:         this,
	}, nil
}

func (this *subStore) IsSystemError(err error) bool {
	switch err {
	case consumergroup.ErrTooManyConsumers, store.ErrTooManyConsumers:
		return false

	default:
		return true
	}
}
<|endoftext|>


<commit_before>package frame

import (
	"image"
	"image/draw"
)

// Insert inserts the contents of s at index p0 in
// the frame and returns the number of characters
// written.
func (f *Frame) Insert(s []byte, p0 int64) (wrote int) {
	defer f.DumpBoxes()
	type Pts [2]image.Point
	if p0 > f.Nchars || len(s) == 0 || f.b == nil {
		return
	}

	// find p0, it's box, and its point in the box its in
	b0 := f.Find(0, 0, p0)
	//	eb := f.StartCell(b0)
	//ob0 := b0
	cb0 := p0
	b1 := b0
	pt0 := f.pointOfBox(p0, b0)

	// find p1
	ppt0, pt1 := f.boxscan(s, pt0)
	opt0 := pt0
	ppt1 := pt1
	// Line wrap
	if b0 < f.Nbox {
		b := &f.Box[b0]
		pt0 = f.wrapMax(pt0, b)
		ppt1 = f.wrapMin(ppt1, b)
	}
	f.modified = true

	if f.p0 == f.p1 {
		f.tickat(f.PointOf(int64(f.p0)), false)
	}

	cb0, b0, pt0, pt1 = f.boxalign(cb0, b0, pt0, pt1)
	f.boxpush(p0, b0, b1, pt0, pt1, ppt1)
	f.bitblt(cb0, b0, pt0, pt1, opt0)
	text, back := f.pick(p0, f.p0+1, f.p1+1)
	f.Paint(ppt0, ppt1, back)
	f.redrawRun0(f.ir, ppt0, text, back)
	f.Run.Combine(f.ir, b1)
	if b1 > 0 && f.Box[b1-1].Nrune >= 0 && ppt0.X-f.Box[b1-1].Width >= f.r.Min.X {
		b1--
		ppt0.X -= f.Box[b1].Width
	}
	b0 += f.ir.Nbox
	if b0 < f.Nbox-1 {
		b0++
	}
	f.clean(ppt0, b1, b0)
	f.Nchars += f.ir.Nchars
	f.p0, f.p1 = coInsert(p0, p0+f.Nchars, f.p0, f.p1)
	if f.p0 == f.p1 {
		f.tickat(f.PointOf(f.p0), true)
	}
//	f.badElasticAlg()
	return int(f.ir.Nchars)
}

func (f *Frame) Draw(dst draw.Image, r image.Rectangle, src image.Image, sp image.Point, op draw.Op) {
	if f == nil {
		panic("nil frame")
	}
	if f.Drawer == nil {
		panic("nil drawer")
	}
	f.Drawer.Draw(dst, r, src, sp, op)
	f.Flush(r)
}
func (f *Frame) badElasticAlg() {
	if f.elastic() {
		if f.Nbox <= 1 {
			return
		}
		b := 0
		b1 := 0
		for b != f.Nbox {
			b1 = b
			b = f.NextCell(b)
			if b == 0 || b == b1 {
				break
			}
		}
		f.Stretch(f.Nbox)
		b = b1
		for ; b1 > 1; b1 = f.Stretch(b1) {
			if b == b1 {
				break
			}
			b = b1
		}
		f.Stretch(b1)
		f.Refresh()
	}
}

// Mark marks the frame as dirty
func (f *Frame) Mark() {
	f.modified = true
}

// boxalign collects a list of pts of each box
// on the frame before and after an insertion occurs
func (f *Frame) boxalign(cb0 int64, b0 int, pt0, pt1 image.Point) (int64, int, image.Point, image.Point) {
	type Pts [2]image.Point
	f.pts = f.pts[:0]

	// collect the start pts for each box on the plane
	// before and after the insertion
	for {
		if pt0.X == pt1.X || pt1.Y == f.r.Max.Y || b0 == f.Nbox {
			break
		}
		b := &f.Box[b0]
		pt0 = f.wrapMax(pt0, b)
		pt1 = f.wrapMin(pt1, b)
		if b.Nrune > 0 {
			if n := f.fits(pt1, b); n != b.Nrune {
				f.Split(b0, n)
				b = &f.Box[b0]
			}
		}

		// early exit - point went off the frame
		if pt1.Y == f.r.Max.Y {
			break
		}

		f.pts = append(f.pts, Pts{pt0, pt1})

		pt0 = f.advance(pt0, b)
		pt1.X += f.plot(pt1, b)

		cb0 += int64(b.Len())
		b0++
	}
	return cb0, b0, pt0, pt1
}

// boxpush moves boxes down the frame to make room for an insertion
// from pt0 to pt1
func (f *Frame) boxpush(p0 int64, b0, b1 int, pt0, pt1, ppt1 image.Point) {
	h := f.Font.Dy()
	// delete boxes that ran off the frame
	// and update the char count
	if pt1.Y == f.r.Max.Y && b0 < f.Nbox {
		f.Nchars -= f.Count(b0)
		f.Run.Delete(b0, f.Nbox-1)
	}

	// update the line count
	if b0 == f.Nbox {
		f.Nlines = (pt1.Y - f.r.Min.Y) / h
		if pt1.X > f.r.Min.X {
			f.Nlines++
		}
		return
	}

	if pt1.Y == pt0.Y {
		// insertion won't propagate down
		return
	}

	qt0 := pt0.Y + h
	qt1 := pt1.Y + h
	f.Nlines += (qt1 - qt0) / h
	if f.Nlines > f.maxlines {
		f.trim(ppt1, p0, b1)
	}

	// shift down the existing boxes
	// on the bitmap
	if r := f.r; pt1.Y < r.Max.Y {
		r.Min.Y = qt1

		// rectangular group of boxes
		if qt1 < f.r.Max.Y {
			f.Draw(f.b, r, f.b, image.Pt(f.r.Min.X, qt0), f.op)
		}

		// partial line
		r.Min = pt1
		r.Max.X = pt1.X + (f.r.Max.X - pt0.X)
		r.Max.Y = qt1
		f.Draw(f.b, r, f.b, pt0, f.op)
	}
}

func (f *Frame) bitblt(cb0 int64, b0 int, pt0, pt1, opt0 image.Point) (res image.Rectangle) {
	h := f.Font.Dy()
	y := 0
	if pt1.Y == f.r.Max.Y {
		y = pt1.Y
	}
	x := len(f.pts)
	if x != 0 {
		res = image.Rectangle{pt0, pt1}
		res.Canon().Max.Add(f.pts[x-1][0])
	}
	run := f.Box[b0-x:]
	x--
	_, back := f.pick(cb0, f.p0, f.p1)
	for ; x >= 0; x-- {
		b := &run[x]
		br := image.Rect(0, 0, b.Width, h)
		pt := f.pts[x]
		if b.Nrune > 0 {
			f.Draw(f.b, br.Add(pt[1]), f.b, pt[0], f.op)
			// clear bit hanging off right
			if x == 0 && pt[1].Y > pt0.Y {

				// new char was wider than the old
				// one so the line wrapped anyway

				_, back = f.pick(cb0, f.p0, f.p1)
				r := br.Add(opt0)
				r.Max.X = f.r.Max.X
				f.Draw(f.b, r, back, r.Min, f.op)

			} else if pt[1].Y < y {

				// copy from left to right, bottom to top

				_, back = f.pick(cb0, f.p0, f.p1)
				r := image.ZR.Add(pt[1])
				r.Min.X += b.Width
				r.Max.X += f.r.Max.X
				r.Max.Y += h
				f.Draw(f.b, r, back, r.Min, f.op)

			}
			y = pt[1].Y
			cb0 -= int64(b.Nrune)
		} else {
			r := br.Add(pt[1])
			if r.Max.X >= f.r.Max.X {
				r.Max.X = f.r.Max.X
			}
			cb0--
			_, back = f.pick(cb0, f.p0, f.p1)
			f.Draw(f.b, r, back, r.Min, f.op)
			y = 0
			if pt[1].X == f.r.Min.X {
				y = pt[1].Y
			}
		}
	}
	return res
}

func (f *Frame) pick(c, p0, p1 int64) (text, back image.Image) {
	if p0 <= c && c < p1 {
		return f.Color.Hi.Text, f.Color.Hi.Back
	}
	return f.Color.Text, f.Color.Back
}

func region(c, p0, p1 int64) int {
	if c < p0 {
		return -1
	}
	if c >= p1 {
		return 1
	}
	return 0
}

//
// Below ideas

type offset struct {
	p0   int64
	b0   int
	cb0  int64
	b1   int64
	pt0  image.Point
	pt1  image.Point
	opt0 image.Point
}

func (f *Frame) zInsertElastic(s []byte, p0 int64) (wrote int) {
	type Pts [2]image.Point
	if p0 > f.Nchars || len(s) == 0 || f.b == nil {
		return
	}

	// find p0, it's box, and its point in the box its in
	b0 := f.Find(0, 0, p0)
	cb0 := p0

	eb0 := f.StartCell(b0)
	pt0 := f.pointOfBox(p0, b0)
	ppt0, pt1 := f.boxscan(s, pt0)
	//	ept0 := f.pointOfBox(p0, eb0)
	f.Box = append(f.Box[:b0], append(f.ir.Box[:f.ir.Nbox], f.Box[b0:]...)...)
	f.Nbox += f.ir.Nbox

	b1 := b0

	for bn := f.NextCell(b1 + f.ir.Nbox); bn > eb0; bn = f.Stretch(bn) {
	}
	f.Stretch(eb0)

	//		pt0 = f.pointOfBox(p0, b0)
	//		pt1 = f.pointOfBox(65535, b1)

	opt0 := pt0
	ppt1 := pt1
	// Line wrap
	if b0 < f.Nbox {
		b := &f.Box[b0]
		pt0 = f.wrapMax(pt0, b)
		ppt1 = f.wrapMin(ppt1, b)
	}
	f.modified = true

	if f.p0 == f.p1 {
		f.tickat(f.PointOf(int64(f.p0)), false)
	}

	cb0, b0, pt0, pt1 = f.boxalign(cb0, b0+f.ir.Nbox, pt0, pt1)
	f.boxpush(p0, b0+f.ir.Nbox, b1+f.ir.Nbox, pt0, pt1, ppt1)
	f.bitblt(cb0, b0, pt0, pt1, opt0)
	text, back := f.pick(p0, f.p0+1, f.p1+1)
	f.Paint(ppt0, ppt1, back)
	f.redrawRun0(f.ir, ppt0, text, back)

	b1 = b0
	if b1 > 0 && f.Box[b1-1].Nrune >= 0 && ppt0.X-f.Box[b1-1].Width >= f.r.Min.X {
		b1--
		ppt0.X -= f.Box[b1].Width
	}

	b0 += f.ir.Nbox
	if b0 < f.Nbox-1 {
		b0++
	}
	f.clean(f.pointOfBox(p0, b1), b0, b1)
	f.Nchars += f.ir.Nchars

	f.p0, f.p1 = coInsert(p0, p0+f.Nchars, f.p0, f.p1)
	if f.p0 == f.p1 {
		f.tickat(f.PointOf(f.p0), true)
	}
	return int(f.ir.Nchars)

}

func drawBorder(dst draw.Image, r image.Rectangle, src image.Image, sp image.Point, thick int) {
	draw.Draw(dst, image.Rect(r.Min.X, r.Min.Y, r.Max.X, r.Min.Y+thick), src, sp, draw.Src)
	draw.Draw(dst, image.Rect(r.Min.X, r.Max.Y-thick, r.Max.X, r.Max.Y), src, sp, draw.Src)
	draw.Draw(dst, image.Rect(r.Min.X, r.Min.Y, r.Min.X+thick, r.Max.Y), src, sp, draw.Src)
	draw.Draw(dst, image.Rect(r.Max.X-thick, r.Min.Y, r.Max.X, r.Max.Y), src, sp, draw.Src)
}
<commit_msg>frame: insert: remove debug stmt<commit_after>package frame

import (
	"image"
	"image/draw"
)

// Insert inserts the contents of s at index p0 in
// the frame and returns the number of characters
// written.
func (f *Frame) Insert(s []byte, p0 int64) (wrote int) {
	type Pts [2]image.Point
	if p0 > f.Nchars || len(s) == 0 || f.b == nil {
		return
	}

	// find p0, it's box, and its point in the box its in
	b0 := f.Find(0, 0, p0)
	//	eb := f.StartCell(b0)
	//ob0 := b0
	cb0 := p0
	b1 := b0
	pt0 := f.pointOfBox(p0, b0)

	// find p1
	ppt0, pt1 := f.boxscan(s, pt0)
	opt0 := pt0
	ppt1 := pt1
	// Line wrap
	if b0 < f.Nbox {
		b := &f.Box[b0]
		pt0 = f.wrapMax(pt0, b)
		ppt1 = f.wrapMin(ppt1, b)
	}
	f.modified = true

	if f.p0 == f.p1 {
		f.tickat(f.PointOf(int64(f.p0)), false)
	}

	cb0, b0, pt0, pt1 = f.boxalign(cb0, b0, pt0, pt1)
	f.boxpush(p0, b0, b1, pt0, pt1, ppt1)
	f.bitblt(cb0, b0, pt0, pt1, opt0)
	text, back := f.pick(p0, f.p0+1, f.p1+1)
	f.Paint(ppt0, ppt1, back)
	f.redrawRun0(f.ir, ppt0, text, back)
	f.Run.Combine(f.ir, b1)
	if b1 > 0 && f.Box[b1-1].Nrune >= 0 && ppt0.X-f.Box[b1-1].Width >= f.r.Min.X {
		b1--
		ppt0.X -= f.Box[b1].Width
	}
	b0 += f.ir.Nbox
	if b0 < f.Nbox-1 {
		b0++
	}
	f.clean(ppt0, b1, b0)
	f.Nchars += f.ir.Nchars
	f.p0, f.p1 = coInsert(p0, p0+f.Nchars, f.p0, f.p1)
	if f.p0 == f.p1 {
		f.tickat(f.PointOf(f.p0), true)
	}
//	f.badElasticAlg()
	return int(f.ir.Nchars)
}

func (f *Frame) Draw(dst draw.Image, r image.Rectangle, src image.Image, sp image.Point, op draw.Op) {
	if f == nil {
		panic("nil frame")
	}
	if f.Drawer == nil {
		panic("nil drawer")
	}
	f.Drawer.Draw(dst, r, src, sp, op)
	f.Flush(r)
}
func (f *Frame) badElasticAlg() {
	if f.elastic() {
		if f.Nbox <= 1 {
			return
		}
		b := 0
		b1 := 0
		for b != f.Nbox {
			b1 = b
			b = f.NextCell(b)
			if b == 0 || b == b1 {
				break
			}
		}
		f.Stretch(f.Nbox)
		b = b1
		for ; b1 > 1; b1 = f.Stretch(b1) {
			if b == b1 {
				break
			}
			b = b1
		}
		f.Stretch(b1)
		f.Refresh()
	}
}

// Mark marks the frame as dirty
func (f *Frame) Mark() {
	f.modified = true
}

// boxalign collects a list of pts of each box
// on the frame before and after an insertion occurs
func (f *Frame) boxalign(cb0 int64, b0 int, pt0, pt1 image.Point) (int64, int, image.Point, image.Point) {
	type Pts [2]image.Point
	f.pts = f.pts[:0]

	// collect the start pts for each box on the plane
	// before and after the insertion
	for {
		if pt0.X == pt1.X || pt1.Y == f.r.Max.Y || b0 == f.Nbox {
			break
		}
		b := &f.Box[b0]
		pt0 = f.wrapMax(pt0, b)
		pt1 = f.wrapMin(pt1, b)
		if b.Nrune > 0 {
			if n := f.fits(pt1, b); n != b.Nrune {
				f.Split(b0, n)
				b = &f.Box[b0]
			}
		}

		// early exit - point went off the frame
		if pt1.Y == f.r.Max.Y {
			break
		}

		f.pts = append(f.pts, Pts{pt0, pt1})

		pt0 = f.advance(pt0, b)
		pt1.X += f.plot(pt1, b)

		cb0 += int64(b.Len())
		b0++
	}
	return cb0, b0, pt0, pt1
}

// boxpush moves boxes down the frame to make room for an insertion
// from pt0 to pt1
func (f *Frame) boxpush(p0 int64, b0, b1 int, pt0, pt1, ppt1 image.Point) {
	h := f.Font.Dy()
	// delete boxes that ran off the frame
	// and update the char count
	if pt1.Y == f.r.Max.Y && b0 < f.Nbox {
		f.Nchars -= f.Count(b0)
		f.Run.Delete(b0, f.Nbox-1)
	}

	// update the line count
	if b0 == f.Nbox {
		f.Nlines = (pt1.Y - f.r.Min.Y) / h
		if pt1.X > f.r.Min.X {
			f.Nlines++
		}
		return
	}

	if pt1.Y == pt0.Y {
		// insertion won't propagate down
		return
	}

	qt0 := pt0.Y + h
	qt1 := pt1.Y + h
	f.Nlines += (qt1 - qt0) / h
	if f.Nlines > f.maxlines {
		f.trim(ppt1, p0, b1)
	}

	// shift down the existing boxes
	// on the bitmap
	if r := f.r; pt1.Y < r.Max.Y {
		r.Min.Y = qt1

		// rectangular group of boxes
		if qt1 < f.r.Max.Y {
			f.Draw(f.b, r, f.b, image.Pt(f.r.Min.X, qt0), f.op)
		}

		// partial line
		r.Min = pt1
		r.Max.X = pt1.X + (f.r.Max.X - pt0.X)
		r.Max.Y = qt1
		f.Draw(f.b, r, f.b, pt0, f.op)
	}
}

func (f *Frame) bitblt(cb0 int64, b0 int, pt0, pt1, opt0 image.Point) (res image.Rectangle) {
	h := f.Font.Dy()
	y := 0
	if pt1.Y == f.r.Max.Y {
		y = pt1.Y
	}
	x := len(f.pts)
	if x != 0 {
		res = image.Rectangle{pt0, pt1}
		res.Canon().Max.Add(f.pts[x-1][0])
	}
	run := f.Box[b0-x:]
	x--
	_, back := f.pick(cb0, f.p0, f.p1)
	for ; x >= 0; x-- {
		b := &run[x]
		br := image.Rect(0, 0, b.Width, h)
		pt := f.pts[x]
		if b.Nrune > 0 {
			f.Draw(f.b, br.Add(pt[1]), f.b, pt[0], f.op)
			// clear bit hanging off right
			if x == 0 && pt[1].Y > pt0.Y {

				// new char was wider than the old
				// one so the line wrapped anyway

				_, back = f.pick(cb0, f.p0, f.p1)
				r := br.Add(opt0)
				r.Max.X = f.r.Max.X
				f.Draw(f.b, r, back, r.Min, f.op)

			} else if pt[1].Y < y {

				// copy from left to right, bottom to top

				_, back = f.pick(cb0, f.p0, f.p1)
				r := image.ZR.Add(pt[1])
				r.Min.X += b.Width
				r.Max.X += f.r.Max.X
				r.Max.Y += h
				f.Draw(f.b, r, back, r.Min, f.op)

			}
			y = pt[1].Y
			cb0 -= int64(b.Nrune)
		} else {
			r := br.Add(pt[1])
			if r.Max.X >= f.r.Max.X {
				r.Max.X = f.r.Max.X
			}
			cb0--
			_, back = f.pick(cb0, f.p0, f.p1)
			f.Draw(f.b, r, back, r.Min, f.op)
			y = 0
			if pt[1].X == f.r.Min.X {
				y = pt[1].Y
			}
		}
	}
	return res
}

func (f *Frame) pick(c, p0, p1 int64) (text, back image.Image) {
	if p0 <= c && c < p1 {
		return f.Color.Hi.Text, f.Color.Hi.Back
	}
	return f.Color.Text, f.Color.Back
}

func region(c, p0, p1 int64) int {
	if c < p0 {
		return -1
	}
	if c >= p1 {
		return 1
	}
	return 0
}

//
// Below ideas

type offset struct {
	p0   int64
	b0   int
	cb0  int64
	b1   int64
	pt0  image.Point
	pt1  image.Point
	opt0 image.Point
}

func (f *Frame) zInsertElastic(s []byte, p0 int64) (wrote int) {
	type Pts [2]image.Point
	if p0 > f.Nchars || len(s) == 0 || f.b == nil {
		return
	}

	// find p0, it's box, and its point in the box its in
	b0 := f.Find(0, 0, p0)
	cb0 := p0

	eb0 := f.StartCell(b0)
	pt0 := f.pointOfBox(p0, b0)
	ppt0, pt1 := f.boxscan(s, pt0)
	//	ept0 := f.pointOfBox(p0, eb0)
	f.Box = append(f.Box[:b0], append(f.ir.Box[:f.ir.Nbox], f.Box[b0:]...)...)
	f.Nbox += f.ir.Nbox

	b1 := b0

	for bn := f.NextCell(b1 + f.ir.Nbox); bn > eb0; bn = f.Stretch(bn) {
	}
	f.Stretch(eb0)

	//		pt0 = f.pointOfBox(p0, b0)
	//		pt1 = f.pointOfBox(65535, b1)

	opt0 := pt0
	ppt1 := pt1
	// Line wrap
	if b0 < f.Nbox {
		b := &f.Box[b0]
		pt0 = f.wrapMax(pt0, b)
		ppt1 = f.wrapMin(ppt1, b)
	}
	f.modified = true

	if f.p0 == f.p1 {
		f.tickat(f.PointOf(int64(f.p0)), false)
	}

	cb0, b0, pt0, pt1 = f.boxalign(cb0, b0+f.ir.Nbox, pt0, pt1)
	f.boxpush(p0, b0+f.ir.Nbox, b1+f.ir.Nbox, pt0, pt1, ppt1)
	f.bitblt(cb0, b0, pt0, pt1, opt0)
	text, back := f.pick(p0, f.p0+1, f.p1+1)
	f.Paint(ppt0, ppt1, back)
	f.redrawRun0(f.ir, ppt0, text, back)

	b1 = b0
	if b1 > 0 && f.Box[b1-1].Nrune >= 0 && ppt0.X-f.Box[b1-1].Width >= f.r.Min.X {
		b1--
		ppt0.X -= f.Box[b1].Width
	}

	b0 += f.ir.Nbox
	if b0 < f.Nbox-1 {
		b0++
	}
	f.clean(f.pointOfBox(p0, b1), b0, b1)
	f.Nchars += f.ir.Nchars

	f.p0, f.p1 = coInsert(p0, p0+f.Nchars, f.p0, f.p1)
	if f.p0 == f.p1 {
		f.tickat(f.PointOf(f.p0), true)
	}
	return int(f.ir.Nchars)

}

func drawBorder(dst draw.Image, r image.Rectangle, src image.Image, sp image.Point, thick int) {
	draw.Draw(dst, image.Rect(r.Min.X, r.Min.Y, r.Max.X, r.Min.Y+thick), src, sp, draw.Src)
	draw.Draw(dst, image.Rect(r.Min.X, r.Max.Y-thick, r.Max.X, r.Max.Y), src, sp, draw.Src)
	draw.Draw(dst, image.Rect(r.Min.X, r.Min.Y, r.Min.X+thick, r.Max.Y), src, sp, draw.Src)
	draw.Draw(dst, image.Rect(r.Max.X-thick, r.Min.Y, r.Max.X, r.Max.Y), src, sp, draw.Src)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package master

import (
	"crypto/x509"
	"encoding/json"
	"fmt"
	"time"

	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/apis/extensions"
	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
	certutil "k8s.io/kubernetes/pkg/util/cert"
	"k8s.io/kubernetes/pkg/util/wait"
)

type kubeDiscovery struct {
	Deployment *extensions.Deployment
	Secret     *api.Secret
}

const (
	kubeDiscoveryName       = "kube-discovery"
	kubeDiscoverySecretName = "clusterinfo"
)

func encodeKubeDiscoverySecretData(s *kubeadmapi.KubeadmConfig, caCert *x509.Certificate) map[string][]byte {
	var (
		data         = map[string][]byte{}
		endpointList = []string{}
		tokenMap     = map[string]string{}
	)

	for _, addr := range s.InitFlags.API.AdvertiseAddrs {
		endpointList = append(endpointList, fmt.Sprintf("https://%s:443", addr.String()))
	}

	tokenMap[s.Secrets.TokenID] = s.Secrets.BearerToken

	data["endpoint-list.json"], _ = json.Marshal(endpointList)
	data["token-map.json"], _ = json.Marshal(tokenMap)
	data["ca.pem"] = certutil.EncodeCertPEM(caCert)

	return data
}

func newKubeDiscoveryPodSpec(s *kubeadmapi.KubeadmConfig) api.PodSpec {
	return api.PodSpec{
		// We have to use host network namespace, as `HostPort`/`HostIP` are Docker's
		// buisness and CNI support isn't quite there yet (except for kubenet)
		// (see https://github.com/kubernetes/kubernetes/issues/31307)
		// TODO update this when #31307 is resolved
		SecurityContext: &api.PodSecurityContext{HostNetwork: true},
		Containers: []api.Container{{
			Name:    kubeDiscoveryName,
			Image:   s.EnvParams["discovery_image"],
			Command: []string{"/usr/local/bin/kube-discovery"},
			VolumeMounts: []api.VolumeMount{{
				Name:      kubeDiscoverySecretName,
				MountPath: "/tmp/secret", // TODO use a shared constant
				ReadOnly:  true,
			}},
			Ports: []api.ContainerPort{
				// TODO when CNI issue (#31307) is resolved, we should consider adding
				// `HostIP: s.API.AdvertiseAddrs[0]`, if there is only one address`
				{Name: "http", ContainerPort: 9898, HostPort: 9898},
			},
		}},
		Volumes: []api.Volume{{
			Name: kubeDiscoverySecretName,
			VolumeSource: api.VolumeSource{
				Secret: &api.SecretVolumeSource{SecretName: kubeDiscoverySecretName},
			}},
		},
	}
}

func newKubeDiscovery(s *kubeadmapi.KubeadmConfig, caCert *x509.Certificate) kubeDiscovery {
	kd := kubeDiscovery{
		Deployment: NewDeployment(kubeDiscoveryName, 1, newKubeDiscoveryPodSpec(s)),
		Secret: &api.Secret{
			ObjectMeta: api.ObjectMeta{Name: kubeDiscoverySecretName},
			Type:       api.SecretTypeOpaque,
			Data:       encodeKubeDiscoverySecretData(s, caCert),
		},
	}

	SetMasterTaintTolerations(&kd.Deployment.Spec.Template.ObjectMeta)
	SetMasterNodeAffinity(&kd.Deployment.Spec.Template.ObjectMeta)

	return kd
}

func CreateDiscoveryDeploymentAndSecret(s *kubeadmapi.KubeadmConfig, client *clientset.Clientset, caCert *x509.Certificate) error {
	kd := newKubeDiscovery(s, caCert)

	if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(kd.Deployment); err != nil {
		return fmt.Errorf("<master/discovery> failed to create %q deployment [%v]", kubeDiscoveryName, err)
	}
	if _, err := client.Secrets(api.NamespaceSystem).Create(kd.Secret); err != nil {
		return fmt.Errorf("<master/discovery> failed to create %q secret [%v]", kubeDiscoverySecretName, err)
	}

	fmt.Println("<master/discovery> created essential addon: kube-discovery, waiting for it to become ready")

	start := time.Now()
	wait.PollInfinite(apiCallRetryInterval, func() (bool, error) {
		d, err := client.Extensions().Deployments(api.NamespaceSystem).Get(kubeDiscoveryName)
		if err != nil {
			return false, nil
		}
		if d.Status.AvailableReplicas < 1 {
			return false, nil
		}
		return true, nil
	})
	fmt.Printf("<master/discovery> kube-discovery is ready after %f seconds\n", time.Since(start).Seconds())

	return nil
}
<commit_msg>Kubeadm: fix SELinux rules for kubernetes discovery service<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package master

import (
	"crypto/x509"
	"encoding/json"
	"fmt"
	"time"

	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/api"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/apis/extensions"
	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
	certutil "k8s.io/kubernetes/pkg/util/cert"
	"k8s.io/kubernetes/pkg/util/wait"
)

type kubeDiscovery struct {
	Deployment *extensions.Deployment
	Secret     *api.Secret
}

const (
	kubeDiscoveryName       = "kube-discovery"
	kubeDiscoverySecretName = "clusterinfo"
)

func encodeKubeDiscoverySecretData(s *kubeadmapi.KubeadmConfig, caCert *x509.Certificate) map[string][]byte {
	var (
		data         = map[string][]byte{}
		endpointList = []string{}
		tokenMap     = map[string]string{}
	)

	for _, addr := range s.InitFlags.API.AdvertiseAddrs {
		endpointList = append(endpointList, fmt.Sprintf("https://%s:443", addr.String()))
	}

	tokenMap[s.Secrets.TokenID] = s.Secrets.BearerToken

	data["endpoint-list.json"], _ = json.Marshal(endpointList)
	data["token-map.json"], _ = json.Marshal(tokenMap)
	data["ca.pem"] = certutil.EncodeCertPEM(caCert)

	return data
}

func newKubeDiscoveryPodSpec(s *kubeadmapi.KubeadmConfig) api.PodSpec {
	return api.PodSpec{
		// We have to use host network namespace, as `HostPort`/`HostIP` are Docker's
		// buisness and CNI support isn't quite there yet (except for kubenet)
		// (see https://github.com/kubernetes/kubernetes/issues/31307)
		// TODO update this when #31307 is resolved
		SecurityContext: &api.PodSecurityContext{HostNetwork: true},
		Containers: []api.Container{{
			Name:    kubeDiscoveryName,
			Image:   s.EnvParams["discovery_image"],
			Command: []string{"/usr/local/bin/kube-discovery"},
			VolumeMounts: []api.VolumeMount{{
				Name:      kubeDiscoverySecretName,
				MountPath: "/tmp/secret", // TODO use a shared constant
				ReadOnly:  true,
			}},
			Ports: []api.ContainerPort{
				// TODO when CNI issue (#31307) is resolved, we should consider adding
				// `HostIP: s.API.AdvertiseAddrs[0]`, if there is only one address`
				{Name: "http", ContainerPort: 9898, HostPort: 9898},
			},
			SecurityContext: &api.SecurityContext{
				SELinuxOptions: &api.SELinuxOptions{
					// TODO: This implies our discovery container is not being restricted by
					// SELinux. This is not optimal and would be nice to adjust in future
					// so it can read /tmp/secret, but for now this avoids recommending
					// setenforce 0 system-wide.
					Type: "unconfined_t",
				},
			},
		}},
		Volumes: []api.Volume{{
			Name: kubeDiscoverySecretName,
			VolumeSource: api.VolumeSource{
				Secret: &api.SecretVolumeSource{SecretName: kubeDiscoverySecretName},
			}},
		},
	}
}

func newKubeDiscovery(s *kubeadmapi.KubeadmConfig, caCert *x509.Certificate) kubeDiscovery {
	kd := kubeDiscovery{
		Deployment: NewDeployment(kubeDiscoveryName, 1, newKubeDiscoveryPodSpec(s)),
		Secret: &api.Secret{
			ObjectMeta: api.ObjectMeta{Name: kubeDiscoverySecretName},
			Type:       api.SecretTypeOpaque,
			Data:       encodeKubeDiscoverySecretData(s, caCert),
		},
	}

	SetMasterTaintTolerations(&kd.Deployment.Spec.Template.ObjectMeta)
	SetMasterNodeAffinity(&kd.Deployment.Spec.Template.ObjectMeta)

	return kd
}

func CreateDiscoveryDeploymentAndSecret(s *kubeadmapi.KubeadmConfig, client *clientset.Clientset, caCert *x509.Certificate) error {
	kd := newKubeDiscovery(s, caCert)

	if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(kd.Deployment); err != nil {
		return fmt.Errorf("<master/discovery> failed to create %q deployment [%v]", kubeDiscoveryName, err)
	}
	if _, err := client.Secrets(api.NamespaceSystem).Create(kd.Secret); err != nil {
		return fmt.Errorf("<master/discovery> failed to create %q secret [%v]", kubeDiscoverySecretName, err)
	}

	fmt.Println("<master/discovery> created essential addon: kube-discovery, waiting for it to become ready")

	start := time.Now()
	wait.PollInfinite(apiCallRetryInterval, func() (bool, error) {
		d, err := client.Extensions().Deployments(api.NamespaceSystem).Get(kubeDiscoveryName)
		if err != nil {
			return false, nil
		}
		if d.Status.AvailableReplicas < 1 {
			return false, nil
		}
		return true, nil
	})
	fmt.Printf("<master/discovery> kube-discovery is ready after %f seconds\n", time.Since(start).Seconds())

	return nil
}
<|endoftext|>