<commit_before>package cli

import (
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"strings"

	"github.com/gin-gonic/gin"
	"github.com/jessevdk/go-flags"

	"github.com/sosedoff/pgweb/pkg/api"
	"github.com/sosedoff/pgweb/pkg/bookmarks"
	"github.com/sosedoff/pgweb/pkg/client"
	"github.com/sosedoff/pgweb/pkg/command"
	"github.com/sosedoff/pgweb/pkg/connection"
	"github.com/sosedoff/pgweb/pkg/shared"
	"github.com/sosedoff/pgweb/pkg/util"
)

var options command.Options

func exitWithMessage(message string) {
	fmt.Println("Error:", message)
	os.Exit(1)
}

func initClientUsingBookmark(bookmarkPath, bookmarkName string) (*client.Client, error) {
	bookmark, err := bookmarks.GetBookmark(bookmarkPath, bookmarkName)
	if err != nil {
		return nil, err
	}

	opt := bookmark.ConvertToOptions()
	var connStr string

	if opt.Url != "" { // if the bookmark has url set, use it
		connStr = opt.Url
	} else {
		connStr, err = connection.BuildStringFromOptions(opt)
		if err != nil {
			return nil, fmt.Errorf("error building connection string: %v", err)
		}
	}

	var ssh *shared.SSHInfo
	if !bookmark.SSHInfoIsEmpty() {
		ssh = bookmark.Ssh
	}

	return client.NewFromUrl(connStr, ssh)
}

func initClient() {
	if connection.IsBlank(command.Opts) && options.Bookmark == "" {
		return
	}

	var cl *client.Client
	var err error

	if options.Bookmark != "" {
		cl, err = initClientUsingBookmark(bookmarks.Path(options.BookmarksDir), options.Bookmark)
	} else {
		cl, err = client.New()
	}

	if err != nil {
		exitWithMessage(err.Error())
	}

	if command.Opts.Debug {
		fmt.Println("Server connection string:", cl.ConnectionString)
	}

	fmt.Println("Connecting to server...")
	err = cl.Test()
	if err != nil {
		msg := err.Error()

		// Check if we're trying to connect to the default database.
		if command.Opts.DbName == "" {
			// If database does not exist, allow user to connect from the UI.
			if strings.Contains(msg, "database") && strings.Contains(msg, "does not exist") {
				fmt.Println("Error:", msg)
				return
			}
			// Do not bail if local server is not running.
			if strings.Contains(msg, "connection refused") {
				fmt.Println("Error:", msg)
				return
			}
		}

		exitWithMessage(msg)
	}

	if !command.Opts.Sessions {
		fmt.Printf("Connected to %s\n", cl.ServerVersion())
	}

	fmt.Println("Checking database objects...")
	_, err = cl.Objects()
	if err != nil {
		exitWithMessage(err.Error())
	}

	api.DbClient = cl
}

func initOptions() {
	opts, err := command.ParseOptions(os.Args)
	if err != nil {
		switch err.(type) {
		case *flags.Error:
			// no need to print error, flags package already does that
		default:
			fmt.Println(err.Error())
		}
		os.Exit(1)
	}
	command.Opts = opts
	options = opts

	if options.Version {
		printVersion()
		os.Exit(0)
	}

	if options.ReadOnly {
		msg := `------------------------------------------------------
SECURITY WARNING: You are running pgweb in read-only mode.
This mode is designed for environments where users could potentially delete / change data.
For proper read-only access please follow postgresql role management documentation.
------------------------------------------------------`
		fmt.Println(msg)
	}

	printVersion()
}

func printVersion() {
	str := fmt.Sprintf("Pgweb v%s", command.Version)
	if command.GitCommit != "" {
		str += fmt.Sprintf(" (git: %s)", command.GitCommit)
	}

	fmt.Println(str)
}

func startServer() {
	router := gin.Default()

	// Enable HTTP basic authentication only if both user and password are set
	if options.AuthUser != "" && options.AuthPass != "" {
		auth := map[string]string{options.AuthUser: options.AuthPass}
		router.Use(gin.BasicAuth(auth))
	}

	api.SetupRoutes(router)

	fmt.Println("Starting server...")
	go func() {
		err := router.Run(fmt.Sprintf("%v:%v", options.HttpHost, options.HttpPort))
		if err != nil {
			fmt.Println("Cant start server:", err)
			if strings.Contains(err.Error(), "address already in use") {
				openPage()
			}
			os.Exit(1)
		}
	}()
}

func handleSignals() {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill)
	<-c
}

func openPage() {
	url := fmt.Sprintf("http://%v:%v/%s", options.HttpHost, options.HttpPort, options.Prefix)
	fmt.Println("To view database open", url, "in browser")

	if options.SkipOpen {
		return
	}

	_, err := exec.Command("which", "open").Output()
	if err != nil {
		return
	}

	exec.Command("open", url).Output()
}

func Run() {
	initOptions()
	initClient()

	if api.DbClient != nil {
		defer api.DbClient.Close()
	}

	if !options.Debug {
		gin.SetMode("release")
	}

	// Print memory usage every 30 seconds with debug flag
	if options.Debug {
		util.StartProfiler()
	}

	// Start session cleanup worker
	if options.Sessions && !command.Opts.DisableConnectionIdleTimeout {
		go api.StartSessionCleanup()
	}

	startServer()
	openPage()
	handleSignals()
}
<commit_msg>Fix startup error when DATABASE_URL is set<commit_after>package cli

import (
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"strings"

	"github.com/gin-gonic/gin"
	"github.com/jessevdk/go-flags"

	"github.com/sosedoff/pgweb/pkg/api"
	"github.com/sosedoff/pgweb/pkg/bookmarks"
	"github.com/sosedoff/pgweb/pkg/client"
	"github.com/sosedoff/pgweb/pkg/command"
	"github.com/sosedoff/pgweb/pkg/connection"
	"github.com/sosedoff/pgweb/pkg/shared"
	"github.com/sosedoff/pgweb/pkg/util"
)

var options command.Options

func exitWithMessage(message string) {
	fmt.Println("Error:", message)
	os.Exit(1)
}

func initClientUsingBookmark(bookmarkPath, bookmarkName string) (*client.Client, error) {
	bookmark, err := bookmarks.GetBookmark(bookmarkPath, bookmarkName)
	if err != nil {
		return nil, err
	}

	opt := bookmark.ConvertToOptions()
	var connStr string

	if opt.Url != "" { // if the bookmark has url set, use it
		connStr = opt.Url
	} else {
		connStr, err = connection.BuildStringFromOptions(opt)
		if err != nil {
			return nil, fmt.Errorf("error building connection string: %v", err)
		}
	}

	var ssh *shared.SSHInfo
	if !bookmark.SSHInfoIsEmpty() {
		ssh = bookmark.Ssh
	}

	return client.NewFromUrl(connStr, ssh)
}

func initClient() {
	if connection.IsBlank(command.Opts) && options.Bookmark == "" {
		return
	}

	var cl *client.Client
	var err error

	if options.Bookmark != "" {
		cl, err = initClientUsingBookmark(bookmarks.Path(options.BookmarksDir), options.Bookmark)
	} else {
		cl, err = client.New()
	}
	if err != nil {
		exitWithMessage(err.Error())
	}

	if command.Opts.Debug {
		fmt.Println("Server connection string:", cl.ConnectionString)
	}

	fmt.Println("Connecting to server...")
	if err := cl.Test(); err != nil {
		msg := err.Error()

		// Check if we're trying to connect to the default database.
		if command.Opts.DbName == "" && command.Opts.Url == "" {
			// If database does not exist, allow user to connect from the UI.
			if strings.Contains(msg, "database") && strings.Contains(msg, "does not exist") {
				fmt.Println("Error:", msg)
				return
			}
			// Do not bail if local server is not running.
			if strings.Contains(msg, "connection refused") {
				fmt.Println("Error:", msg)
				return
			}
		}

		exitWithMessage(msg)
	}

	if !command.Opts.Sessions {
		fmt.Printf("Connected to %s\n", cl.ServerVersion())
	}

	fmt.Println("Checking database objects...")
	_, err = cl.Objects()
	if err != nil {
		exitWithMessage(err.Error())
	}

	api.DbClient = cl
}

func initOptions() {
	opts, err := command.ParseOptions(os.Args)
	if err != nil {
		switch err.(type) {
		case *flags.Error:
			// no need to print error, flags package already does that
		default:
			fmt.Println(err.Error())
		}
		os.Exit(1)
	}
	command.Opts = opts
	options = opts

	if options.Version {
		printVersion()
		os.Exit(0)
	}

	if options.ReadOnly {
		msg := `------------------------------------------------------
SECURITY WARNING: You are running pgweb in read-only mode.
This mode is designed for environments where users could potentially delete / change data.
For proper read-only access please follow postgresql role management documentation.
------------------------------------------------------`
		fmt.Println(msg)
	}

	printVersion()
}

func printVersion() {
	str := fmt.Sprintf("Pgweb v%s", command.Version)
	if command.GitCommit != "" {
		str += fmt.Sprintf(" (git: %s)", command.GitCommit)
	}

	fmt.Println(str)
}

func startServer() {
	router := gin.Default()

	// Enable HTTP basic authentication only if both user and password are set
	if options.AuthUser != "" && options.AuthPass != "" {
		auth := map[string]string{options.AuthUser: options.AuthPass}
		router.Use(gin.BasicAuth(auth))
	}

	api.SetupRoutes(router)

	fmt.Println("Starting server...")
	go func() {
		err := router.Run(fmt.Sprintf("%v:%v", options.HttpHost, options.HttpPort))
		if err != nil {
			fmt.Println("Cant start server:", err)
			if strings.Contains(err.Error(), "address already in use") {
				openPage()
			}
			os.Exit(1)
		}
	}()
}

func handleSignals() {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill)
	<-c
}

func openPage() {
	url := fmt.Sprintf("http://%v:%v/%s", options.HttpHost, options.HttpPort, options.Prefix)
	fmt.Println("To view database open", url, "in browser")

	if options.SkipOpen {
		return
	}

	_, err := exec.Command("which", "open").Output()
	if err != nil {
		return
	}

	exec.Command("open", url).Output()
}

func Run() {
	initOptions()
	initClient()

	if api.DbClient != nil {
		defer api.DbClient.Close()
	}

	if !options.Debug {
		gin.SetMode("release")
	}

	// Print memory usage every 30 seconds with debug flag
	if options.Debug {
		util.StartProfiler()
	}

	// Start session cleanup worker
	if options.Sessions && !command.Opts.DisableConnectionIdleTimeout {
		go api.StartSessionCleanup()
	}

	startServer()
	openPage()
	handleSignals()
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Doctl Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ssh

import (
	"fmt"
	"os"
	"os/exec"
	"strconv"

	"github.com/digitalocean/doctl/pkg/runner"
	"github.com/fatih/color"
)

// Options is the type used to specify options passed to the SSH command
type Options map[string]interface{}

// Runner runs ssh commands.
type Runner struct {
	User            string
	Host            string
	KeyPath         string
	Port            int
	AgentForwarding bool
	Command         string
}

var _ runner.Runner = &Runner{}

// Run ssh.
func (r *Runner) Run() error {
	args := []string{}
	if r.KeyPath != "" {
		args = append(args, "-i", r.KeyPath)
	}

	sshHost := r.Host
	if r.User != "" {
		sshHost = r.User + "@" + sshHost
	}

	if r.Port > 0 {
		args = append(args, "-p", strconv.Itoa(r.Port))
	}

	if r.AgentForwarding {
		args = append(args, "-A")
	}

	args = append(args, sshHost)
	if r.Command != "" {
		args = append(args, r.Command)
	}

	cmd := exec.Command("ssh", args...)

	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	cmd.Stdin = os.Stdin

	err := cmd.Run()
	if err != nil {
		_, isSnap := os.LookupEnv("SNAP")

		if os.IsPermission(err) && isSnap {
			msg := "Using the doctl Snap? Grant access to the ssh-keys interface with this command: sudo snap connect doctl:ssh-keys :ssh-keys"
			fmt.Fprintf(color.Error, "%s: %s", color.YellowString("Warning"), msg)
			return err
		}

		return err
	}

	return nil
}
<commit_msg>line break and updated error message<commit_after>/*
Copyright 2018 The Doctl Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ssh

import (
	"fmt"
	"os"
	"os/exec"
	"strconv"

	"github.com/digitalocean/doctl/pkg/runner"
	"github.com/fatih/color"
)

// Options is the type used to specify options passed to the SSH command
type Options map[string]interface{}

// Runner runs ssh commands.
type Runner struct {
	User            string
	Host            string
	KeyPath         string
	Port            int
	AgentForwarding bool
	Command         string
}

var _ runner.Runner = &Runner{}

// Run ssh.
func (r *Runner) Run() error {
	args := []string{}
	if r.KeyPath != "" {
		args = append(args, "-i", r.KeyPath)
	}

	sshHost := r.Host
	if r.User != "" {
		sshHost = r.User + "@" + sshHost
	}

	if r.Port > 0 {
		args = append(args, "-p", strconv.Itoa(r.Port))
	}

	if r.AgentForwarding {
		args = append(args, "-A")
	}

	args = append(args, sshHost)
	if r.Command != "" {
		args = append(args, r.Command)
	}

	cmd := exec.Command("ssh", args...)

	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	cmd.Stdin = os.Stdin

	err := cmd.Run()
	if err != nil {
		_, isSnap := os.LookupEnv("SNAP")

		if os.IsPermission(err) && isSnap {
			msg := "Using the doctl Snap? Grant access to the ssh-keys interface with this command: sudo snap connect doctl:ssh-keys"
			fmt.Fprintf(color.Error, "%s: %s\n", color.YellowString("Warning"), msg)
			return err
		}

		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package plex

import (
	"fmt"
	"io/ioutil"
	"net/http"
)

// Host defines the data to be stored for server objects
type Host struct {
	Name     string
	Hostname string
	Port     int
	Ssl      bool
	Token    string
}

// CreateURI assembles the URI for an API request
func CreateURI(server Host, path string) string {
	if server.Ssl {
		return fmt.Sprintf("https://%v:%v/%v", server.Hostname, server.Port, path)
	}
	return fmt.Sprintf("http://%v:%v/%v", server.Hostname, server.Port, path)
}

// Search returns all episodes for a given TV Show
func Search(server Host, title string) (string, error) {
	uri := CreateURI(server, fmt.Sprintf("search?type=2&query=%v", title))
	resp, err := apiRequest("GET", uri, server.Token, nil)
	if err != nil {
		return "", fmt.Errorf("Failed to get episodes for show %q from server %q", title, server.Name)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return "", fmt.Errorf("Failed with HTTP Response %q", resp.Status)
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", fmt.Errorf("Failed to read body content")
	}

	return string(body), nil
}
<commit_msg>Update search to work correctly<commit_after>package plex

import (
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"net/http"
)

// Host defines the data to be stored for server objects
type Host struct {
	Name     string
	Hostname string
	Port     int
	Ssl      bool
	Token    string
}

// CreateURI assembles the URI for an API request
func CreateURI(server Host, path string) string {
	if server.Ssl {
		return fmt.Sprintf("https://%v:%v/%v", server.Hostname, server.Port, path)
	}
	return fmt.Sprintf("http://%v:%v/%v", server.Hostname, server.Port, path)
}

// Search returns all episodes for a given TV Show
func Search(server Host, title string) (Show, error) {

	uri := CreateURI(server, fmt.Sprintf("search?type=2&query=%v", title))
	// log.Printf("Performing REST request to %q", uri)
	resp, err := apiRequest("GET", uri, server.Token, nil)
	if err != nil {
		return Show{}, fmt.Errorf("error getting episodes for show %q from server %q", title, server.Name)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return Show{}, fmt.Errorf("unexpected HTTP Response %q", resp.Status)
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return Show{}, fmt.Errorf("error reading response %v", err)
	}

	results := SR{}

	err = xml.Unmarshal(body, &results)
	if err != nil {
		return Show{}, fmt.Errorf("error parsing xml response: %v", err)
	}
	for _, x := range results.Directories {
		if x.Name == title {
			return x, nil
		}
	}
	return Show{}, fmt.Errorf("no show found matching name %q", title)
}

//SR contains search results
type SR struct {
	XMLName     xml.Name `xml:"MediaContainer"`
	Directories []Show   `xml:"Directory"`
}

//Show defines the structure of a Plex TV Show
type Show struct {
	ID           int    `xml:"ratingKey,attr"`
	Name         string `xml:"title,attr"`
	EpisodeCount int    `xml:"leafCount,attr"`
}
<|endoftext|>


<commit_before>package loader

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	urlModule "net/url"
	"strconv"
	"sync"

	"github.com/AxelUser/gowork/errors"
	"github.com/AxelUser/gowork/events"
	"github.com/AxelUser/gowork/models"
)

func createBaseURL(config models.ParserConfig) (string, error) {
	url, err := urlModule.Parse(config.URL)
	if err != nil {
		return "", errors.NewLoadDataError(config.URL, "Could not parse Base URL", err)
	}

	q := url.Query()
	for k, v := range config.Defaults {
		q.Add(k, fmt.Sprintf("%v", v))
	}

	url.RawQuery = q.Encode()
	urlString := url.String()
	return urlString, nil
}

func createURLs(baseURL string, queries []models.ParserQuery) (map[string]string, error) {
	skillURLMap := make(map[string]string)

	for _, query := range queries {
		req, err := http.NewRequest("GET", baseURL, nil)
		if err != nil {
			return nil, errors.NewLoadSkillError(query.Alias, "Could not create request for skill", err)
		}

		q := req.URL.Query()
		q.Add("text", query.Text)

		req.URL.RawQuery = q.Encode()

		skillURL := req.URL.String()

		skillURLMap[query.Alias] = skillURL
	}

	return skillURLMap, nil
}

func getNextPageURL(alias string, url string, nextPage int) (string, error) {
	urlBuild, err := urlModule.Parse(url)
	if err != nil {
		return "", errors.NewLoadSkillError(alias, "Could not create URL for next page", err)
	}

	q := urlBuild.Query()
	q.Set("page", strconv.Itoa(nextPage))
	urlBuild.RawQuery = q.Encode()
	nextPageURL := urlBuild.String()

	return nextPageURL, nil
}

func loadPage(alias string, url string) (*models.VacancySearchPage, error) {
	res, httpErr := http.Get(url)
	if httpErr != nil {
		return nil, errors.NewLoadSkillError(alias, "Could not send GET request", httpErr)
	}
	defer res.Body.Close()

	body, bodyErr := ioutil.ReadAll(res.Body)
	if bodyErr != nil {
		return nil, errors.NewLoadSkillError(alias, "Could not read data from Body", bodyErr)
	}

	var page models.VacancySearchPage
	jsonErr := json.Unmarshal(body, &page)
	if jsonErr != nil {
		return nil, errors.NewLoadSkillError(alias, "Could not unmarshal JSON", jsonErr)
	}

	return &page, nil
}

func parseVacancyStats(page models.VacancySearchPage) []models.VacancyStats {
	data := make([]models.VacancyStats, len(page.Items))
	// Handle empty items and log!
	for i, v := range page.Items {
		data[i] = models.NewVacancyStats(v.ID, v.URL, v.Salary.From, v.Salary.To, v.Salary.Currency)
	}

	return data
}

func isLastPage(page models.VacancySearchPage) bool {
	return page.Page >= page.Pages-1
}

func loadDataPerSkillAsync(jobsCh <-chan models.LoaderJob, eventCh chan<- events.DataLoadedEvent, wg *sync.WaitGroup) {
	defer wg.Done()
	for job := range jobsCh {
		pageURL := job.URL
		var pages []models.VacancySearchPage
		pageModel, err := loadPage(job.Alias, pageURL)
		if err != nil {
			eventCh <- events.NewDataLoadedEventWithError(job.Alias, job.URL, err)
		} else {
			pages = append(pages, *pageModel)
			var allStats []models.VacancyStats

			for !isLastPage(*pageModel) {
				pageURL, err = getNextPageURL(job.Alias, pageURL, pageModel.Page+1)
				if err != nil {
					eventCh <- events.NewDataLoadedEventWithError(job.Alias, job.URL, err)
					return
				}

				pageModel, err = loadPage(job.Alias, pageURL)
				if err != nil {
					eventCh <- events.NewDataLoadedEventWithError(job.Alias, job.URL, err)
					return
				}

				pages = append(pages, *pageModel)
			}

			for _, page := range pages {
				allStats = append(allStats, parseVacancyStats(page)...)
			}
			eventCh <- events.NewDataLoadedEvent(job.Alias, job.URL, allStats)
		}
	}

}

func loadAll(urls map[string]string, count int) ([]models.VacancyStats, error) {
	var all []models.VacancyStats
	dataCh := make(chan events.DataLoadedEvent, len(urls))
	jobsCh := make(chan models.LoaderJob, len(urls))
	var wg sync.WaitGroup

	wg.Add(count)
	//create workers for loading vacancies
	for i := 0; i < count; i++ {
		go loadDataPerSkillAsync(jobsCh, dataCh, &wg)
	}

	log.Printf("Workers in pool: %d\n", count)

	for alias, url := range urls {
		jobsCh <- models.NewLoaderJob(alias, url)
	}
	close(jobsCh)

	for i := 0; i < len(urls); i++ {
		event := <-dataCh
		log.Println(event.String())
		all = append(all, event.Data...)
	}
	wg.Wait()
	return all, nil
}

// Load data from HeadHunter API
func Load(config models.ParserConfig) ([]models.VacancyStats, error) {
	var allStats []models.VacancyStats

	baseURL, err := createBaseURL(config)
	if err != nil {
		return allStats, err
	}
	urls, err := createURLs(baseURL, config.Queries)
	if err != nil {
		return allStats, err
	}
	log.Printf("Loading vacancies from %s", config.URL)

	allStats, err = loadAll(urls, config.WorkersCount)
	if err != nil {
		return nil, err
	}
	log.Printf("\nLoaded %d item(s)\n", len(allStats))

	return allStats, nil
}
<commit_msg>Fixed adding null array<commit_after>package loader

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	urlModule "net/url"
	"strconv"
	"sync"

	"github.com/AxelUser/gowork/errors"
	"github.com/AxelUser/gowork/events"
	"github.com/AxelUser/gowork/models"
)

func createBaseURL(config models.ParserConfig) (string, error) {
	url, err := urlModule.Parse(config.URL)
	if err != nil {
		return "", errors.NewLoadDataError(config.URL, "Could not parse Base URL", err)
	}

	q := url.Query()
	for k, v := range config.Defaults {
		q.Add(k, fmt.Sprintf("%v", v))
	}

	url.RawQuery = q.Encode()
	urlString := url.String()
	return urlString, nil
}

func createURLs(baseURL string, queries []models.ParserQuery) (map[string]string, error) {
	skillURLMap := make(map[string]string)

	for _, query := range queries {
		req, err := http.NewRequest("GET", baseURL, nil)
		if err != nil {
			return nil, errors.NewLoadSkillError(query.Alias, "Could not create request for skill", err)
		}

		q := req.URL.Query()
		q.Add("text", query.Text)

		req.URL.RawQuery = q.Encode()

		skillURL := req.URL.String()

		skillURLMap[query.Alias] = skillURL
	}

	return skillURLMap, nil
}

func getNextPageURL(alias string, url string, nextPage int) (string, error) {
	urlBuild, err := urlModule.Parse(url)
	if err != nil {
		return "", errors.NewLoadSkillError(alias, "Could not create URL for next page", err)
	}

	q := urlBuild.Query()
	q.Set("page", strconv.Itoa(nextPage))
	urlBuild.RawQuery = q.Encode()
	nextPageURL := urlBuild.String()

	return nextPageURL, nil
}

func loadPage(alias string, url string) (*models.VacancySearchPage, error) {
	res, httpErr := http.Get(url)
	if httpErr != nil {
		return nil, errors.NewLoadSkillError(alias, "Could not send GET request", httpErr)
	}
	defer res.Body.Close()

	body, bodyErr := ioutil.ReadAll(res.Body)
	if bodyErr != nil {
		return nil, errors.NewLoadSkillError(alias, "Could not read data from Body", bodyErr)
	}

	var page models.VacancySearchPage
	jsonErr := json.Unmarshal(body, &page)
	if jsonErr != nil {
		return nil, errors.NewLoadSkillError(alias, "Could not unmarshal JSON", jsonErr)
	}

	return &page, nil
}

func parseVacancyStats(page models.VacancySearchPage) []models.VacancyStats {
	data := make([]models.VacancyStats, len(page.Items))
	// Handle empty items and log!
	for i, v := range page.Items {
		data[i] = models.NewVacancyStats(v.ID, v.URL, v.Salary.From, v.Salary.To, v.Salary.Currency)
	}

	return data
}

func isLastPage(page models.VacancySearchPage) bool {
	return page.Page >= page.Pages-1
}

func loadDataPerSkillAsync(jobsCh <-chan models.LoaderJob, eventCh chan<- events.DataLoadedEvent, wg *sync.WaitGroup) {
	defer wg.Done()
	for job := range jobsCh {
		pageURL := job.URL
		var pages []models.VacancySearchPage
		pageModel, err := loadPage(job.Alias, pageURL)
		if err != nil {
			eventCh <- events.NewDataLoadedEventWithError(job.Alias, job.URL, err)
		} else {
			pages = append(pages, *pageModel)
			var allStats []models.VacancyStats

			for !isLastPage(*pageModel) {
				pageURL, err = getNextPageURL(job.Alias, pageURL, pageModel.Page+1)
				if err != nil {
					eventCh <- events.NewDataLoadedEventWithError(job.Alias, job.URL, err)
					return
				}

				pageModel, err = loadPage(job.Alias, pageURL)
				if err != nil {
					eventCh <- events.NewDataLoadedEventWithError(job.Alias, job.URL, err)
					return
				}

				pages = append(pages, *pageModel)
			}

			for _, page := range pages {
				allStats = append(allStats, parseVacancyStats(page)...)
			}
			eventCh <- events.NewDataLoadedEvent(job.Alias, job.URL, allStats)
		}
	}

}

func loadAll(urls map[string]string, count int) ([]models.VacancyStats, error) {
	var all []models.VacancyStats
	dataCh := make(chan events.DataLoadedEvent, len(urls))
	jobsCh := make(chan models.LoaderJob, len(urls))
	var wg sync.WaitGroup

	wg.Add(count)
	//create workers for loading vacancies
	for i := 0; i < count; i++ {
		go loadDataPerSkillAsync(jobsCh, dataCh, &wg)
	}

	log.Printf("Workers in pool: %d\n", count)

	for alias, url := range urls {
		jobsCh <- models.NewLoaderJob(alias, url)
	}
	close(jobsCh)

	for i := 0; i < len(urls); i++ {
		event := <-dataCh
		log.Println(event.String())
		if event.IsSuccess() {
			all = append(all, event.Data...)
		}
	}
	wg.Wait()
	return all, nil
}

// Load data from HeadHunter API
func Load(config models.ParserConfig) ([]models.VacancyStats, error) {
	var allStats []models.VacancyStats

	baseURL, err := createBaseURL(config)
	if err != nil {
		return allStats, err
	}
	urls, err := createURLs(baseURL, config.Queries)
	if err != nil {
		return allStats, err
	}
	log.Printf("Loading vacancies from %s", config.URL)

	allStats, err = loadAll(urls, config.WorkersCount)
	if err != nil {
		return nil, err
	}
	log.Printf("\nLoaded %d item(s)\n", len(allStats))

	return allStats, nil
}
<|endoftext|>


<commit_before>package migrator

import (
	"path/filepath"
	"runtime"

	"github.com/armon/gomdb"
	"github.com/boltdb/bolt"
)

const (
	dbLogs   = "logs"
	dbConf   = "conf"
	raftPath = "raft"

	mdbPath = "mdb"
	mdbMode = 0755

	boltFilename = "raft.db"
	boltFileMode = 0600

	// Maximum log sizes for LMDB. These mirror settings in Consul
	// and are automatically set based on the runtime.
	maxLogSize32bit uint64 = 8 * 1024 * 1024 * 1024
	maxLogSize64bit uint64 = 64 * 1024 * 1024 * 1024
)

type Migrator struct {
	dataDir string
	mdb     *mdb.Env
	bolt    *bolt.DB
}

func NewMigrator(dataDir string) (*Migrator, error) {
	// Create the struct
	m := &Migrator{
		dataDir: dataDir,
	}

	// Connect MDB
	if err := m.mdbConnect(); err != nil {
		return nil, err
	}

	// Connect BoltDB
	if err := m.boltConnect(); err != nil {
		return nil, err
	}
	return m, nil
}

func (m *Migrator) mdbConnect() error {
	// Create the env
	env, err := mdb.NewEnv()
	if err != nil {
		return err
	}

	// Calculate and set the max size
	size := maxLogSize32bit
	if runtime.GOARCH == "amd64" {
		size = maxLogSize64bit
	}
	if err := env.SetMapSize(size); err != nil {
		return err
	}

	// Open the connection
	err = env.Open(filepath.Join(m.dataDir, mdbPath), mdb.NOTLS, mdbMode)
	if err != nil {
		return err
	}

	// Return the new environment
	m.mdb = env
	return nil
}

func (m *Migrator) boltConnect() error {
	// Connect to the new bolt raft store
	file := filepath.Join(m.dataDir, raftPath, boltFilename)
	b, err := bolt.Open(file, boltFileMode, nil)
	if err != nil {
		return err
	}
	m.bolt = b
	return nil
}

func (m *Migrator) migrateStableStore() error {
	// Begin a new BoltDB transaction
	btx, err := m.bolt.Begin(true)
	if err != nil {
		return err
	}
	defer btx.Rollback()
	bucket := btx.Bucket([]byte(dbConf))

	// Begin a transaction
	mtx, err := m.mdb.BeginTxn(nil, mdb.RDONLY)
	if err != nil {
		return err
	}

	// Open the sub-db
	dbi, err := mtx.DBIOpen(dbConf, 0)
	if err != nil {
		mtx.Abort()
		return err
	}
	defer mtx.Abort()

	// Get a new cursor and seek to the first key
	mcurs, err := mtx.CursorOpen(dbi)
	if err != nil {
		return err
	}
	if _, _, err := mcurs.Get(nil, mdb.FIRST); err != nil {
		return err
	}

	// Loop through all of the keys, writing them out to the bolt store
	// as we go. We will stop when we reach the end of the StableStore.
	for {
		// Get the current key
		k, v, err := mcurs.Get(nil, mdb.GET_CURRENT)
		if err != nil {
			return err
		}
		if err := bucket.Put(k, v); err != nil {
			return err
		}
		if err := btx.Commit(); err != nil {
			return err
		}
		if _, _, err := mcurs.Get(nil, mdb.NEXT); err != nil {
			return nil
		}
	}
}

func (m *Migrator) Migrate() error {
	return m.migrateStableStore()
}
<commit_msg>Basic stablestore migrator works<commit_after>package migrator

import (
	"path/filepath"
	"runtime"

	"github.com/armon/gomdb"
	"github.com/hashicorp/raft-boltdb"
)

const (
	dbLogs   = "logs"
	dbConf   = "conf"
	raftPath = "raft"

	mdbPath = "mdb"
	mdbMode = 0755

	boltFilename = "raft.db"

	// Maximum log sizes for LMDB. These mirror settings in Consul
	// and are automatically set based on the runtime.
	maxLogSize32bit uint64 = 8 * 1024 * 1024 * 1024
	maxLogSize64bit uint64 = 64 * 1024 * 1024 * 1024
)

type Migrator struct {
	dataDir   string
	mdb       *mdb.Env
	boltStore *raftboltdb.BoltStore
}

func NewMigrator(dataDir string) (*Migrator, error) {
	// Create the struct
	m := &Migrator{
		dataDir: dataDir,
	}

	// Connect MDB
	if err := m.mdbConnect(); err != nil {
		return nil, err
	}

	// Connect BoltDB
	if err := m.boltConnect(); err != nil {
		return nil, err
	}
	return m, nil
}

func (m *Migrator) mdbConnect() error {
	// Create the env
	env, err := mdb.NewEnv()
	if err != nil {
		return err
	}

	// Allow 2 sub-dbs
	if err := env.SetMaxDBs(mdb.DBI(2)); err != nil {
		return err
	}

	// Calculate and set the max size
	size := maxLogSize32bit
	if runtime.GOARCH == "amd64" {
		size = maxLogSize64bit
	}
	if err := env.SetMapSize(size); err != nil {
		return err
	}

	// Open the connection
	err = env.Open(filepath.Join(m.dataDir, raftPath, mdbPath), mdb.NOTLS, mdbMode)
	if err != nil {
		return err
	}

	// Return the new environment
	m.mdb = env
	return nil
}

func (m *Migrator) boltConnect() error {
	// Connect to the new BoltStore
	raftFile := filepath.Join(m.dataDir, raftPath, boltFilename)
	store, err := raftboltdb.NewBoltStore(raftFile)
	if err != nil {
		return err
	}

	m.boltStore = store
	return nil
}

func (m *Migrator) migrateStableStore() error {
	// Begin a new MDB transaction
	mtx, err := m.mdb.BeginTxn(nil, mdb.RDONLY)
	if err != nil {
		return err
	}

	// Open the sub-db
	dbi, err := mtx.DBIOpen(dbConf, 0)
	if err != nil {
		mtx.Abort()
		return err
	}
	defer mtx.Abort()

	// Get a new cursor and seek to the first key
	mcurs, err := mtx.CursorOpen(dbi)
	if err != nil {
		return err
	}
	if _, _, err := mcurs.Get(nil, mdb.FIRST); err != nil {
		return err
	}

	// Loop through all of the keys, writing them out to the bolt store
	// as we go. We will stop when we reach the end of the StableStore.
	for {
		// Get the current key
		k, v, err := mcurs.Get(nil, mdb.GET_CURRENT)
		if err != nil {
			return err
		}

		// Write the value into the BoltStore
		if err := m.boltStore.Set(k, v); err != nil {
			return err
		}

		// Move the cursor to the next entry
		if k, _, err := mcurs.Get(nil, mdb.NEXT); err != nil {
			if err == mdb.NotFound || len(k) == 0 {
				return nil
			}
			return err
		}
	}
}

func (m *Migrator) Migrate() error {
	return m.migrateStableStore()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
)

// Index is just a test actually
func Index(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, "Insapp REST API - v.0.1")
}

func Credit(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, "")
	fmt.Fprintln(w, "
Insapp
")

	fmt.Fprintln(w, "
Quoi ?
")

	fmt.Fprintln(w, `

Insapp est une application qui a pour but de promouvoir l'associatif au sein de l'INSA Rennes.
L'application a été initialement imaginée et développée par Antoine Crochet et Florent Thomas-Morel en 2016.
Nous avons travaillé de pair avec l'AEIR et notamment Théau Jubin et Antoine Tulasne ainsi qu'avec le CRI de l'INSA Rennes.

`)

	fmt.Fprintln(w, "
Données
")

	fmt.Fprintln(w, `

Les données transmises à l'application sont matériellement stockées sur les serveurs du CRI.
Vos identifiants INSA, utilisés lors de l'inscription ne sont pas stockés pour Insapp pour des raisons de sécurité et de vie privée.
À aucun moment nous ne stockons, diffusons ou avons accès à vos identifiants INSA.
Vous n'êtes en aucun cas obligé de renseigner votre identité dans la page profil de l'application qui n'est là qu'à titre facultatif.

`)

	fmt.Fprintln(w, "
Associations
")

	fmt.Fprintln(w, `

Les associations ont accès au nombre de "like", le nombre de participants pour leurs évènements, ainsi qu'au contenu des commentaires postés sur leurs posts.
Ces commentaires restent anonymes et les associations se réservent le droit de supprimer des commentaires.
Les notifications peuvent être activées ou désactivées dans les réglages du téléphone.

`)

	fmt.Fprintln(w, "
Testeurs
")

	fmt.Fprintln(w, `

Insapp a été testé pendant 3 semaines par des beta-testeurs volontaires issuent de differents départements de l'INSA. Nous tenons à remercier :

Nans Préjean

Hugo David

Valentin Marc

Alex Gravier

Mathieu Cassard

Célestin Bodet

`)

	fmt.Fprintln(w, "")
}

func Legal(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, "")
	fmt.Fprintln(w, "
Insapp
")

	fmt.Fprintln(w, `

Les sources des informations diffusées sur INSAPP sont réputées fiables. Toutefois, INSAPP se réserve la faculté d'une non-garantie de la fiabilité des sources. Les informations données sur le site le sont à titre purement informatif. Ainsi, l'Utilisateur assume seul l'entière responsabilité de l'utilisation des informations et contenus du présent site.
L'Utilisateur s'assure de garder son mot de passe secret. Toute divulgation du mot de passe, quelle que soit sa forme, est interdite.

`)

	fmt.Fprintln(w, `

L'Utilisateur assume les risques liés à l'utilisation de son identifiant et mot de passe. Le site décline toute responsabilité.
Une garantie optimale de la sécurité et de la confidentialité des données transmises n'est pas assurée par INSAPP. Toutefois, le site s'engage à mettre en œuvre tous les moyens nécessaires afin de garantir au mieux la sécurité et la confidentialité des données.

`)

	fmt.Fprintln(w, `

Le site permet aux membres de publier des commentaires.
Dans ses publications, l'Utilisateur s’engage à respecter la charte informatique de l’INSA Rennes et les règles de droit en vigueur.
Le site exerce une modération a posteriori sur les publications et se réserve le droit de refuser leur mise en ligne, sans avoir à s’en justifier auprès du membre.

`)

	fmt.Fprintln(w, "")
}
<commit_msg>update credit<commit_after>package main

import (
	"fmt"
	"net/http"
)

// Index is just a test actually
func Index(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, "Insapp REST API - v.0.1")
}

func Credit(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, "")
	fmt.Fprintln(w, "
Insapp
")

	fmt.Fprintln(w, "
Quoi ?
")

	fmt.Fprintln(w, `

Insapp est une application qui a pour but de promouvoir l'associatif au sein de l'INSA Rennes.
L'application a été initialement imaginée et développée par Antoine Crochet et Florent Thomas-Morel en 2016.
Nous avons travaillé de pair avec l'AEIR et notamment Théau Jubin et Antoine Tulasne ainsi qu'avec le CRI de l'INSA Rennes.

`)

	fmt.Fprintln(w, "
Données
")

	fmt.Fprintln(w, `

Les données transmises à l'application sont matériellement stockées sur les serveurs du CRI.
Vos identifiants INSA, utilisés lors de l'inscription ne sont pas stockés pour Insapp pour des raisons de sécurité et de vie privée.
À aucun moment nous ne stockons, diffusons ou avons accès à vos identifiants INSA.
Vous n'êtes en aucun cas obligé de renseigner votre identité dans la page profil de l'application qui n'est là qu'à titre facultatif.

`)

	fmt.Fprintln(w, "
Associations
")

	fmt.Fprintln(w, `

Les associations ont accès au nombre de "like", le nombre de participants pour leurs évènements, ainsi qu'au contenu des commentaires postés sur leurs posts.
Ces commentaires restent anonymes et les associations se réservent le droit de supprimer des commentaires.
Les notifications peuvent être activées ou désactivées dans les réglages du téléphone.

`)

	fmt.Fprintln(w, "
Testeurs
")

	fmt.Fprintln(w, `

Insapp a été testé pendant 3 semaines par des beta-testeurs volontaires issuent de differents départements de l'INSA. Nous tenons à remercier :

Nans Préjean

Hugo David

Valentin Marc

Alex Gravier

Mathieu Cassard

Célestin Bodet

Etienne Rebout

Rémy Garcia

Jean-Baptiste Nou

Thomas Bouvier

Tanguy Le Quéré

Laurent Quénach de Quivillic

Florian Arnoud

Anthony Sharpe

`)

	fmt.Fprintln(w, "")
}

func Legal(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, "")
	fmt.Fprintln(w, "
Insapp
")

	fmt.Fprintln(w, `

Les sources des informations diffusées sur INSAPP sont réputées fiables. Toutefois, INSAPP se réserve la faculté d'une non-garantie de la fiabilité des sources. Les informations données sur le site le sont à titre purement informatif. Ainsi, l'Utilisateur assume seul l'entière responsabilité de l'utilisation des informations et contenus du présent site.
L'Utilisateur s'assure de garder son mot de passe secret. Toute divulgation du mot de passe, quelle que soit sa forme, est interdite.

`)

	fmt.Fprintln(w, `

L'Utilisateur assume les risques liés à l'utilisation de son identifiant et mot de passe. Le site décline toute responsabilité.
Une garantie optimale de la sécurité et de la confidentialité des données transmises n'est pas assurée par INSAPP. Toutefois, le site s'engage à mettre en œuvre tous les moyens nécessaires afin de garantir au mieux la sécurité et la confidentialité des données.

`)

	fmt.Fprintln(w, `

Le site permet aux membres de publier des commentaires.
Dans ses publications, l'Utilisateur s’engage à respecter la charte informatique de l’INSA Rennes et les règles de droit en vigueur.
Le site exerce une modération a posteriori sur les publications et se réserve le droit de refuser leur mise en ligne, sans avoir à s’en justifier auprès du membre.

`)

	fmt.Fprintln(w, "")
}
<|endoftext|>


<commit_before>// Package build implements Pipe and can build Go projects for
// several platforms, with pre and post hook support.
package build

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/apex/log"
	"github.com/goreleaser/goreleaser/config"
	"github.com/goreleaser/goreleaser/context"
	"github.com/goreleaser/goreleaser/internal/buildtarget"
	"github.com/goreleaser/goreleaser/internal/ext"
	"github.com/goreleaser/goreleaser/internal/name"
	zglob "github.com/mattn/go-zglob"
	"github.com/pkg/errors"
	"golang.org/x/sync/errgroup"
)

// Pipe for build
type Pipe struct{}

// Description of the pipe
func (Pipe) Description() string {
	return "Building binaries"
}

// Run the pipe
func (Pipe) Run(ctx *context.Context) error {
	for _, build := range ctx.Config.Builds {
		log.WithField("build", build).Debug("building")
		if err := checkMain(ctx, build); err != nil {
			return err
		}
		if err := runPipeOnBuild(ctx, build); err != nil {
			return err
		}
	}
	return nil
}

func checkMain(ctx *context.Context, build config.Build) error {
	var glob = build.Main
	if !strings.HasSuffix(glob, "main.go") {
		glob = glob + "/" + "*.go"
	}
	log.Debugf("glob is %s", glob)
	files, err := zglob.Glob(glob)
	if err != nil {
		return errors.Wrapf(err, "glob %s is not valid, please file a bug", glob)
	}
	log.Debugf("files %v", files)
	for _, file := range files {
		bts, err := ioutil.ReadFile(file)
		if err != nil {
			return errors.Wrapf(err, "failed to read main file %s", file)
		}
		if strings.Contains(string(bts), "func main() {") {
			return nil
		}
	}
	return fmt.Errorf("build for %s does not contain a main function", build.Binary)
}

func runPipeOnBuild(ctx *context.Context, build config.Build) error {
	if err := runHook(build.Env, build.Hooks.Pre); err != nil {
		return errors.Wrap(err, "pre hook failed")
	}
	sem := make(chan bool, ctx.Parallelism)
	var g errgroup.Group
	for _, target := range buildtarget.All(build) {
		sem <- true
		target := target
		build := build
		g.Go(func() error {
			defer func() {
				<-sem
			}()
			return doBuild(ctx, build, target)
		})
	}
	if err := g.Wait(); err != nil {
		return err
	}
	if err := runHook(build.Env, build.Hooks.Post); err != nil {
		return errors.Wrap(err, "post hook failed")
	}
	return nil
}

func runHook(env []string, hook string) error {
	if hook == "" {
		return nil
	}
	log.WithField("hook", hook).Info("running hook")
	cmd := strings.Fields(hook)
	return run(buildtarget.Runtime, cmd, env)
}

func doBuild(ctx *context.Context, build config.Build, target buildtarget.Target) error {
	folder, err := name.For(ctx, target)
	if err != nil {
		return err
	}
	var binaryName = build.Binary + ext.For(target)
	var prettyName = binaryName
	if ctx.Config.Archive.Format == "binary" {
		binaryName, err = name.ForBuild(ctx, build, target)
		if err != nil {
			return err
		}
		binaryName = binaryName + ext.For(target)
	}
	var binary = filepath.Join(ctx.Config.Dist, folder, binaryName)
	ctx.AddBinary(target.String(), folder, prettyName, binary)
	log.WithField("binary", binary).Info("building")
	cmd := []string{"go", "build"}
	if build.Flags != "" {
		cmd = append(cmd, strings.Fields(build.Flags)...)
	}
	flags, err := ldflags(ctx, build)
	if err != nil {
		return err
	}
	cmd = append(cmd, "-ldflags="+flags, "-o", binary, build.Main)
	if err := run(target, cmd, build.Env); err != nil {
		return errors.Wrapf(err, "failed to build for %s", target)
	}
	return nil
}

func run(target buildtarget.Target, command, env []string) error {
	var cmd = exec.Command(command[0], command[1:]...)
	env = append(env, target.Env()...)
	var log = log.WithField("target", target.PrettyString()).
		WithField("env", env).
		WithField("cmd", command)
	cmd.Env = append(cmd.Env, os.Environ()...)
	cmd.Env = append(cmd.Env, env...)
	log.Debug("running")
	if out, err := cmd.CombinedOutput(); err != nil {
		log.WithError(err).Debug("failed")
		return errors.New(string(out))
	}
	return nil
}
<commit_msg>refactor: improved error messages<commit_after>// Package build implements Pipe and can build Go projects for
// several platforms, with pre and post hook support.
package build

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/apex/log"
	"github.com/goreleaser/goreleaser/config"
	"github.com/goreleaser/goreleaser/context"
	"github.com/goreleaser/goreleaser/internal/buildtarget"
	"github.com/goreleaser/goreleaser/internal/ext"
	"github.com/goreleaser/goreleaser/internal/name"
	zglob "github.com/mattn/go-zglob"
	"github.com/pkg/errors"
	"golang.org/x/sync/errgroup"
)

// Pipe for build
type Pipe struct{}

// Description of the pipe
func (Pipe) Description() string {
	return "Building binaries"
}

// Run the pipe
func (Pipe) Run(ctx *context.Context) error {
	for _, build := range ctx.Config.Builds {
		log.WithField("build", build).Debug("building")
		if err := checkMain(ctx, build); err != nil {
			return err
		}
		if err := runPipeOnBuild(ctx, build); err != nil {
			return err
		}
	}
	return nil
}

func checkMain(ctx *context.Context, build config.Build) error {
	var glob = build.Main
	if !strings.HasSuffix(glob, "main.go") {
		// TODO: in real live , glob will never be empty. Maybe this is worth
		// guarding here anyway
		glob = glob + "/" + "*.go"
	}
	log.Debugf("glob is %s", glob)
	files, err := zglob.Glob(glob)
	if err != nil {
		return errors.Wrap(err, "failed to find go files")
	}
	log.WithField("files", files).Debug("go files")
	for _, file := range files {
		bts, err := ioutil.ReadFile(file)
		if err != nil {
			return errors.Wrapf(err, "failed to read file: %s", file)
		}
		if strings.Contains(string(bts), "func main() {") {
			return nil
		}
	}
	return fmt.Errorf("build for %s does not contain a main function", build.Binary)
}

func runPipeOnBuild(ctx *context.Context, build config.Build) error {
	if err := runHook(build.Env, build.Hooks.Pre); err != nil {
		return errors.Wrap(err, "pre hook failed")
	}
	sem := make(chan bool, ctx.Parallelism)
	var g errgroup.Group
	for _, target := range buildtarget.All(build) {
		sem <- true
		target := target
		build := build
		g.Go(func() error {
			defer func() {
				<-sem
			}()
			return doBuild(ctx, build, target)
		})
	}
	if err := g.Wait(); err != nil {
		return err
	}
	if err := runHook(build.Env, build.Hooks.Post); err != nil {
		return errors.Wrap(err, "post hook failed")
	}
	return nil
}

func runHook(env []string, hook string) error {
	if hook == "" {
		return nil
	}
	log.WithField("hook", hook).Info("running hook")
	cmd := strings.Fields(hook)
	return run(buildtarget.Runtime, cmd, env)
}

func doBuild(ctx *context.Context, build config.Build, target buildtarget.Target) error {
	folder, err := name.For(ctx, target)
	if err != nil {
		return err
	}
	var binaryName = build.Binary + ext.For(target)
	var prettyName = binaryName
	if ctx.Config.Archive.Format == "binary" {
		binaryName, err = name.ForBuild(ctx, build, target)
		if err != nil {
			return err
		}
		binaryName = binaryName + ext.For(target)
	}
	var binary = filepath.Join(ctx.Config.Dist, folder, binaryName)
	ctx.AddBinary(target.String(), folder, prettyName, binary)
	log.WithField("binary", binary).Info("building")
	cmd := []string{"go", "build"}
	if build.Flags != "" {
		cmd = append(cmd, strings.Fields(build.Flags)...)
	}
	flags, err := ldflags(ctx, build)
	if err != nil {
		return err
	}
	cmd = append(cmd, "-ldflags="+flags, "-o", binary, build.Main)
	if err := run(target, cmd, build.Env); err != nil {
		return errors.Wrapf(err, "failed to build for %s", target)
	}
	return nil
}

func run(target buildtarget.Target, command, env []string) error {
	var cmd = exec.Command(command[0], command[1:]...)
	env = append(env, target.Env()...)
	var log = log.WithField("target", target.PrettyString()).
		WithField("env", env).
		WithField("cmd", command)
	cmd.Env = append(cmd.Env, os.Environ()...)
	cmd.Env = append(cmd.Env, env...)
	log.Debug("running")
	if out, err := cmd.CombinedOutput(); err != nil {
		log.WithError(err).Debug("failed")
		return errors.New(string(out))
	}
	return nil
}
<|endoftext|>


<commit_before>package irckit

import (
	"fmt"
	"net"
	"strings"
	"sync"
	"time"

	"github.com/42wim/matterircd/bridge"
	"github.com/desertbit/timer"
	"github.com/sorcix/irc"
	"github.com/spf13/viper"
)

// NewUser creates a *User, wrapping a connection with metadata we need for our server.
func NewUser(c Conn) *User {
	return &User{
		Conn: c,
		UserInfo: &bridge.UserInfo{
			Host: "*",
		},
		channels: map[Channel]struct{}{},
		DecodeCh: make(chan *irc.Message),
	}
}

// NewUserNet creates a *User from a net.Conn connection.
func NewUserNet(c net.Conn) *User {
	return NewUser(&conn{
		Conn:    c,
		Encoder: irc.NewEncoder(c),
		Decoder: irc.NewDecoder(c),
	})
}

const defaultCloseMsg = "Closed."

type User struct {
	Conn

	sync.RWMutex
	*bridge.UserInfo

	BufferedMsg *irc.Message
	DecodeCh    chan *irc.Message

	channels map[Channel]struct{}

	v *viper.Viper

	UserBridge
}

func (u *User) ID() string {
	// return strings.ToLower(u.Nick)
	return strings.ToLower(u.User)
}

func (u *User) Prefix() *irc.Prefix {
	return &irc.Prefix{
		Name: u.Nick,
		User: u.User,
		Host: u.Host,
	}
}

func (u *User) Close() error {
	for ch := range u.channels {
		ch.Part(u, defaultCloseMsg)
	}
	return u.Conn.Close()
}

func (u *User) String() string {
	return u.Prefix().String()
}

func (u *User) NumChannels() int {
	u.RLock()
	defer u.RUnlock()
	return len(u.channels)
}

func (u *User) Channels() []Channel {
	u.RLock()
	channels := make([]Channel, 0, len(u.channels))
	for ch := range u.channels {
		channels = append(channels, ch)
	}
	u.RUnlock()
	return channels
}

func (u *User) VisibleTo() []*User {
	seen := map[*User]struct{}{}
	seen[u] = struct{}{}

	channels := u.Channels()
	num := 0
	for _, ch := range channels {
		// Don't include self
		num += ch.Len()
	}

	// Pre-allocate
	users := make([]*User, 0, num)
	if num == 0 {
		return users
	}

	// Get all unique users
	for _, ch := range channels {
		for _, other := range ch.Users() {
			if _, dupe := seen[other]; dupe {
				continue
			}
			seen[other] = struct{}{}
			// TODO: Check visibility (once it's implemented)
			users = append(users, other)
		}
	}
	return users
}

// Encode and send each msg until an error occurs, then returns.
func (u *User) Encode(msgs ...*irc.Message) (err error) {
	if u.Ghost {
		return nil
	}

	for _, msg := range msgs {
		if msg.Command == "PRIVMSG" && (msg.Prefix.Name == "slack" || msg.Prefix.Name == "mattermost") && msg.Prefix.Host == "service" && strings.Contains(msg.Trailing, "token") {
			logger.Debugf("-> %s %s %s", msg.Command, msg.Prefix.Name, "[token redacted]")

			err := u.Conn.Encode(msg)
			if err != nil {
				return err
			}

			continue
		}

		logger.Debugf("-> %s", msg)

		err := u.Conn.Encode(msg)
		if err != nil {
			return err
		}
	}

	return nil
}

// Decode will receive and return a decoded message, or an error.
// nolint:funlen,gocognit,gocyclo
func (u *User) Decode() {
	if u.Ghost {
		// block
		c := make(chan struct{})
		<-c
	}
	buffer := make(chan *irc.Message)
	stop := make(chan struct{})
	bufferTimeout := u.v.GetInt("PasteBufferTimeout")
	// we need at least 100
	if bufferTimeout < 100 {
		bufferTimeout = 100
	}
	logger.Debugf("using paste buffer timeout: %#v\n", bufferTimeout)
	t := timer.NewTimer(time.Duration(bufferTimeout) * time.Millisecond)
	t.Stop()
	go func(buffer chan *irc.Message, stop chan struct{}) {
		for {
			select {
			case msg := <-buffer:
				// are we starting a new buffer ?
				if u.BufferedMsg == nil {
					u.BufferedMsg = msg
					// start timer now
					t.Reset(time.Duration(bufferTimeout) * time.Millisecond)
				} else {
					// make sure we're sending to the same recipient in the buffer
					if u.BufferedMsg.Params[0] == msg.Params[0] {
						u.BufferedMsg.Trailing += "\n" + msg.Trailing
					} else {
						u.DecodeCh <- msg
					}
				}
			case <-t.C:
				if u.BufferedMsg != nil {
					// trim last newline
					u.BufferedMsg.Trailing = strings.TrimSpace(u.BufferedMsg.Trailing)
					logger.Debugf("flushing buffer: %#v\n", u.BufferedMsg)
					u.DecodeCh <- u.BufferedMsg
					// clear buffer
					u.BufferedMsg = nil
					t.Stop()
				}
			case <-stop:
				logger.Debug("closing decode()")
				return
			}
		}
	}(buffer, stop)
	for {
		msg, err := u.Conn.Decode()
		if err != nil {
			close(stop)
			if err.Error() != "EOF" {
				logger.Errorf("msg: %s err: %s", msg, err)
			}
			break
		}

		if msg == nil {
			continue
		}

		dmsg := fmt.Sprintf("<- %s", msg)
		if msg.Command == "PRIVMSG" && msg.Params != nil && (msg.Params[0] == "slack" || msg.Params[0] == "mattermost") {
			// Don't log sensitive information
			trail := strings.Split(msg.Trailing, " ")
			if (msg.Trailing != "" && trail[0] == "login") || (len(msg.Params) > 1 && msg.Params[1] == "login") {
				dmsg = fmt.Sprintf("<- PRIVMSG %s :login [redacted]", msg.Params[0])
			}
		}
		// PRIVMSG can be buffered
		if msg.Command == "PRIVMSG" {
			logger.Debugf("B: %#v\n", dmsg)
			buffer <- msg
		} else {
			logger.Debug(dmsg)
			u.DecodeCh <- msg
		}
	}
}

func (u *User) createService(nick string, what string) {
	u.CreateUserFromInfo(
		&bridge.UserInfo{
			Nick:  nick,
			User:  nick,
			Real:  what,
			Host:  "service",
			Ghost: true,
		})
}
<commit_msg>Flush buffer on ctcp action. Fixes #367 (#371)<commit_after>package irckit

import (
	"fmt"
	"net"
	"strings"
	"sync"
	"time"

	"github.com/42wim/matterircd/bridge"
	"github.com/desertbit/timer"
	"github.com/sorcix/irc"
	"github.com/spf13/viper"
)

// NewUser creates a *User, wrapping a connection with metadata we need for our server.
func NewUser(c Conn) *User {
	return &User{
		Conn: c,
		UserInfo: &bridge.UserInfo{
			Host: "*",
		},
		channels: map[Channel]struct{}{},
		DecodeCh: make(chan *irc.Message),
	}
}

// NewUserNet creates a *User from a net.Conn connection.
func NewUserNet(c net.Conn) *User {
	return NewUser(&conn{
		Conn:    c,
		Encoder: irc.NewEncoder(c),
		Decoder: irc.NewDecoder(c),
	})
}

const defaultCloseMsg = "Closed."

type User struct {
	Conn

	sync.RWMutex
	*bridge.UserInfo

	BufferedMsg *irc.Message
	DecodeCh    chan *irc.Message

	channels map[Channel]struct{}

	v *viper.Viper

	UserBridge
}

func (u *User) ID() string {
	// return strings.ToLower(u.Nick)
	return strings.ToLower(u.User)
}

func (u *User) Prefix() *irc.Prefix {
	return &irc.Prefix{
		Name: u.Nick,
		User: u.User,
		Host: u.Host,
	}
}

func (u *User) Close() error {
	for ch := range u.channels {
		ch.Part(u, defaultCloseMsg)
	}
	return u.Conn.Close()
}

func (u *User) String() string {
	return u.Prefix().String()
}

func (u *User) NumChannels() int {
	u.RLock()
	defer u.RUnlock()
	return len(u.channels)
}

func (u *User) Channels() []Channel {
	u.RLock()
	channels := make([]Channel, 0, len(u.channels))
	for ch := range u.channels {
		channels = append(channels, ch)
	}
	u.RUnlock()
	return channels
}

func (u *User) VisibleTo() []*User {
	seen := map[*User]struct{}{}
	seen[u] = struct{}{}

	channels := u.Channels()
	num := 0
	for _, ch := range channels {
		// Don't include self
		num += ch.Len()
	}

	// Pre-allocate
	users := make([]*User, 0, num)
	if num == 0 {
		return users
	}

	// Get all unique users
	for _, ch := range channels {
		for _, other := range ch.Users() {
			if _, dupe := seen[other]; dupe {
				continue
			}
			seen[other] = struct{}{}
			// TODO: Check visibility (once it's implemented)
			users = append(users, other)
		}
	}
	return users
}

// Encode and send each msg until an error occurs, then returns.
func (u *User) Encode(msgs ...*irc.Message) (err error) {
	if u.Ghost {
		return nil
	}

	for _, msg := range msgs {
		if msg.Command == "PRIVMSG" && (msg.Prefix.Name == "slack" || msg.Prefix.Name == "mattermost") && msg.Prefix.Host == "service" && strings.Contains(msg.Trailing, "token") {
			logger.Debugf("-> %s %s %s", msg.Command, msg.Prefix.Name, "[token redacted]")

			err := u.Conn.Encode(msg)
			if err != nil {
				return err
			}

			continue
		}

		logger.Debugf("-> %s", msg)

		err := u.Conn.Encode(msg)
		if err != nil {
			return err
		}
	}

	return nil
}

// Decode will receive and return a decoded message, or an error.
// nolint:funlen,gocognit,gocyclo
func (u *User) Decode() {
	if u.Ghost {
		// block
		c := make(chan struct{})
		<-c
	}
	buffer := make(chan *irc.Message)
	stop := make(chan struct{})
	bufferTimeout := u.v.GetInt("PasteBufferTimeout")
	// we need at least 100
	if bufferTimeout < 100 {
		bufferTimeout = 100
	}
	logger.Debugf("using paste buffer timeout: %#v\n", bufferTimeout)
	t := timer.NewTimer(time.Duration(bufferTimeout) * time.Millisecond)
	t.Stop()
	go func(buffer chan *irc.Message, stop chan struct{}) {
		for {
			select {
			case msg := <-buffer:
				// are we starting a new buffer ?
				if u.BufferedMsg == nil {
					u.BufferedMsg = msg
					// start timer now
					t.Reset(time.Duration(bufferTimeout) * time.Millisecond)
				} else {
					if strings.HasPrefix(msg.Trailing, "\x01ACTION") {
						// flush buffer
						logger.Debug("flushing buffer because of /me")
						u.BufferedMsg.Trailing = strings.TrimSpace(u.BufferedMsg.Trailing)
						u.DecodeCh <- u.BufferedMsg
						u.BufferedMsg = nil
						// send CTCP message
						u.DecodeCh <- msg
						continue
					}
					// make sure we're sending to the same recipient in the buffer
					if u.BufferedMsg.Params[0] == msg.Params[0] {
						u.BufferedMsg.Trailing += "\n" + msg.Trailing
					} else {
						u.DecodeCh <- msg
					}
				}
			case <-t.C:
				if u.BufferedMsg != nil {
					// trim last newline
					u.BufferedMsg.Trailing = strings.TrimSpace(u.BufferedMsg.Trailing)
					logger.Debugf("flushing buffer: %#v\n", u.BufferedMsg)
					u.DecodeCh <- u.BufferedMsg
					// clear buffer
					u.BufferedMsg = nil
					t.Stop()
				}
			case <-stop:
				logger.Debug("closing decode()")
				return
			}
		}
	}(buffer, stop)
	for {
		msg, err := u.Conn.Decode()
		if err != nil {
			close(stop)
			if err.Error() != "EOF" {
				logger.Errorf("msg: %s err: %s", msg, err)
			}
			break
		}

		if msg == nil {
			continue
		}

		dmsg := fmt.Sprintf("<- %s", msg)
		if msg.Command == "PRIVMSG" && msg.Params != nil && (msg.Params[0] == "slack" || msg.Params[0] == "mattermost") {
			// Don't log sensitive information
			trail := strings.Split(msg.Trailing, " ")
			if (msg.Trailing != "" && trail[0] == "login") || (len(msg.Params) > 1 && msg.Params[1] == "login") {
				dmsg = fmt.Sprintf("<- PRIVMSG %s :login [redacted]", msg.Params[0])
			}
		}
		// PRIVMSG can be buffered
		if msg.Command == "PRIVMSG" {
			logger.Debugf("B: %#v\n", dmsg)
			buffer <- msg
		} else {
			logger.Debug(dmsg)
			u.DecodeCh <- msg
		}
	}
}

func (u *User) createService(nick string, what string) {
	u.CreateUserFromInfo(
		&bridge.UserInfo{
			Nick:  nick,
			User:  nick,
			Real:  what,
			Host:  "service",
			Ghost: true,
		})
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/stretchr/testify/assert"

	"testing"
)

func TestMissingLineConfig(t *testing.T) {
	var plugin Plugin

	err := plugin.Exec()

	assert.NotNil(t, err)
}

func TestWrongChannelID(t *testing.T) {
	var plugin Plugin

	plugin.Config.ChannelID = "test wrong id"
	plugin.Config.ChannelSecret = "test wrong id"
	plugin.Config.MID = "test wrong id"

	err := plugin.Exec()

	assert.NotNil(t, err)
}

func TestMissingUserConfig(t *testing.T) {
	plugin := Plugin{
		Config: Config{
			ChannelID:     "123456789",
			ChannelSecret: "test wrong id",
			MID:           "test wrong id",
		},
	}

	err := plugin.Exec()

	assert.NotNil(t, err)
}

func TestSendTextError(t *testing.T) {
	plugin := Plugin{
		Repo: Repo{
			Name:  "go-hello",
			Owner: "appleboy",
		},
		Build: Build{
			Number: 101,
			Status: "success",
			Link:   "https://github.com/appleboy/go-hello",
			Author: "Bo-Yi Wu",
			Branch: "master",
			Commit: "e7c4f0a63ceeb42a39ac7806f7b51f3f0d204fd2",
		},
		Config: Config{
			ChannelID:     "1465486347",
			ChannelSecret: "ChannelSecret",
			MID:           "MID",
			To:            []string{"1234567890"},
			Message:       []string{"Test"},
		},
	}

	// enable message
	err := plugin.Exec()
	assert.NotNil(t, err)

	// disable message
	plugin.Config.Message = []string{}
	err = plugin.Exec()
	assert.NotNil(t, err)
}

func TestDefaultMessage(t *testing.T) {
	plugin := Plugin{
		Repo: Repo{
			Name:  "go-hello",
			Owner: "appleboy",
		},
		Build: Build{
			Number: 101,
			Status: "success",
			Link:   "https://github.com/appleboy/go-hello",
			Author: "Bo-Yi Wu",
			Branch: "master",
			Commit: "e7c4f0a63ceeb42a39ac7806f7b51f3f0d204fd2",
		},
		Config: Config{
			ChannelID:     "1465486347",
			ChannelSecret: "ChannelSecret",
			MID:           "MID",
			To:            []string{"1234567890"},
			Message:       []string{"Test"},
		},
	}

	message := plugin.Message(plugin.Repo, plugin.Build)

	assert.Equal(t, []string{"[success] <https://github.com/appleboy/go-hello> (master) by Bo-Yi Wu"}, message)
}
<commit_msg>test error send notification.<commit_after>package main

import (
	"github.com/stretchr/testify/assert"

	"os"
	"testing"
)

func TestMissingLineConfig(t *testing.T) {
	var plugin Plugin

	err := plugin.Exec()

	assert.NotNil(t, err)
}

func TestWrongChannelID(t *testing.T) {
	var plugin Plugin

	plugin.Config.ChannelID = "test wrong id"
	plugin.Config.ChannelSecret = "test wrong id"
	plugin.Config.MID = "test wrong id"

	err := plugin.Exec()

	assert.NotNil(t, err)
}

func TestMissingUserConfig(t *testing.T) {
	plugin := Plugin{
		Config: Config{
			ChannelID:     "123456789",
			ChannelSecret: "test wrong id",
			MID:           "test wrong id",
		},
	}

	err := plugin.Exec()

	assert.NotNil(t, err)
}

func TestSendTextError(t *testing.T) {
	plugin := Plugin{
		Repo: Repo{
			Name:  "go-hello",
			Owner: "appleboy",
		},
		Build: Build{
			Number: 101,
			Status: "success",
			Link:   "https://github.com/appleboy/go-hello",
			Author: "Bo-Yi Wu",
			Branch: "master",
			Commit: "e7c4f0a63ceeb42a39ac7806f7b51f3f0d204fd2",
		},
		Config: Config{
			ChannelID:     "1465486347",
			ChannelSecret: "ChannelSecret",
			MID:           "MID",
			To:            []string{"1234567890"},
			Message:       []string{"Test"},
		},
	}

	// enable message
	err := plugin.Exec()
	assert.NotNil(t, err)

	// disable message
	plugin.Config.Message = []string{}
	err = plugin.Exec()
	assert.NotNil(t, err)
}

func TestDefaultMessageFormat(t *testing.T) {
	plugin := Plugin{
		Repo: Repo{
			Name:  "go-hello",
			Owner: "appleboy",
		},
		Build: Build{
			Number: 101,
			Status: "success",
			Link:   "https://github.com/appleboy/go-hello",
			Author: "Bo-Yi Wu",
			Branch: "master",
			Commit: "e7c4f0a63ceeb42a39ac7806f7b51f3f0d204fd2",
		},
	}

	message := plugin.Message(plugin.Repo, plugin.Build)

	assert.Equal(t, []string{"[success] <https://github.com/appleboy/go-hello> (master) by Bo-Yi Wu"}, message)
}

func TestErrorSendMessage(t *testing.T) {
	plugin := Plugin{
		Repo: Repo{
			Name:  "go-hello",
			Owner: "appleboy",
		},
		Build: Build{
			Number: 101,
			Status: "success",
			Link:   "https://github.com/appleboy/go-hello",
			Author: "Bo-Yi Wu",
			Branch: "master",
			Commit: "e7c4f0a63ceeb42a39ac7806f7b51f3f0d204fd2",
		},

		Config: Config{
			ChannelID:     os.Getenv("LINE_CHANNEL_ID"),
			ChannelSecret: os.Getenv("LINE_CHANNEL_SECRET"),
			MID:           os.Getenv("LINE_MID"),
			To:            []string{os.Getenv("LINE_TO")},
			Message:       []string{"Test Line Bot From Travis or Local"},
		},
	}

	err := plugin.Exec()
	// error message: Your ip address [xxx.xxx.xxx.xxx] is not allowed to access this API.
	// Please add your IP to the IP whitelist in the developer center.
	assert.NotNil(t, err)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package chartutil

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"k8s.io/helm/pkg/proto/hapi/chart"
)

const (
	// ChartfileName is the default Chart file name.
	ChartfileName = "Chart.yaml"
	// ValuesfileName is the default values file name.
	ValuesfileName = "values.yaml"
	// TemplatesDir is the relative directory name for templates.
	TemplatesDir = "templates"
	// ChartsDir is the relative directory name for charts dependencies.
	ChartsDir = "charts"
	// IgnorefileName is the name of the Helm ignore file.
	IgnorefileName = ".helmignore"
	// IngressFileName is the name of the example ingress file.
	IngressFileName = "ingress.yaml"
	// DeploymentName is the name of the example deployment file.
	DeploymentName = "deployment.yaml"
	// ServiceName is the name of the example service file.
	ServiceName = "service.yaml"
	// NotesName is the name of the example NOTES.txt file.
	NotesName = "NOTES.txt"
	// HelpersName is the name of the example NOTES.txt file.
	HelpersName = "_helpers.tpl"
)

const defaultValues = `# Default values for %s.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
replicaCount: 1
image:
  repository: nginx
  tag: stable
  pullPolicy: IfNotPresent
service:
  name: nginx
  type: ClusterIP
  externalPort: 80
  internalPort: 80
ingress:
  enabled: false
  # Used to create Ingress record (should used with service.type: ClusterIP).
  hosts:
    - chart-example.local
  annotations:
    # kubernetes.io/ingress.class: nginx
    # kubernetes.io/tls-acme: "true"
  tls:
    # Secrets must be manually created in the namespace.
    # - secretName: chart-example.local
    #   hosts:
    #     - chart-example.local
resources:
  limits:
    cpu: 100m
    memory: 128Mi
  requests:
    cpu: 100m
    memory: 128Mi

`

const defaultIgnore = `# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*~
# Various IDEs
.project
.idea/
*.tmproj
`

const defaultIngress = `{{- if .Values.ingress.enabled -}}
{{- $serviceName := include "fullname" . -}}
{{- $servicePort := .Values.service.externalPort -}}
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: {{ template "fullname" . }}
  labels:
    app: {{ template "fullname" . }}
    chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
    release: "{{ .Release.Name }}"
    heritage: "{{ .Release.Service }}"
  annotations:
    {{- range $key, $value := .Values.ingress.annotations }}
      {{ $key }}: {{ $value | quote }}
    {{- end }}
spec:
  rules:
    {{- range $host := .Values.ingress.hosts }}
    - host: {{ $host }}
      http:
        paths:
          - path: /
            backend:
              serviceName: {{ $serviceName }}
              servicePort: {{ $servicePort }}
    {{- end -}}
  {{- if .Values.ingress.tls }}
  tls:
{{ toYaml .Values.ingress.tls | indent 4 }}
  {{- end -}}
{{- end -}}
`

const defaultDeployment = `apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: {{ template "fullname" . }}
  labels:
    chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
spec:
  replicas: {{ .Values.replicaCount }}
  template:
    metadata:
      labels:
        app: {{ template "fullname" . }}
    spec:
      containers:
      - name: {{ .Chart.Name }}
        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
        imagePullPolicy: {{ .Values.image.pullPolicy }}
        ports:
        - containerPort: {{ .Values.service.internalPort }}
        livenessProbe:
          httpGet:
            path: /
            port: {{ .Values.service.internalPort }}
        readinessProbe:
          httpGet:
            path: /
            port: {{ .Values.service.internalPort }}
        resources:
{{ toYaml .Values.resources | indent 12 }}
`

const defaultService = `apiVersion: v1
kind: Service
metadata:
  name: {{ template "fullname" . }}
  labels:
    chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
spec:
  type: {{ .Values.service.type }}
  ports:
  - port: {{ .Values.service.externalPort }}
    targetPort: {{ .Values.service.internalPort }}
    protocol: TCP
    name: {{ .Values.service.name }}
  selector:
    app: {{ template "fullname" . }}
`

const defaultNotes = `1. Get the application URL by running these commands:
{{- if .Values.ingress.hostname }}
  http://{{- .Values.ingress.hostname }}
{{- else if contains "NodePort" .Values.service.type }}
  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "fullname" . }})
  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
  echo http://$NODE_IP:$NODE_PORT/login
{{- else if contains "LoadBalancer" .Values.service.type }}
     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
           You can watch the status of by running 'kubectl get svc -w {{ template "fullname" . }}'
  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
  echo http://$SERVICE_IP:{{ .Values.service.externalPort }}
{{- else if contains "ClusterIP"  .Values.service.type }}
  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app={{ template "fullname" . }}" -o jsonpath="{.items[0].metadata.name}")
  echo "Visit http://127.0.0.1:8080 to use your application"
  kubectl port-forward $POD_NAME 8080:{{ .Values.service.externalPort }}
{{- end }}
`

const defaultHelpers = `{{/* vim: set filetype=mustache: */}}
{{/*
Expand the name of the chart.
*/}}
{{- define "name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}

{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
*/}}
{{- define "fullname" -}}
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
`

// CreateFrom creates a new chart, but scaffolds it from the src chart.
func CreateFrom(chartfile *chart.Metadata, dest string, src string) error {
	schart, err := Load(src)
	if err != nil {
		return fmt.Errorf("could not load %s: %s", src, err)
	}

	schart.Metadata = chartfile
	return SaveDir(schart, dest)
}

// Create creates a new chart in a directory.
//
// Inside of dir, this will create a directory based on the name of
// chartfile.Name. It will then write the Chart.yaml into this directory and
// create the (empty) appropriate directories.
//
// The returned string will point to the newly created directory. It will be
// an absolute path, even if the provided base directory was relative.
//
// If dir does not exist, this will return an error.
// If Chart.yaml or any directories cannot be created, this will return an
// error. In such a case, this will attempt to clean up by removing the
// new chart directory.
func Create(chartfile *chart.Metadata, dir string) (string, error) {
	path, err := filepath.Abs(dir)
	if err != nil {
		return path, err
	}

	if fi, err := os.Stat(path); err != nil {
		return path, err
	} else if !fi.IsDir() {
		return path, fmt.Errorf("no such directory %s", path)
	}

	n := chartfile.Name
	cdir := filepath.Join(path, n)
	if fi, err := os.Stat(cdir); err == nil && !fi.IsDir() {
		return cdir, fmt.Errorf("file %s already exists and is not a directory", cdir)
	}
	if err := os.MkdirAll(cdir, 0755); err != nil {
		return cdir, err
	}

	cf := filepath.Join(cdir, ChartfileName)
	if _, err := os.Stat(cf); err != nil {
		if err := SaveChartfile(cf, chartfile); err != nil {
			return cdir, err
		}
	}

	for _, d := range []string{TemplatesDir, ChartsDir} {
		if err := os.MkdirAll(filepath.Join(cdir, d), 0755); err != nil {
			return cdir, err
		}
	}

	files := []struct {
		path    string
		content []byte
	}{
		{
			// values.yaml
			path:    filepath.Join(cdir, ValuesfileName),
			content: []byte(fmt.Sprintf(defaultValues, chartfile.Name)),
		},
		{
			// .helmignore
			path:    filepath.Join(cdir, IgnorefileName),
			content: []byte(defaultIgnore),
		},
		{
			// ingress.yaml
			path:    filepath.Join(cdir, TemplatesDir, IngressFileName),
			content: []byte(defaultIngress),
		},
		{
			// deployment.yaml
			path:    filepath.Join(cdir, TemplatesDir, DeploymentName),
			content: []byte(defaultDeployment),
		},
		{
			// service.yaml
			path:    filepath.Join(cdir, TemplatesDir, ServiceName),
			content: []byte(defaultService),
		},
		{
			// NOTES.txt
			path:    filepath.Join(cdir, TemplatesDir, NotesName),
			content: []byte(defaultNotes),
		},
		{
			// _helpers.tpl
			path:    filepath.Join(cdir, TemplatesDir, HelpersName),
			content: []byte(defaultHelpers),
		},
	}

	for _, file := range files {
		if _, err := os.Stat(file.path); err == nil {
			// File exists and is okay. Skip it.
			continue
		}
		if err := ioutil.WriteFile(file.path, file.content, 0644); err != nil {
			return cdir, err
		}
	}
	return cdir, nil
}
<commit_msg>Change secret name to something more standard<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package chartutil

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"k8s.io/helm/pkg/proto/hapi/chart"
)

const (
	// ChartfileName is the default Chart file name.
	ChartfileName = "Chart.yaml"
	// ValuesfileName is the default values file name.
	ValuesfileName = "values.yaml"
	// TemplatesDir is the relative directory name for templates.
	TemplatesDir = "templates"
	// ChartsDir is the relative directory name for charts dependencies.
	ChartsDir = "charts"
	// IgnorefileName is the name of the Helm ignore file.
	IgnorefileName = ".helmignore"
	// IngressFileName is the name of the example ingress file.
	IngressFileName = "ingress.yaml"
	// DeploymentName is the name of the example deployment file.
	DeploymentName = "deployment.yaml"
	// ServiceName is the name of the example service file.
	ServiceName = "service.yaml"
	// NotesName is the name of the example NOTES.txt file.
	NotesName = "NOTES.txt"
	// HelpersName is the name of the example NOTES.txt file.
	HelpersName = "_helpers.tpl"
)

const defaultValues = `# Default values for %s.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
replicaCount: 1
image:
  repository: nginx
  tag: stable
  pullPolicy: IfNotPresent
service:
  name: nginx
  type: ClusterIP
  externalPort: 80
  internalPort: 80
ingress:
  enabled: false
  # Used to create Ingress record (should used with service.type: ClusterIP).
  hosts:
    - chart-example.local
  annotations:
    # kubernetes.io/ingress.class: nginx
    # kubernetes.io/tls-acme: "true"
  tls:
    # Secrets must be manually created in the namespace.
    # - secretName: chart-example-tls
    #   hosts:
    #     - chart-example.local
resources:
  limits:
    cpu: 100m
    memory: 128Mi
  requests:
    cpu: 100m
    memory: 128Mi

`

const defaultIgnore = `# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*~
# Various IDEs
.project
.idea/
*.tmproj
`

const defaultIngress = `{{- if .Values.ingress.enabled -}}
{{- $serviceName := include "fullname" . -}}
{{- $servicePort := .Values.service.externalPort -}}
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: {{ template "fullname" . }}
  labels:
    app: {{ template "fullname" . }}
    chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
    release: "{{ .Release.Name }}"
    heritage: "{{ .Release.Service }}"
  annotations:
    {{- range $key, $value := .Values.ingress.annotations }}
      {{ $key }}: {{ $value | quote }}
    {{- end }}
spec:
  rules:
    {{- range $host := .Values.ingress.hosts }}
    - host: {{ $host }}
      http:
        paths:
          - path: /
            backend:
              serviceName: {{ $serviceName }}
              servicePort: {{ $servicePort }}
    {{- end -}}
  {{- if .Values.ingress.tls }}
  tls:
{{ toYaml .Values.ingress.tls | indent 4 }}
  {{- end -}}
{{- end -}}
`

const defaultDeployment = `apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: {{ template "fullname" . }}
  labels:
    chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
spec:
  replicas: {{ .Values.replicaCount }}
  template:
    metadata:
      labels:
        app: {{ template "fullname" . }}
    spec:
      containers:
      - name: {{ .Chart.Name }}
        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
        imagePullPolicy: {{ .Values.image.pullPolicy }}
        ports:
        - containerPort: {{ .Values.service.internalPort }}
        livenessProbe:
          httpGet:
            path: /
            port: {{ .Values.service.internalPort }}
        readinessProbe:
          httpGet:
            path: /
            port: {{ .Values.service.internalPort }}
        resources:
{{ toYaml .Values.resources | indent 12 }}
`

const defaultService = `apiVersion: v1
kind: Service
metadata:
  name: {{ template "fullname" . }}
  labels:
    chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
spec:
  type: {{ .Values.service.type }}
  ports:
  - port: {{ .Values.service.externalPort }}
    targetPort: {{ .Values.service.internalPort }}
    protocol: TCP
    name: {{ .Values.service.name }}
  selector:
    app: {{ template "fullname" . }}
`

const defaultNotes = `1. Get the application URL by running these commands:
{{- if .Values.ingress.hostname }}
  http://{{- .Values.ingress.hostname }}
{{- else if contains "NodePort" .Values.service.type }}
  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "fullname" . }})
  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
  echo http://$NODE_IP:$NODE_PORT/login
{{- else if contains "LoadBalancer" .Values.service.type }}
     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
           You can watch the status of by running 'kubectl get svc -w {{ template "fullname" . }}'
  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
  echo http://$SERVICE_IP:{{ .Values.service.externalPort }}
{{- else if contains "ClusterIP"  .Values.service.type }}
  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app={{ template "fullname" . }}" -o jsonpath="{.items[0].metadata.name}")
  echo "Visit http://127.0.0.1:8080 to use your application"
  kubectl port-forward $POD_NAME 8080:{{ .Values.service.externalPort }}
{{- end }}
`

const defaultHelpers = `{{/* vim: set filetype=mustache: */}}
{{/*
Expand the name of the chart.
*/}}
{{- define "name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}

{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
*/}}
{{- define "fullname" -}}
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
`

// CreateFrom creates a new chart, but scaffolds it from the src chart.
func CreateFrom(chartfile *chart.Metadata, dest string, src string) error {
	schart, err := Load(src)
	if err != nil {
		return fmt.Errorf("could not load %s: %s", src, err)
	}

	schart.Metadata = chartfile
	return SaveDir(schart, dest)
}

// Create creates a new chart in a directory.
//
// Inside of dir, this will create a directory based on the name of
// chartfile.Name. It will then write the Chart.yaml into this directory and
// create the (empty) appropriate directories.
//
// The returned string will point to the newly created directory. It will be
// an absolute path, even if the provided base directory was relative.
//
// If dir does not exist, this will return an error.
// If Chart.yaml or any directories cannot be created, this will return an
// error. In such a case, this will attempt to clean up by removing the
// new chart directory.
func Create(chartfile *chart.Metadata, dir string) (string, error) {
	path, err := filepath.Abs(dir)
	if err != nil {
		return path, err
	}

	if fi, err := os.Stat(path); err != nil {
		return path, err
	} else if !fi.IsDir() {
		return path, fmt.Errorf("no such directory %s", path)
	}

	n := chartfile.Name
	cdir := filepath.Join(path, n)
	if fi, err := os.Stat(cdir); err == nil && !fi.IsDir() {
		return cdir, fmt.Errorf("file %s already exists and is not a directory", cdir)
	}
	if err := os.MkdirAll(cdir, 0755); err != nil {
		return cdir, err
	}

	cf := filepath.Join(cdir, ChartfileName)
	if _, err := os.Stat(cf); err != nil {
		if err := SaveChartfile(cf, chartfile); err != nil {
			return cdir, err
		}
	}

	for _, d := range []string{TemplatesDir, ChartsDir} {
		if err := os.MkdirAll(filepath.Join(cdir, d), 0755); err != nil {
			return cdir, err
		}
	}

	files := []struct {
		path    string
		content []byte
	}{
		{
			// values.yaml
			path:    filepath.Join(cdir, ValuesfileName),
			content: []byte(fmt.Sprintf(defaultValues, chartfile.Name)),
		},
		{
			// .helmignore
			path:    filepath.Join(cdir, IgnorefileName),
			content: []byte(defaultIgnore),
		},
		{
			// ingress.yaml
			path:    filepath.Join(cdir, TemplatesDir, IngressFileName),
			content: []byte(defaultIngress),
		},
		{
			// deployment.yaml
			path:    filepath.Join(cdir, TemplatesDir, DeploymentName),
			content: []byte(defaultDeployment),
		},
		{
			// service.yaml
			path:    filepath.Join(cdir, TemplatesDir, ServiceName),
			content: []byte(defaultService),
		},
		{
			// NOTES.txt
			path:    filepath.Join(cdir, TemplatesDir, NotesName),
			content: []byte(defaultNotes),
		},
		{
			// _helpers.tpl
			path:    filepath.Join(cdir, TemplatesDir, HelpersName),
			content: []byte(defaultHelpers),
		},
	}

	for _, file := range files {
		if _, err := os.Stat(file.path); err == nil {
			// File exists and is okay. Skip it.
			continue
		}
		if err := ioutil.WriteFile(file.path, file.content, 0644); err != nil {
			return cdir, err
		}
	}
	return cdir, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package server

import (
	"fmt"
	"io"
	"io/ioutil"

	"github.com/spf13/cobra"
	"github.com/spf13/pflag"

	"k8s.io/apimachinery/pkg/util/sets"
	genericapiserver "k8s.io/apiserver/pkg/server"
	"k8s.io/apiserver/pkg/server/filters"
	genericoptions "k8s.io/apiserver/pkg/server/options"
	kubeclientset "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/kube-aggregator/pkg/apis/apiregistration/v1alpha1"
	"k8s.io/kube-aggregator/pkg/apiserver"
)

const defaultEtcdPathPrefix = "/registry/kube-aggregator.kubernetes.io/"

type AggregatorOptions struct {
	RecommendedOptions *genericoptions.RecommendedOptions

	// ProxyClientCert/Key are the client cert used to identify this proxy. Backing APIServices use
	// this to confirm the proxy's identity
	ProxyClientCertFile string
	ProxyClientKeyFile  string

	// CoreAPIKubeconfig is a filename for a kubeconfig file to contact the core API server wtih
	// If it is not set, the in cluster config is used
	CoreAPIKubeconfig string

	StdOut io.Writer
	StdErr io.Writer
}

// NewCommandStartAggregator provides a CLI handler for 'start master' command
func NewCommandStartAggregator(out, err io.Writer, stopCh <-chan struct{}) *cobra.Command {
	o := NewDefaultOptions(out, err)

	cmd := &cobra.Command{
		Short: "Launch a API aggregator and proxy server",
		Long:  "Launch a API aggregator and proxy server",
		RunE: func(c *cobra.Command, args []string) error {
			if err := o.Complete(); err != nil {
				return err
			}
			if err := o.Validate(args); err != nil {
				return err
			}
			if err := o.RunAggregator(stopCh); err != nil {
				return err
			}
			return nil
		},
	}

	o.AddFlags(cmd.Flags())
	return cmd
}

// AddFlags is necessary because hyperkube doesn't work using cobra, so we have to have different registration and execution paths
func (o *AggregatorOptions) AddFlags(fs *pflag.FlagSet) {
	o.RecommendedOptions.AddFlags(fs)
	fs.StringVar(&o.ProxyClientCertFile, "proxy-client-cert-file", o.ProxyClientCertFile, "client certificate used identify the proxy to the API server")
	fs.StringVar(&o.ProxyClientKeyFile, "proxy-client-key-file", o.ProxyClientKeyFile, "client certificate key used identify the proxy to the API server")
	fs.StringVar(&o.CoreAPIKubeconfig, "core-kubeconfig", o.CoreAPIKubeconfig, ""+
		"kubeconfig file pointing at the 'core' kubernetes server with enough rights to get,list,watch "+
		" services,endpoints.  If not set, the in-cluster config is used")
}

// NewDefaultOptions builds a "normal" set of options.  You wouldn't normally expose this, but hyperkube isn't cobra compatible
func NewDefaultOptions(out, err io.Writer) *AggregatorOptions {
	o := &AggregatorOptions{
		RecommendedOptions: genericoptions.NewRecommendedOptions(defaultEtcdPathPrefix, apiserver.Scheme, apiserver.Codecs.LegacyCodec(v1alpha1.SchemeGroupVersion)),

		StdOut: out,
		StdErr: err,
	}
	o.RecommendedOptions.SecureServing.ServingOptions.BindPort = 443
	return o
}

func (o AggregatorOptions) Validate(args []string) error {
	return nil
}

func (o *AggregatorOptions) Complete() error {
	return nil
}

func (o AggregatorOptions) RunAggregator(stopCh <-chan struct{}) error {
	// TODO have a "real" external address
	if err := o.RecommendedOptions.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost"); err != nil {
		return fmt.Errorf("error creating self-signed certificates: %v", err)
	}

	serverConfig := genericapiserver.NewConfig().
		WithSerializer(apiserver.Codecs)

	if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil {
		return err
	}
	serverConfig.LongRunningFunc = filters.BasicLongRunningRequestCheck(
		sets.NewString("watch", "proxy"),
		sets.NewString("attach", "exec", "proxy", "log", "portforward"),
	)

	var kubeconfig *rest.Config
	var err error
	if len(o.CoreAPIKubeconfig) > 0 {
		loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: o.CoreAPIKubeconfig}
		loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{})

		kubeconfig, err = loader.ClientConfig()

	} else {
		kubeconfig, err = rest.InClusterConfig()
	}
	if err != nil {
		return err
	}

	coreAPIServerClient, err := kubeclientset.NewForConfig(kubeconfig)
	if err != nil {
		return err
	}

	config := apiserver.Config{
		GenericConfig:       serverConfig,
		CoreAPIServerClient: coreAPIServerClient,
	}

	config.ProxyClientCert, err = ioutil.ReadFile(o.ProxyClientCertFile)
	if err != nil {
		return err
	}
	config.ProxyClientKey, err = ioutil.ReadFile(o.ProxyClientKeyFile)
	if err != nil {
		return err
	}

	server, err := config.Complete().New(stopCh)
	if err != nil {
		return err
	}
	return server.GenericAPIServer.PrepareRun().Run(stopCh)
}
<commit_msg>force callers to specify the cert dns names<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package server

import (
	"fmt"
	"io"
	"io/ioutil"

	"github.com/spf13/cobra"
	"github.com/spf13/pflag"

	"k8s.io/apimachinery/pkg/util/sets"
	genericapiserver "k8s.io/apiserver/pkg/server"
	"k8s.io/apiserver/pkg/server/filters"
	genericoptions "k8s.io/apiserver/pkg/server/options"
	kubeclientset "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/kube-aggregator/pkg/apis/apiregistration/v1alpha1"
	"k8s.io/kube-aggregator/pkg/apiserver"
)

const defaultEtcdPathPrefix = "/registry/kube-aggregator.kubernetes.io/"

type AggregatorOptions struct {
	RecommendedOptions *genericoptions.RecommendedOptions

	// ProxyClientCert/Key are the client cert used to identify this proxy. Backing APIServices use
	// this to confirm the proxy's identity
	ProxyClientCertFile string
	ProxyClientKeyFile  string

	// CoreAPIKubeconfig is a filename for a kubeconfig file to contact the core API server wtih
	// If it is not set, the in cluster config is used
	CoreAPIKubeconfig string

	StdOut io.Writer
	StdErr io.Writer
}

// NewCommandStartAggregator provides a CLI handler for 'start master' command
func NewCommandStartAggregator(out, err io.Writer, stopCh <-chan struct{}) *cobra.Command {
	o := NewDefaultOptions(out, err)

	cmd := &cobra.Command{
		Short: "Launch a API aggregator and proxy server",
		Long:  "Launch a API aggregator and proxy server",
		RunE: func(c *cobra.Command, args []string) error {
			if err := o.Complete(); err != nil {
				return err
			}
			if err := o.Validate(args); err != nil {
				return err
			}
			if err := o.RunAggregator(stopCh); err != nil {
				return err
			}
			return nil
		},
	}

	o.AddFlags(cmd.Flags())
	return cmd
}

// AddFlags is necessary because hyperkube doesn't work using cobra, so we have to have different registration and execution paths
func (o *AggregatorOptions) AddFlags(fs *pflag.FlagSet) {
	o.RecommendedOptions.AddFlags(fs)
	fs.StringVar(&o.ProxyClientCertFile, "proxy-client-cert-file", o.ProxyClientCertFile, "client certificate used identify the proxy to the API server")
	fs.StringVar(&o.ProxyClientKeyFile, "proxy-client-key-file", o.ProxyClientKeyFile, "client certificate key used identify the proxy to the API server")
	fs.StringVar(&o.CoreAPIKubeconfig, "core-kubeconfig", o.CoreAPIKubeconfig, ""+
		"kubeconfig file pointing at the 'core' kubernetes server with enough rights to get,list,watch "+
		" services,endpoints.  If not set, the in-cluster config is used")
}

// NewDefaultOptions builds a "normal" set of options.  You wouldn't normally expose this, but hyperkube isn't cobra compatible
func NewDefaultOptions(out, err io.Writer) *AggregatorOptions {
	o := &AggregatorOptions{
		RecommendedOptions: genericoptions.NewRecommendedOptions(defaultEtcdPathPrefix, apiserver.Scheme, apiserver.Codecs.LegacyCodec(v1alpha1.SchemeGroupVersion)),

		StdOut: out,
		StdErr: err,
	}
	o.RecommendedOptions.SecureServing.ServingOptions.BindPort = 443
	return o
}

func (o AggregatorOptions) Validate(args []string) error {
	return nil
}

func (o *AggregatorOptions) Complete() error {
	return nil
}

func (o AggregatorOptions) RunAggregator(stopCh <-chan struct{}) error {
	// TODO have a "real" external address
	if err := o.RecommendedOptions.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, nil); err != nil {
		return fmt.Errorf("error creating self-signed certificates: %v", err)
	}

	serverConfig := genericapiserver.NewConfig().
		WithSerializer(apiserver.Codecs)

	if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil {
		return err
	}
	serverConfig.LongRunningFunc = filters.BasicLongRunningRequestCheck(
		sets.NewString("watch", "proxy"),
		sets.NewString("attach", "exec", "proxy", "log", "portforward"),
	)

	var kubeconfig *rest.Config
	var err error
	if len(o.CoreAPIKubeconfig) > 0 {
		loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: o.CoreAPIKubeconfig}
		loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{})

		kubeconfig, err = loader.ClientConfig()

	} else {
		kubeconfig, err = rest.InClusterConfig()
	}
	if err != nil {
		return err
	}

	coreAPIServerClient, err := kubeclientset.NewForConfig(kubeconfig)
	if err != nil {
		return err
	}

	config := apiserver.Config{
		GenericConfig:       serverConfig,
		CoreAPIServerClient: coreAPIServerClient,
	}

	config.ProxyClientCert, err = ioutil.ReadFile(o.ProxyClientCertFile)
	if err != nil {
		return err
	}
	config.ProxyClientKey, err = ioutil.ReadFile(o.ProxyClientKeyFile)
	if err != nil {
		return err
	}

	server, err := config.Complete().New(stopCh)
	if err != nil {
		return err
	}
	return server.GenericAPIServer.PrepareRun().Run(stopCh)
}
<|endoftext|>