<commit_before>// Copyright © 2017 Karl Hepworth <Karl.Hepworth@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"
	"os"

	"strings"

	homedir "github.com/mitchellh/go-homedir"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

var (

	// add is a boolean which will indicate to install a composer application.
	add bool

	// alias is the destination drush alias this site should be using.
	// in many places this will default to the domain name if not specified.
	alias string

	// alias_template is the path to the drush alias template file
	// processed by this application. It can be blank/empty and
	// it will use the template file located parallel to this.cmd
	// this value is configurable through configuration management.
	alias_template string

	// aliases is a comma-separated list of aliases which translates into a
	// []string. each string in the slice represents a target in commands
	// which use multiple aliases. the usage of this variable often will
	// accompany the use of the pattern variable to match aliases against
	// patterns.
	aliases string

	// cfgFile is the path to the config file in use.
	// cfgFile will default to $HOME/drubuild/config.yml
	// Other formats are supported natively by Viper,
	// however in this case yaml is recommended.
	cfgFile string

	// commands is a comma-separated string which contains translates to a
	// []string of drush commands which are executed upon a list of aliases
	// provided with the aliases and pattern variables.
	commands string

	// composer represents the path to the composer file to be used.
	// it also represents a source file, in the event a composer.json file
	// does not exist at the destination path.
	// this flag will also supersede the necessity and the functionality
	// associated with legacy make files.
	composer string

	// db_host is the string which represents the configured database host
	// this host path can be configured at $HOME/drubuild/config.yml, and
	// defaults to '127.0.0.1'.
	db_host string

	// db_pass is an unprotected string which represents the configured user
	// password. this user account should have permission to create
	// databases, and this password can be configured at
	// $HOME/drubuild/config.yml, and defaults to 'root'.
	db_pass string

	// db_port is an integer which represents the configured database port
	// this port path can be configured at $HOME/drubuild/config.yml, and
	// defaults to 3306.
	db_port int

	// db_user is the string which represents the configured user account.
	// this user account should have permission to create databases, and
	// this user can be configured at $HOME/drubuild/config.yml, and defaults
	// to 'root'.
	db_user string

	// destination is the destination alias or path for the desired action.
	// it will be determined based upon the command in use.
	destination string

	// docroot is a string which indicates the nested file system destination.
	// it is not currently used, but is intended to open up flexibility in case
	// your composer file builds to a different root, such as 'web' or 'pub'.
	docroot = "docroot"

	// domain is the destination domain to be used when setting up a new site
	domain string

	// drupal will always be true, which indicates the site is a drupal website.
	// this ties into the build process when drush aliases are created.
	//
	// by setting this to false, it will not create a drush alias.
	drupal = true

	// makes is a comma-separated list of legacy make files to be used.
	// it will be automatically superseded by the use of the composer flag
	// however there is a lot of available deprecated functionality here.
	//
	// it is still appropriate to use this subset of functionality when
	// working with older codebases still accommodated to using make files.
	// these features may or may not be removed or isolated at a later time.
	//
	// Deprecated: use composer instead.
	makes string

	// name is the human-readable name for the target of this application.
	name string

	// noInstall marks the site build so that an installation will not trigger.
	// this is useful when rebuilding a codebase with no intention of using the
	// site, other than committing to source control.
	noInstall bool

	// pattern is a string which replaces the substring '%v' with another string
	// when dealing with operational work - most commonly aliases.
	pattern string

	// remove is a boolean which will indicate to remove a composer application.
	remove bool

	// when working with make files, you can tell the system to rewrite
	// a given module branch to change via a unique string inside the make
	// file(s). this represents the destination result of that change, what the
	// string is to be replaced to be in the generated make file.
	//
	// Deprecated: used exclusively by make file functionality.
	// upgrade to use composer instead.
	rewriteDestination string

	// when working with make files, you can tell the system to rewrite
	// a given module branch to change via a unique string inside the make
	// file(s). this represents the source of that change, what string is to be
	// replaced in the generated make file.
	//
	// Deprecated: used exclusively by make file functionality.
	// upgrade to use composer instead.
	rewriteSource string

	// sites_php_template is the path to a template to be used for sites.php
	// for the default multi-site installation which must accompany builds.
	// this is to match server-side consistency for multi-sites or non-default
	// file system naming conventions.
	sites_php_template string

	// source is the source alias or path to a source file to be used.
	// the specific action will be determined based on the command.
	source string

	// syncDatabase is a bool which represents the expressive action to
	// syncronise databases between a source and destination in the sync
	// command.
	syncDatabase bool

	// syncFiles is a bool which represents the expressive action to
	// syncronise public and private file systems between a source
	// and destination in the sync command.
	syncFiles bool

	// timestamp is an int64 which is representational of a date format in the
	// format of YYYYMMDDHHMMSS. An example of this is
	// 19700101000000.
	//
	// if the timestamp is not set, a timestamp will be generated.
	// this value will affect the output file-system paths used by drush
	// aliases and virtual hosts which are created or modified.
	//
	// these formats are used depending on this timestamp, and if composer
	// is being used, or drush make is used:
	//  - drush, timestamp
	//     /path/to/sites/mysite/mysite.timestamp/
	//  - drush, no timestamp
	//     /path/to/sites/mysite/mysite/
	//  - composer, timestamp
	//     /path/to/sites/mysite/mysite.timestamp/docroot
	//  - composer, no timestamp
	//     /path/to/sites/mysite/mysite/docroot
	//
	timestamp int64

	// user_block is a boolean which controls command action in the user command.
	// in this case, the user block action will be invoked.
	user_block bool

	// user_create is a boolean which controls command action in the user command.
	// in this case, the user create action will be invoked.
	user_create bool

	// user_delete is a boolean which controls command action in the user command.
	// in this case, the user delete action will be invoked.
	user_delete bool

	// user_email is a string which represents the drupal user's emails to be affected.
	user_email string

	// user_name is a string which represents the drupal user to be affected.
	user_name string

	// user_password is a string which represents the drupal user's password to be affected.
	user_password string

	// user_role is a string which represents the drupal user's role to be affected.
	user_role string

	// user_unblock is a boolean which controls command action in the user command.
	// in this case, the user unblock action will be invoked.
	user_unblock bool

	// user_verify is a boolean which controls command action in the user command.
	// in this case, the user verification action will be invoked.
	user_verify bool

	// version is a string which represents the version of a packages. it is only used
	// by the project command, to decouple the package name and version. it is optional
	// and can otherwise be placed explicitly in the name flag.
	version string

	// vhost is a bool which indicates that a virtual host should be created.
	// it will default to true, so if you're rebuilding a codebase or an existing
	// site, it would be logical to be able to skip that process.
	vhost = true

	// webserver is the name of the software package which handles HTTP
	// and HTTPS requests. this variable simply represents the name of
	// the service associated with web request handling.
	// setting this value is done through configuration management.
	webserver string

	// working-copy identifies if the build should leave the .git file-system
	// in-tact during the build. this would be useful when you are expecting
	// to send a file system to production, or for local development.
	// a working-copy is the local file-system including any development
	// file system data associated with each project/module.
	workingCopy bool

	// virtualhost_path is the path which the web server uses to store
	// all virtual hosts for the server. this is to identify where
	// processed templates should live and be removed from.
	// setting this value is done through configuration management.
	virtualhost_path string

	// path to the template to be used for virtual hosts supported by
	// the websrver variable. This assists to provide the webserver
	// a pre-configured set of defaults for a working site.
	virtualhost_template string
)

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:   "drubuild",
	Short: "A Drupal build system.",
	Long:  ``,
}

// Execute is the main function for the root command.
func Execute() {

	if cfgFile != "" {
		// Use config file from the flag.
		viper.SetConfigFile(cfgFile)
	} else {
		// Find home directory.
		home, err := homedir.Dir()
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}

		r := strings.Join([]string{home, "drubuild"}, string(os.PathSeparator))

		// Search config in home directory with name "drubuild" (without extension).
		viper.AddConfigPath(r)
		viper.SetConfigName("config")
		viper.SetConfigType("yml")
	}

	viper.AutomaticEnv() // read in environment variables that match

	// If a config file is found, read it in.
	if err := viper.ReadInConfig(); err == nil {
		fmt.Println("Using config file:", viper.ConfigFileUsed())
	}

	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<commit_msg>Improve documentation for the workingCopy global variable.<commit_after>// Copyright © 2017 Karl Hepworth <Karl.Hepworth@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"
	"os"

	"strings"

	homedir "github.com/mitchellh/go-homedir"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

var (

	// add is a boolean which will indicate to install a composer application.
	add bool

	// alias is the destination drush alias this site should be using.
	// in many places this will default to the domain name if not specified.
	alias string

	// alias_template is the path to the drush alias template file
	// processed by this application. It can be blank/empty and
	// it will use the template file located parallel to this.cmd
	// this value is configurable through configuration management.
	alias_template string

	// aliases is a comma-separated list of aliases which translates into a
	// []string. each string in the slice represents a target in commands
	// which use multiple aliases. the usage of this variable often will
	// accompany the use of the pattern variable to match aliases against
	// patterns.
	aliases string

	// cfgFile is the path to the config file in use.
	// cfgFile will default to $HOME/drubuild/config.yml
	// Other formats are supported natively by Viper,
	// however in this case yaml is recommended.
	cfgFile string

	// commands is a comma-separated string which contains translates to a
	// []string of drush commands which are executed upon a list of aliases
	// provided with the aliases and pattern variables.
	commands string

	// composer represents the path to the composer file to be used.
	// it also represents a source file, in the event a composer.json file
	// does not exist at the destination path.
	// this flag will also supersede the necessity and the functionality
	// associated with legacy make files.
	composer string

	// db_host is the string which represents the configured database host
	// this host path can be configured at $HOME/drubuild/config.yml, and
	// defaults to '127.0.0.1'.
	db_host string

	// db_pass is an unprotected string which represents the configured user
	// password. this user account should have permission to create
	// databases, and this password can be configured at
	// $HOME/drubuild/config.yml, and defaults to 'root'.
	db_pass string

	// db_port is an integer which represents the configured database port
	// this port path can be configured at $HOME/drubuild/config.yml, and
	// defaults to 3306.
	db_port int

	// db_user is the string which represents the configured user account.
	// this user account should have permission to create databases, and
	// this user can be configured at $HOME/drubuild/config.yml, and defaults
	// to 'root'.
	db_user string

	// destination is the destination alias or path for the desired action.
	// it will be determined based upon the command in use.
	destination string

	// docroot is a string which indicates the nested file system destination.
	// it is not currently used, but is intended to open up flexibility in case
	// your composer file builds to a different root, such as 'web' or 'pub'.
	docroot = "docroot"

	// domain is the destination domain to be used when setting up a new site
	domain string

	// drupal will always be true, which indicates the site is a drupal website.
	// this ties into the build process when drush aliases are created.
	//
	// by setting this to false, it will not create a drush alias.
	drupal = true

	// makes is a comma-separated list of legacy make files to be used.
	// it will be automatically superseded by the use of the composer flag
	// however there is a lot of available deprecated functionality here.
	//
	// it is still appropriate to use this subset of functionality when
	// working with older codebases still accommodated to using make files.
	// these features may or may not be removed or isolated at a later time.
	//
	// Deprecated: use composer instead.
	makes string

	// name is the human-readable name for the target of this application.
	name string

	// noInstall marks the site build so that an installation will not trigger.
	// this is useful when rebuilding a codebase with no intention of using the
	// site, other than committing to source control.
	noInstall bool

	// pattern is a string which replaces the substring '%v' with another string
	// when dealing with operational work - most commonly aliases.
	pattern string

	// remove is a boolean which will indicate to remove a composer application.
	remove bool

	// when working with make files, you can tell the system to rewrite
	// a given module branch to change via a unique string inside the make
	// file(s). this represents the destination result of that change, what the
	// string is to be replaced to be in the generated make file.
	//
	// Deprecated: used exclusively by make file functionality.
	// upgrade to use composer instead.
	rewriteDestination string

	// when working with make files, you can tell the system to rewrite
	// a given module branch to change via a unique string inside the make
	// file(s). this represents the source of that change, what string is to be
	// replaced in the generated make file.
	//
	// Deprecated: used exclusively by make file functionality.
	// upgrade to use composer instead.
	rewriteSource string

	// sites_php_template is the path to a template to be used for sites.php
	// for the default multi-site installation which must accompany builds.
	// this is to match server-side consistency for multi-sites or non-default
	// file system naming conventions.
	sites_php_template string

	// source is the source alias or path to a source file to be used.
	// the specific action will be determined based on the command.
	source string

	// syncDatabase is a bool which represents the expressive action to
	// syncronise databases between a source and destination in the sync
	// command.
	syncDatabase bool

	// syncFiles is a bool which represents the expressive action to
	// syncronise public and private file systems between a source
	// and destination in the sync command.
	syncFiles bool

	// timestamp is an int64 which is representational of a date format in the
	// format of YYYYMMDDHHMMSS. An example of this is
	// 19700101000000.
	//
	// if the timestamp is not set, a timestamp will be generated.
	// this value will affect the output file-system paths used by drush
	// aliases and virtual hosts which are created or modified.
	//
	// these formats are used depending on this timestamp, and if composer
	// is being used, or drush make is used:
	//  - drush, timestamp
	//     /path/to/sites/mysite/mysite.timestamp/
	//  - drush, no timestamp
	//     /path/to/sites/mysite/mysite/
	//  - composer, timestamp
	//     /path/to/sites/mysite/mysite.timestamp/docroot
	//  - composer, no timestamp
	//     /path/to/sites/mysite/mysite/docroot
	//
	timestamp int64

	// user_block is a boolean which controls command action in the user command.
	// in this case, the user block action will be invoked.
	user_block bool

	// user_create is a boolean which controls command action in the user command.
	// in this case, the user create action will be invoked.
	user_create bool

	// user_delete is a boolean which controls command action in the user command.
	// in this case, the user delete action will be invoked.
	user_delete bool

	// user_email is a string which represents the drupal user's emails to be affected.
	user_email string

	// user_name is a string which represents the drupal user to be affected.
	user_name string

	// user_password is a string which represents the drupal user's password to be affected.
	user_password string

	// user_role is a string which represents the drupal user's role to be affected.
	user_role string

	// user_unblock is a boolean which controls command action in the user command.
	// in this case, the user unblock action will be invoked.
	user_unblock bool

	// user_verify is a boolean which controls command action in the user command.
	// in this case, the user verification action will be invoked.
	user_verify bool

	// version is a string which represents the version of a packages. it is only used
	// by the project command, to decouple the package name and version. it is optional
	// and can otherwise be placed explicitly in the name flag.
	version string

	// vhost is a bool which indicates that a virtual host should be created.
	// it will default to true, so if you're rebuilding a codebase or an existing
	// site, it would be logical to be able to skip that process.
	vhost = true

	// webserver is the name of the software package which handles HTTP
	// and HTTPS requests. this variable simply represents the name of
	// the service associated with web request handling.
	// setting this value is done through configuration management.
	webserver string

	// working-copy identifies if the build should leave the .git file-system
	// in-tact during the build. this would be useful when you are expecting
	// to send a file system to production, or for local development.
	// a working-copy is the local file-system including any development
	// file system data associated with each project/module.
	//
	// although it will be treated as logic for --prefer-dist or
	// --prefer-source, it will also always git file systems when
	// it is false. the removal of .git folders will be done upon
	// completion of the build and project commands.
	workingCopy bool

	// virtualhost_path is the path which the web server uses to store
	// all virtual hosts for the server. this is to identify where
	// processed templates should live and be removed from.
	// setting this value is done through configuration management.
	virtualhost_path string

	// path to the template to be used for virtual hosts supported by
	// the websrver variable. This assists to provide the webserver
	// a pre-configured set of defaults for a working site.
	virtualhost_template string
)

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:   "drubuild",
	Short: "A Drupal build system.",
	Long:  ``,
}

// Execute is the main function for the root command.
func Execute() {

	if cfgFile != "" {
		// Use config file from the flag.
		viper.SetConfigFile(cfgFile)
	} else {
		// Find home directory.
		home, err := homedir.Dir()
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}

		r := strings.Join([]string{home, "drubuild"}, string(os.PathSeparator))

		// Search config in home directory with name "drubuild" (without extension).
		viper.AddConfigPath(r)
		viper.SetConfigName("config")
		viper.SetConfigType("yml")
	}

	viper.AutomaticEnv() // read in environment variables that match

	// If a config file is found, read it in.
	if err := viper.ReadInConfig(); err == nil {
		fmt.Println("Using config file:", viper.ConfigFileUsed())
	}

	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package cmd

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/Unknwon/com"
	"github.com/urfave/cli"
	log "gopkg.in/clog.v1"

	"github.com/gogits/gogs/models"
	"github.com/gogits/gogs/models/errors"
	"github.com/gogits/gogs/pkg/setting"
	http "github.com/gogits/gogs/routers/repo"
	"syscall"
)

const (
	_ACCESS_DENIED_MESSAGE = "Repository does not exist or you do not have access"
)

var Serv = cli.Command{
	Name:        "serv",
	Usage:       "This command should only be called by SSH shell",
	Description: `Serv provide access auth for repositories`,
	Action:      runServ,
	Flags: []cli.Flag{
		stringFlag("config, c", "custom/conf/app.ini", "Custom configuration file path"),
	},
}

func fail(userMessage, logMessage string, args ...interface{}) {
	fmt.Fprintln(os.Stderr, "Gogs:", userMessage)

	if len(logMessage) > 0 {
		if !setting.ProdMode {
			fmt.Fprintf(os.Stderr, logMessage+"\n", args...)
		}
		log.Fatal(3, logMessage, args...)
	}

	os.Exit(1)
}

func setup(c *cli.Context, logPath string, connectDB bool) {
	if c.IsSet("config") {
		setting.CustomConf = c.String("config")
	} else if c.GlobalIsSet("config") {
		setting.CustomConf = c.GlobalString("config")
	}

	setting.NewContext()

	level := log.TRACE
	if setting.ProdMode {
		level = log.ERROR
	}
	log.New(log.FILE, log.FileConfig{
		Level:    level,
		Filename: filepath.Join(setting.LogRootPath, logPath),
		FileRotationConfig: log.FileRotationConfig{
			Rotate:  true,
			Daily:   true,
			MaxDays: 3,
		},
	})
	log.Delete(log.CONSOLE) // Remove primary logger

	if !connectDB {
		return
	}

	models.LoadConfigs()

	if setting.UseSQLite3 {
		workDir, _ := setting.WorkDir()
		os.Chdir(workDir)
	}

	if err := models.SetEngine(); err != nil {
		fail("Internal error", "SetEngine: %v", err)
	}
}

func isAnnexShell(cmd string) bool {
	return cmd == "git-annex-shell"
}

func parseSSHCmd(cmd string) (string, string, []string) {
	ss := strings.Split(cmd, " ")
	if len(ss) < 2 {
		return "", "", nil
	}
	if isAnnexShell(ss[0]) {
		return ss[0], strings.Replace(ss[2], "/", "'", 1), ss
	} else {
		return ss[0], strings.Replace(ss[1], "/", "'", 1), ss
	}
}

func checkDeployKey(key *models.PublicKey, repo *models.Repository) {
	// Check if this deploy key belongs to current repository.
	if !models.HasDeployKey(key.ID, repo.ID) {
		fail("Key access denied", "Deploy key access denied: [key_id: %d, repo_id: %d]", key.ID, repo.ID)
	}

	// Update deploy key activity.
	deployKey, err := models.GetDeployKeyByRepo(key.ID, repo.ID)
	if err != nil {
		fail("Internal error", "GetDeployKey: %v", err)
	}

	deployKey.Updated = time.Now()
	if err = models.UpdateDeployKey(deployKey); err != nil {
		fail("Internal error", "UpdateDeployKey: %v", err)
	}
}

var (
	allowedCommands = map[string]models.AccessMode{
		"git-upload-pack":    models.ACCESS_MODE_READ,
		"git-upload-archive": models.ACCESS_MODE_READ,
		"git-receive-pack":   models.ACCESS_MODE_WRITE,
		"git-annex-shell":    models.ACCESS_MODE_WRITE,
	}
)

func runServ(c *cli.Context) error {
	setup(c, "serv.log", true)

	if setting.SSH.Disabled {
		println("Gogs: SSH has been disabled")
		return nil
	}

	if len(c.Args()) < 1 {
		fail("Not enough arguments", "Not enough arguments")
	}

	sshCmd := strings.Replace(os.Getenv("SSH_ORIGINAL_COMMAND"), "'", "", -1)
	log.Info("SSH commadn:%s", sshCmd)
	if len(sshCmd) == 0 {
		println("Hi there, You've successfully authenticated, but Gogs does not provide shell access.")
		println("If this is unexpected, please log in with password and setup Gogs under another user.")
		return nil
	}

	verb, path, args := parseSSHCmd(sshCmd)
	repoFullName := strings.ToLower(strings.Trim(path, "'"))
	repoFields := strings.SplitN(repoFullName, "/", 2)
	if len(repoFields) != 2 {
		fail("Invalid repository path", "Invalid repository path: %v", path)
	}
	ownerName := strings.ToLower(repoFields[0])
	repoName := strings.TrimSuffix(strings.ToLower(repoFields[1]), ".git")
	repoName = strings.TrimSuffix(repoName, ".wiki")

	owner, err := models.GetUserByName(ownerName)
	if err != nil {
		if errors.IsUserNotExist(err) {
			fail("Repository owner does not exist", "Unregistered owner: %s", ownerName)
		}
		fail("Internal error", "Fail to get repository owner '%s': %v", ownerName, err)
	}

	repo, err := models.GetRepositoryByName(owner.ID, repoName)
	if err != nil {
		if errors.IsRepoNotExist(err) {
			fail(_ACCESS_DENIED_MESSAGE, "Repository does not exist: %s/%s", owner.Name, repoName)
		}
		fail("Internal error", "Fail to get repository: %v", err)
	}
	repo.Owner = owner

	requestMode, ok := allowedCommands[verb]
	if !ok {
		fail("Unknown git command", "Unknown git command '%s'", verb)
	}

	// Prohibit push to mirror repositories.
	if requestMode > models.ACCESS_MODE_READ && repo.IsMirror {
		fail("Mirror repository is read-only", "")
	}

	// Allow anonymous (user is nil) clone for public repositories.
	var user *models.User

	key, err := models.GetPublicKeyByID(com.StrTo(strings.TrimPrefix(c.Args()[0], "key-")).MustInt64())
	if err != nil {
		fail("Invalid key ID", "Invalid key ID '%s': %v", c.Args()[0], err)
	}

	if requestMode == models.ACCESS_MODE_WRITE || repo.IsPrivate {
		// Check deploy key or user key.
		if key.IsDeployKey() {
			if key.Mode < requestMode {
				fail("Key permission denied", "Cannot push with deployment key: %d", key.ID)
			}
			checkDeployKey(key, repo)
		} else {
			user, err = models.GetUserByKeyID(key.ID)
			if err != nil {
				fail("Internal error", "Fail to get user by key ID '%d': %v", key.ID, err)
			}

			mode, err := models.AccessLevel(user.ID, repo)
			if err != nil {
				fail("Internal error", "Fail to check access: %v", err)
			}

			if mode < requestMode {
				clientMessage := _ACCESS_DENIED_MESSAGE
				if mode >= models.ACCESS_MODE_READ {
					clientMessage = "You do not have sufficient authorization for this action"
				}
				fail(clientMessage,
					"User '%s' does not have level '%v' access to repository '%s'",
					user.Name, requestMode, repoFullName)
			}
		}
	} else {
		setting.NewService()
		// Check if the key can access to the repository in case of it is a deploy key (a deploy keys != user key).
		// A deploy key doesn't represent a signed in user, so in a site with Service.RequireSignInView activated
		// we should give read access only in repositories where this deploy key is in use. In other case, a server
		// or system using an active deploy key can get read access to all the repositories in a Gogs service.
		if key.IsDeployKey() && setting.Service.RequireSignInView {
			checkDeployKey(key, repo)
		}
	}

	// Update user key activity.
	if key.ID > 0 {
		key, err := models.GetPublicKeyByID(key.ID)
		if err != nil {
			fail("Internal error", "GetPublicKeyByID: %v", err)
		}

		key.Updated = time.Now()
		if err = models.UpdatePublicKey(key); err != nil {
			fail("Internal error", "UpdatePublicKey: %v", err)
		}
	}

	// Special handle for Windows.
	// Todo will break with annex
	if setting.IsWindows {
		verb = strings.Replace(verb, "-", " ", 1)
	}
	verbs := strings.Split(verb, " ")
	var cmd []string
	if len(verbs) == 2 {
		cmd = []string{verbs[0], verbs[1], repoFullName}
	} else if isAnnexShell(verb) {
		repoAbsPath := setting.RepoRootPath + "/" + repoFullName
		//		if err := secureGitAnnex(repoAbsPath, requestMode); err != nil {
		//			fail("Git annex failed", "Git annex failed: %s", err)
		//}
		cmd = args
		// Setting full path to repo as git-annex-shell requires it
		cmd[2] = repoAbsPath
	} else {
		cmd = []string{verb, repoFullName}
	}
	runGit(cmd, requestMode, user, owner, repo)
	return nil

}

func runGit(cmd [] string, requestMode models.AccessMode, user *models.User, owner *models.User,
	repo *models.Repository) error {
	log.Info("will exectute:%s", cmd)
	gitCmd := exec.Command(cmd[0], cmd[1:]...)
	if requestMode == models.ACCESS_MODE_WRITE {
		gitCmd.Env = append(os.Environ(), http.ComposeHookEnvs(http.ComposeHookEnvsOptions{
			AuthUser:  user,
			OwnerName: owner.Name,
			OwnerSalt: owner.Salt,
			RepoID:    repo.ID,
			RepoName:  repo.Name,
			RepoPath:  repo.RepoPath(),
		})...)
	}
	gitCmd.Dir = setting.RepoRootPath
	gitCmd.Stdout = os.Stdout
	gitCmd.Stdin = os.Stdin
	gitCmd.Stderr = os.Stderr
	log.Info("args:%s", gitCmd.Args)
	err := gitCmd.Run()
	log.Info("err:%s", err)
	if t, ok := err.(*exec.ExitError); ok {
		log.Info("t:%s", t)
		os.Exit(t.Sys().(syscall.WaitStatus).ExitStatus())
	}

	return nil
}

// Make sure git-annex-shell does not make "bad" changes (refectored from repo)
func secureGitAnnex(path string, requestMode models.AccessMode) error {
	// "If set, disallows running git-shell to handle unknown commands."
	err := os.Setenv("GIT_ANNEX_SHELL_LIMITED", "True")
	if err != nil {
		return fmt.Errorf("ERROR: Could set annex shell to be limited.")
	}
	// "If set, git-annex-shell will refuse to run commands
	//  that do not operate on the specified directory."
	err = os.Setenv("GIT_ANNEX_SHELL_DIRECTORY", path)
	if err != nil {
		return fmt.Errorf("ERROR: Could set annex shell directory.")
	}
	if ! (requestMode > models.ACCESS_MODE_READ) {
		err = os.Setenv("GIT_ANNEX_SHELL_READONLY", "True")
		if err != nil {
			return fmt.Errorf("ERROR: Could set annex shell to read only.")
		}
	}
	return nil
}
<commit_msg>[annex] secure git-annex configuration<commit_after>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package cmd

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/Unknwon/com"
	"github.com/urfave/cli"
	log "gopkg.in/clog.v1"

	"github.com/gogits/gogs/models"
	"github.com/gogits/gogs/models/errors"
	"github.com/gogits/gogs/pkg/setting"
	http "github.com/gogits/gogs/routers/repo"
	"syscall"
)

const (
	_ACCESS_DENIED_MESSAGE = "Repository does not exist or you do not have access"
)

var Serv = cli.Command{
	Name:        "serv",
	Usage:       "This command should only be called by SSH shell",
	Description: `Serv provide access auth for repositories`,
	Action:      runServ,
	Flags: []cli.Flag{
		stringFlag("config, c", "custom/conf/app.ini", "Custom configuration file path"),
	},
}

func fail(userMessage, logMessage string, args ...interface{}) {
	fmt.Fprintln(os.Stderr, "Gogs:", userMessage)

	if len(logMessage) > 0 {
		if !setting.ProdMode {
			fmt.Fprintf(os.Stderr, logMessage+"\n", args...)
		}
		log.Fatal(3, logMessage, args...)
	}

	os.Exit(1)
}

func setup(c *cli.Context, logPath string, connectDB bool) {
	if c.IsSet("config") {
		setting.CustomConf = c.String("config")
	} else if c.GlobalIsSet("config") {
		setting.CustomConf = c.GlobalString("config")
	}

	setting.NewContext()

	level := log.TRACE
	if setting.ProdMode {
		level = log.ERROR
	}
	log.New(log.FILE, log.FileConfig{
		Level:    level,
		Filename: filepath.Join(setting.LogRootPath, logPath),
		FileRotationConfig: log.FileRotationConfig{
			Rotate:  true,
			Daily:   true,
			MaxDays: 3,
		},
	})
	log.Delete(log.CONSOLE) // Remove primary logger

	if !connectDB {
		return
	}

	models.LoadConfigs()

	if setting.UseSQLite3 {
		workDir, _ := setting.WorkDir()
		os.Chdir(workDir)
	}

	if err := models.SetEngine(); err != nil {
		fail("Internal error", "SetEngine: %v", err)
	}
}

func isAnnexShell(cmd string) bool {
	return cmd == "git-annex-shell"
}

func parseSSHCmd(cmd string) (string, string, []string) {
	ss := strings.Split(cmd, " ")
	if len(ss) < 2 {
		return "", "", nil
	}
	if isAnnexShell(ss[0]) {
		return ss[0], strings.Replace(ss[2], "/", "'", 1), ss
	} else {
		return ss[0], strings.Replace(ss[1], "/", "'", 1), ss
	}
}

func checkDeployKey(key *models.PublicKey, repo *models.Repository) {
	// Check if this deploy key belongs to current repository.
	if !models.HasDeployKey(key.ID, repo.ID) {
		fail("Key access denied", "Deploy key access denied: [key_id: %d, repo_id: %d]", key.ID, repo.ID)
	}

	// Update deploy key activity.
	deployKey, err := models.GetDeployKeyByRepo(key.ID, repo.ID)
	if err != nil {
		fail("Internal error", "GetDeployKey: %v", err)
	}

	deployKey.Updated = time.Now()
	if err = models.UpdateDeployKey(deployKey); err != nil {
		fail("Internal error", "UpdateDeployKey: %v", err)
	}
}

var (
	allowedCommands = map[string]models.AccessMode{
		"git-upload-pack":    models.ACCESS_MODE_READ,
		"git-upload-archive": models.ACCESS_MODE_READ,
		"git-receive-pack":   models.ACCESS_MODE_WRITE,
		"git-annex-shell":    models.ACCESS_MODE_WRITE,
	}
)

func runServ(c *cli.Context) error {
	setup(c, "serv.log", true)

	if setting.SSH.Disabled {
		println("Gogs: SSH has been disabled")
		return nil
	}

	if len(c.Args()) < 1 {
		fail("Not enough arguments", "Not enough arguments")
	}

	sshCmd := strings.Replace(os.Getenv("SSH_ORIGINAL_COMMAND"), "'", "", -1)
	log.Info("SSH commadn:%s", sshCmd)
	if len(sshCmd) == 0 {
		println("Hi there, You've successfully authenticated, but Gogs does not provide shell access.")
		println("If this is unexpected, please log in with password and setup Gogs under another user.")
		return nil
	}

	verb, path, args := parseSSHCmd(sshCmd)
	repoFullName := strings.ToLower(strings.Trim(path, "'"))
	repoFields := strings.SplitN(repoFullName, "/", 2)
	if len(repoFields) != 2 {
		fail("Invalid repository path", "Invalid repository path: %v", path)
	}
	ownerName := strings.ToLower(repoFields[0])
	repoName := strings.TrimSuffix(strings.ToLower(repoFields[1]), ".git")
	repoName = strings.TrimSuffix(repoName, ".wiki")

	owner, err := models.GetUserByName(ownerName)
	if err != nil {
		if errors.IsUserNotExist(err) {
			fail("Repository owner does not exist", "Unregistered owner: %s", ownerName)
		}
		fail("Internal error", "Fail to get repository owner '%s': %v", ownerName, err)
	}

	repo, err := models.GetRepositoryByName(owner.ID, repoName)
	if err != nil {
		if errors.IsRepoNotExist(err) {
			fail(_ACCESS_DENIED_MESSAGE, "Repository does not exist: %s/%s", owner.Name, repoName)
		}
		fail("Internal error", "Fail to get repository: %v", err)
	}
	repo.Owner = owner

	requestMode, ok := allowedCommands[verb]
	if !ok {
		fail("Unknown git command", "Unknown git command '%s'", verb)
	}

	// Prohibit push to mirror repositories.
	if requestMode > models.ACCESS_MODE_READ && repo.IsMirror {
		fail("Mirror repository is read-only", "")
	}

	// Allow anonymous (user is nil) clone for public repositories.
	var user *models.User

	key, err := models.GetPublicKeyByID(com.StrTo(strings.TrimPrefix(c.Args()[0], "key-")).MustInt64())
	if err != nil {
		fail("Invalid key ID", "Invalid key ID '%s': %v", c.Args()[0], err)
	}

	if requestMode == models.ACCESS_MODE_WRITE || repo.IsPrivate {
		// Check deploy key or user key.
		if key.IsDeployKey() {
			if key.Mode < requestMode {
				fail("Key permission denied", "Cannot push with deployment key: %d", key.ID)
			}
			checkDeployKey(key, repo)
		} else {
			user, err = models.GetUserByKeyID(key.ID)
			if err != nil {
				fail("Internal error", "Fail to get user by key ID '%d': %v", key.ID, err)
			}

			mode, err := models.AccessLevel(user.ID, repo)
			if err != nil {
				fail("Internal error", "Fail to check access: %v", err)
			}

			if mode < requestMode {
				clientMessage := _ACCESS_DENIED_MESSAGE
				if mode >= models.ACCESS_MODE_READ {
					clientMessage = "You do not have sufficient authorization for this action"
				}
				fail(clientMessage,
					"User '%s' does not have level '%v' access to repository '%s'",
					user.Name, requestMode, repoFullName)
			}
		}
	} else {
		setting.NewService()
		// Check if the key can access to the repository in case of it is a deploy key (a deploy keys != user key).
		// A deploy key doesn't represent a signed in user, so in a site with Service.RequireSignInView activated
		// we should give read access only in repositories where this deploy key is in use. In other case, a server
		// or system using an active deploy key can get read access to all the repositories in a Gogs service.
		if key.IsDeployKey() && setting.Service.RequireSignInView {
			checkDeployKey(key, repo)
		}
	}

	// Update user key activity.
	if key.ID > 0 {
		key, err := models.GetPublicKeyByID(key.ID)
		if err != nil {
			fail("Internal error", "GetPublicKeyByID: %v", err)
		}

		key.Updated = time.Now()
		if err = models.UpdatePublicKey(key); err != nil {
			fail("Internal error", "UpdatePublicKey: %v", err)
		}
	}

	// Special handle for Windows.
	// Todo will break with annex
	if setting.IsWindows {
		verb = strings.Replace(verb, "-", " ", 1)
	}
	verbs := strings.Split(verb, " ")
	var cmd []string
	if len(verbs) == 2 {
		cmd = []string{verbs[0], verbs[1], repoFullName}
	} else if isAnnexShell(verb) {
		repoAbsPath := setting.RepoRootPath + "/" + repoFullName
		if err := secureGitAnnex(repoAbsPath, requestMode); err != nil {
			fail("Git annex failed", "Git annex failed: %s", err)
		}
		cmd = args
		// Setting full path to repo as git-annex-shell requires it
		cmd[2] = repoAbsPath
	} else {
		cmd = []string{verb, repoFullName}
	}
	runGit(cmd, requestMode, user, owner, repo)
	return nil

}

func runGit(cmd [] string, requestMode models.AccessMode, user *models.User, owner *models.User,
	repo *models.Repository) error {
	log.Info("will exectute:%s", cmd)
	gitCmd := exec.Command(cmd[0], cmd[1:]...)
	if requestMode == models.ACCESS_MODE_WRITE {
		gitCmd.Env = append(os.Environ(), http.ComposeHookEnvs(http.ComposeHookEnvsOptions{
			AuthUser:  user,
			OwnerName: owner.Name,
			OwnerSalt: owner.Salt,
			RepoID:    repo.ID,
			RepoName:  repo.Name,
			RepoPath:  repo.RepoPath(),
		})...)
	}
	gitCmd.Dir = setting.RepoRootPath
	gitCmd.Stdout = os.Stdout
	gitCmd.Stdin = os.Stdin
	gitCmd.Stderr = os.Stderr
	log.Info("args:%s", gitCmd.Args)
	err := gitCmd.Run()
	log.Info("err:%s", err)
	if t, ok := err.(*exec.ExitError); ok {
		log.Info("t:%s", t)
		os.Exit(t.Sys().(syscall.WaitStatus).ExitStatus())
	}

	return nil
}

// Make sure git-annex-shell does not make "bad" changes (refectored from repo)
func secureGitAnnex(path string, requestMode models.AccessMode) error {
	// "If set, disallows running git-shell to handle unknown commands."
	err := os.Setenv("GIT_ANNEX_SHELL_LIMITED", "True")
	if err != nil {
		return fmt.Errorf("ERROR: Could set annex shell to be limited.")
	}
	// "If set, git-annex-shell will refuse to run commands
	//  that do not operate on the specified directory."
	err = os.Setenv("GIT_ANNEX_SHELL_DIRECTORY", path)
	if err != nil {
		return fmt.Errorf("ERROR: Could set annex shell directory.")
	}
	if ! (requestMode > models.ACCESS_MODE_READ) {
		err = os.Setenv("GIT_ANNEX_SHELL_READONLY", "True")
		if err != nil {
			return fmt.Errorf("ERROR: Could set annex shell to read only.")
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package regions

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/leancloud/lean-cli/utils"
)

// Region is region's type
type Region int

func Parse(region string) Region {
	switch region {
	case "cn", "CN", "cn-n1":
		return ChinaNorth
	case "tab", "TAB", "cn-e1":
		return ChinaEast
	case "us", "US", "us-w1":
		return USWest
	case "cn-tds1":
		return ChinaTDS1
	case "ap-sg":
		return APSG
	default:
		return Invalid
	}
}

var regionLoginStatus = make(map[Region]bool)

func init() {
	regionStatus, err := ioutil.ReadFile(filepath.Join(utils.ConfigDir(), "leancloud", "logined_regions.json"))
	if err != nil {
		if !os.IsNotExist(err) {
			panic(err)
		}
	} else {
		if err := json.Unmarshal(regionStatus, &regionLoginStatus); err != nil {
			panic(err)
		}
	}
}
func (r Region) String() string {
	switch r {
	case ChinaNorth:
		return "cn-n1"
	case ChinaEast:
		return "cn-e1"
	case USWest:
		return "us-w1"
	case ChinaTDS1:
		return "cn-tds1"
	case APSG:
		return "ap-sg"
	default:
		return "invalid"
	}
}

func (r Region) EnvString() string {
	switch r {
	case ChinaNorth, ChinaEast, ChinaTDS1:
		return "CN"
	case USWest:
		return "US"
	case APSG:
		return "AP"
	default:
		return "invalid"
	}
}

// Description is region's readable description
func (r Region) Description() string {
	switch r {
	case ChinaNorth:
		return "China North"
	case USWest:
		return "United States"
	case ChinaEast:
		return "China East"
	case ChinaTDS1:
		return "China TDS"
	case APSG:
		return "Singapore TDS"
	default:
		return "invalid"
	}
}

// API server regions
const (
	Invalid Region = iota
	ChinaNorth
	USWest
	ChinaEast
	ChinaTDS1
	APSG
)

// Only return available regions
func GetLoginedRegions(availableRegions []Region) []Region {
	var regions []Region
	for region, logged := range regionLoginStatus {
		if logged {
			for _, v := range availableRegions {
				if region == v {
					regions = append(regions, region)
				}
			}
		}
	}

	return regions
}

func GetRegionLoginStatus() map[Region]bool {
	return regionLoginStatus
}

func SetRegionLoginStatus(region Region) {
	regionLoginStatus[region] = true
}

func SaveRegionLoginStatus() error {
	data, err := json.MarshalIndent(regionLoginStatus, "", "  ")
	if err != nil {
		return err
	}

	return ioutil.WriteFile(filepath.Join(utils.ConfigDir(), "leancloud", "logined_regions.json"), data, 0644)
}
<commit_msg>make region wizard ordered<commit_after>package regions

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/leancloud/lean-cli/utils"
)

// Region is region's type
type Region int

func Parse(region string) Region {
	switch region {
	case "cn", "CN", "cn-n1":
		return ChinaNorth
	case "tab", "TAB", "cn-e1":
		return ChinaEast
	case "us", "US", "us-w1":
		return USWest
	case "cn-tds1":
		return ChinaTDS1
	case "ap-sg":
		return APSG
	default:
		return Invalid
	}
}

var regionLoginStatus = make(map[Region]bool)

func init() {
	regionStatus, err := ioutil.ReadFile(filepath.Join(utils.ConfigDir(), "leancloud", "logined_regions.json"))
	if err != nil {
		if !os.IsNotExist(err) {
			panic(err)
		}
	} else {
		if err := json.Unmarshal(regionStatus, &regionLoginStatus); err != nil {
			panic(err)
		}
	}
}
func (r Region) String() string {
	switch r {
	case ChinaNorth:
		return "cn-n1"
	case ChinaEast:
		return "cn-e1"
	case USWest:
		return "us-w1"
	case ChinaTDS1:
		return "cn-tds1"
	case APSG:
		return "ap-sg"
	default:
		return "invalid"
	}
}

func (r Region) EnvString() string {
	switch r {
	case ChinaNorth, ChinaEast, ChinaTDS1:
		return "CN"
	case USWest:
		return "US"
	case APSG:
		return "AP"
	default:
		return "invalid"
	}
}

// Description is region's readable description
func (r Region) Description() string {
	switch r {
	case ChinaNorth:
		return "China North"
	case USWest:
		return "United States"
	case ChinaEast:
		return "China East"
	case ChinaTDS1:
		return "China TDS"
	case APSG:
		return "Singapore TDS"
	default:
		return "invalid"
	}
}

// API server regions
const (
	Invalid Region = iota
	ChinaNorth
	USWest
	ChinaEast
	ChinaTDS1
	APSG
)

// Only return available regions
func GetLoginedRegions(availableRegions []Region) []Region {
	var regions []Region
	for _, region := range availableRegions {
		if regionLoginStatus[region] {
			regions = append(regions, region)
		}
	}

	return regions
}

func GetRegionLoginStatus() map[Region]bool {
	return regionLoginStatus
}

func SetRegionLoginStatus(region Region) {
	regionLoginStatus[region] = true
}

func SaveRegionLoginStatus() error {
	data, err := json.MarshalIndent(regionLoginStatus, "", "  ")
	if err != nil {
		return err
	}

	return ioutil.WriteFile(filepath.Join(utils.ConfigDir(), "leancloud", "logined_regions.json"), data, 0644)
}
<|endoftext|>


<commit_before>// Copyright 2019 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tracker

import (
	"context"
	"net"
	"os"
	"strconv"
	"sync"
	"time"

	opentracing "github.com/opentracing/opentracing-go"
	"github.com/pkg/errors"
	"github.com/tsuru/config"
	"github.com/tsuru/tsuru/api/shutdown"
	"github.com/tsuru/tsuru/log"
	"github.com/tsuru/tsuru/storage"
	trackerTypes "github.com/tsuru/tsuru/types/tracker"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
)

const (
	defaultUpdateInterval = 15 * time.Second
	defaultStaleTimeout   = 50 * time.Second
)

var (
	_ trackerTypes.InstanceService = &instanceTracker{}
	_ trackerTypes.InstanceService = &k8sInstanceTracker{}
)

func InstanceService() (trackerTypes.InstanceService, error) {
	useK8s, _ := config.GetBool("tracker:use-kubernetes")

	if useK8s {
		return newK8sInstanceTracker()
	}

	dbDriver, err := storage.GetCurrentDbDriver()
	if err != nil {
		dbDriver, err = storage.GetDefaultDbDriver()
		if err != nil {
			return nil, err
		}
	}
	tracker := &instanceTracker{
		storage: dbDriver.InstanceTrackerStorage,
		quit:    make(chan struct{}),
		done:    make(chan struct{}),
	}
	go tracker.start()
	shutdown.Register(tracker)
	return tracker, nil
}

type instanceTracker struct {
	storage      trackerTypes.InstanceStorage
	quit         chan struct{}
	done         chan struct{}
	mu           sync.Mutex
	lastInstance *trackerTypes.TrackedInstance
}

func (t *instanceTracker) start() {
	defer close(t.done)
	for {
		span, ctx := opentracing.StartSpanFromContext(context.Background(), "InstanceTracker notify")
		err := t.notify(ctx)
		if err != nil {
			log.Errorf("[instance-tracker] unable to track instance: %v", err)
		}
		span.Finish()

		var updateInterval time.Duration
		updateIntervalSeconds, _ := config.GetFloat("tracker:update-interval")
		if updateIntervalSeconds != 0 {
			updateInterval = time.Duration(updateIntervalSeconds * float64(time.Second))
		} else {
			updateInterval = defaultUpdateInterval
		}
		select {
		case <-t.quit:
			return
		case <-time.After(updateInterval):
		}
	}
}

func (t *instanceTracker) notify(ctx context.Context) error {
	instance, err := t.getInstance(true)
	if err != nil {
		return err
	}
	return t.storage.Notify(ctx, instance)
}

func (t *instanceTracker) getInstance(update bool) (trackerTypes.TrackedInstance, error) {
	t.mu.Lock()
	defer t.mu.Unlock()
	if update || t.lastInstance == nil {
		instance, err := createInstance()
		if err != nil {
			return instance, err
		}
		t.lastInstance = &instance
	}
	return *t.lastInstance, nil
}

func createInstance() (trackerTypes.TrackedInstance, error) {
	var instance trackerTypes.TrackedInstance
	iface, err := getInterface()
	if err != nil {
		return instance, err
	}
	ipv4Only, err := config.GetBool("tracker:ipv4-only")
	if err != nil {
		ipv4Only = true
	}
	var port, tlsPort string
	tlsListen, _ := config.GetString("tls:listen")
	if tlsListen != "" {
		_, tlsPort, err = net.SplitHostPort(tlsListen)
		if err != nil {
			return instance, err
		}
	}
	listen, _ := config.GetString("listen")
	if listen != "" {
		_, port, err = net.SplitHostPort(listen)
		if err != nil {
			return instance, err
		}
	}
	hostname, err := os.Hostname()
	if err != nil {
		return instance, err
	}
	addresses, err := iface.Addrs()
	if err != nil {
		return instance, err
	}
	ips := make([]string, 0, len(addresses))
	for _, ifaceAddr := range addresses {
		if !ipv4Only {
			ips = append(ips, ifaceAddr.String())
			continue
		}
		if ipNet, ok := ifaceAddr.(*net.IPNet); ok {
			ipv4 := ipNet.IP.To4()
			if ipv4 != nil {
				ips = append(ips, ipv4.String())
			}
		}
	}
	return trackerTypes.TrackedInstance{
		Name:       hostname,
		Port:       port,
		TLSPort:    tlsPort,
		Addresses:  ips,
		LastUpdate: time.Now().UTC().Truncate(time.Millisecond),
	}, nil
}

func (t *instanceTracker) Shutdown(ctx context.Context) error {
	close(t.quit)
	select {
	case <-t.done:
	case <-ctx.Done():
		return ctx.Err()
	}
	return nil
}

func (t *instanceTracker) CurrentInstance(ctx context.Context) (trackerTypes.TrackedInstance, error) {
	return t.getInstance(false)
}

func (t *instanceTracker) LiveInstances(ctx context.Context) ([]trackerTypes.TrackedInstance, error) {
	var staleTimeout time.Duration
	staleTimeoutSeconds, _ := config.GetFloat("tracker:stale-timeout")
	if staleTimeoutSeconds != 0 {
		staleTimeout = time.Duration(staleTimeoutSeconds * float64(time.Second))
	} else {
		staleTimeout = defaultStaleTimeout
	}
	return t.storage.List(ctx, staleTimeout)
}

func getInterface() (net.Interface, error) {
	interfaceName, _ := config.GetString("tracker:interface")
	var interfaceNames []string
	if interfaceName == "" {
		interfaceNames = []string{"eth0", "en0"}
	} else {
		interfaceNames = []string{interfaceName}
	}
	ifaces, err := net.Interfaces()
	if err != nil {
		return net.Interface{}, err
	}
	if len(ifaces) == 0 {
		return net.Interface{}, errors.New("no network interfaces available")
	}
	for _, wanted := range interfaceNames {
		for _, iface := range ifaces {
			if iface.Name == wanted {
				return iface, nil
			}
		}
	}
	if interfaceName != "" {
		return net.Interface{}, errors.Errorf("interface named %q not found", interfaceName)
	}
	return ifaces[0], nil
}

// instanceTracker based on kubernetes service discovery
type k8sInstanceTracker struct {
	ns              string
	service         string
	cli             kubernetes.Interface
	currentInstance trackerTypes.TrackedInstance
}

func (t *k8sInstanceTracker) CurrentInstance(ctx context.Context) (trackerTypes.TrackedInstance, error) {
	return t.currentInstance, nil
}

func (t *k8sInstanceTracker) LiveInstances(ctx context.Context) ([]trackerTypes.TrackedInstance, error) {
	endpoints, err := t.cli.CoreV1().Endpoints(t.ns).Get(ctx, t.service, metav1.GetOptions{})
	if err != nil {
		return nil, err
	}

	instances := []trackerTypes.TrackedInstance{}
	lastUpdate := time.Now()

	for _, subset := range endpoints.Subsets {
		httpPort := "80"
		httpsPort := "443"

		for _, port := range subset.Ports {
			if port.Name == "http" {
				httpPort = strconv.Itoa(int(port.Port))
			}

			if port.Name == "https" {
				httpsPort = strconv.Itoa(int(port.Port))
			}
		}

		for _, address := range subset.Addresses {
			if address.TargetRef == nil {
				continue
			}
			instances = append(instances, trackerTypes.TrackedInstance{
				Name: address.TargetRef.Name,
				Addresses: []string{
					address.IP,
				},
				Port:       httpPort,
				TLSPort:    httpsPort,
				LastUpdate: lastUpdate,
			})
		}
	}

	return instances, nil
}

func newK8sInstanceTracker() (*k8sInstanceTracker, error) {
	inClusterCfg, err := rest.InClusterConfig()
	if err != nil {
		return nil, err
	}

	cli, err := kubernetes.NewForConfig(inClusterCfg)
	if err != nil {
		return nil, err
	}

	ns, err := config.GetString("tracker:kubernetes-namespace")
	if err != nil {
		return nil, err
	}

	service, err := config.GetString("tracker:kubernetes-service")
	if err != nil {
		return nil, err
	}

	currentInstance, err := createInstance()
	if err != nil {
		return nil, err
	}

	return &k8sInstanceTracker{
		cli:             cli,
		ns:              ns,
		service:         service,
		currentInstance: currentInstance,
	}, nil
}
<commit_msg>tracker: use config when kubernetes service does not provide http port<commit_after>// Copyright 2019 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tracker

import (
	"context"
	"net"
	"os"
	"strconv"
	"sync"
	"time"

	opentracing "github.com/opentracing/opentracing-go"
	"github.com/pkg/errors"
	"github.com/tsuru/config"
	"github.com/tsuru/tsuru/api/shutdown"
	"github.com/tsuru/tsuru/log"
	"github.com/tsuru/tsuru/storage"
	trackerTypes "github.com/tsuru/tsuru/types/tracker"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
)

const (
	defaultUpdateInterval = 15 * time.Second
	defaultStaleTimeout   = 50 * time.Second
)

var (
	_ trackerTypes.InstanceService = &instanceTracker{}
	_ trackerTypes.InstanceService = &k8sInstanceTracker{}
)

func InstanceService() (trackerTypes.InstanceService, error) {
	useK8s, _ := config.GetBool("tracker:use-kubernetes")

	if useK8s {
		return newK8sInstanceTracker()
	}

	dbDriver, err := storage.GetCurrentDbDriver()
	if err != nil {
		dbDriver, err = storage.GetDefaultDbDriver()
		if err != nil {
			return nil, err
		}
	}
	tracker := &instanceTracker{
		storage: dbDriver.InstanceTrackerStorage,
		quit:    make(chan struct{}),
		done:    make(chan struct{}),
	}
	go tracker.start()
	shutdown.Register(tracker)
	return tracker, nil
}

type instanceTracker struct {
	storage      trackerTypes.InstanceStorage
	quit         chan struct{}
	done         chan struct{}
	mu           sync.Mutex
	lastInstance *trackerTypes.TrackedInstance
}

func (t *instanceTracker) start() {
	defer close(t.done)
	for {
		span, ctx := opentracing.StartSpanFromContext(context.Background(), "InstanceTracker notify")
		err := t.notify(ctx)
		if err != nil {
			log.Errorf("[instance-tracker] unable to track instance: %v", err)
		}
		span.Finish()

		var updateInterval time.Duration
		updateIntervalSeconds, _ := config.GetFloat("tracker:update-interval")
		if updateIntervalSeconds != 0 {
			updateInterval = time.Duration(updateIntervalSeconds * float64(time.Second))
		} else {
			updateInterval = defaultUpdateInterval
		}
		select {
		case <-t.quit:
			return
		case <-time.After(updateInterval):
		}
	}
}

func (t *instanceTracker) notify(ctx context.Context) error {
	instance, err := t.getInstance(true)
	if err != nil {
		return err
	}
	return t.storage.Notify(ctx, instance)
}

func (t *instanceTracker) getInstance(update bool) (trackerTypes.TrackedInstance, error) {
	t.mu.Lock()
	defer t.mu.Unlock()
	if update || t.lastInstance == nil {
		instance, err := createInstance()
		if err != nil {
			return instance, err
		}
		t.lastInstance = &instance
	}
	return *t.lastInstance, nil
}

func createInstance() (trackerTypes.TrackedInstance, error) {
	var instance trackerTypes.TrackedInstance
	iface, err := getInterface()
	if err != nil {
		return instance, err
	}
	ipv4Only, err := config.GetBool("tracker:ipv4-only")
	if err != nil {
		ipv4Only = true
	}
	port, tlsPort, err := listenPorts()
	if err != nil {
		return instance, err
	}

	hostname, err := os.Hostname()
	if err != nil {
		return instance, err
	}
	addresses, err := iface.Addrs()
	if err != nil {
		return instance, err
	}
	ips := make([]string, 0, len(addresses))
	for _, ifaceAddr := range addresses {
		if !ipv4Only {
			ips = append(ips, ifaceAddr.String())
			continue
		}
		if ipNet, ok := ifaceAddr.(*net.IPNet); ok {
			ipv4 := ipNet.IP.To4()
			if ipv4 != nil {
				ips = append(ips, ipv4.String())
			}
		}
	}
	return trackerTypes.TrackedInstance{
		Name:       hostname,
		Port:       port,
		TLSPort:    tlsPort,
		Addresses:  ips,
		LastUpdate: time.Now().UTC().Truncate(time.Millisecond),
	}, nil
}

func listenPorts() (port string, tlsPort string, err error) {
	tlsListen, _ := config.GetString("tls:listen")
	if tlsListen != "" {
		_, tlsPort, err = net.SplitHostPort(tlsListen)
		if err != nil {
			return "", "", err
		}
	}
	listen, _ := config.GetString("listen")
	if listen != "" {
		_, port, err = net.SplitHostPort(listen)
		if err != nil {
			return "", "", err
		}
	}

	return port, tlsPort, nil
}

func (t *instanceTracker) Shutdown(ctx context.Context) error {
	close(t.quit)
	select {
	case <-t.done:
	case <-ctx.Done():
		return ctx.Err()
	}
	return nil
}

func (t *instanceTracker) CurrentInstance(ctx context.Context) (trackerTypes.TrackedInstance, error) {
	return t.getInstance(false)
}

func (t *instanceTracker) LiveInstances(ctx context.Context) ([]trackerTypes.TrackedInstance, error) {
	var staleTimeout time.Duration
	staleTimeoutSeconds, _ := config.GetFloat("tracker:stale-timeout")
	if staleTimeoutSeconds != 0 {
		staleTimeout = time.Duration(staleTimeoutSeconds * float64(time.Second))
	} else {
		staleTimeout = defaultStaleTimeout
	}
	return t.storage.List(ctx, staleTimeout)
}

func getInterface() (net.Interface, error) {
	interfaceName, _ := config.GetString("tracker:interface")
	var interfaceNames []string
	if interfaceName == "" {
		interfaceNames = []string{"eth0", "en0"}
	} else {
		interfaceNames = []string{interfaceName}
	}
	ifaces, err := net.Interfaces()
	if err != nil {
		return net.Interface{}, err
	}
	if len(ifaces) == 0 {
		return net.Interface{}, errors.New("no network interfaces available")
	}
	for _, wanted := range interfaceNames {
		for _, iface := range ifaces {
			if iface.Name == wanted {
				return iface, nil
			}
		}
	}
	if interfaceName != "" {
		return net.Interface{}, errors.Errorf("interface named %q not found", interfaceName)
	}
	return ifaces[0], nil
}

// instanceTracker based on kubernetes service discovery
type k8sInstanceTracker struct {
	ns              string
	service         string
	cli             kubernetes.Interface
	currentInstance trackerTypes.TrackedInstance
}

func (t *k8sInstanceTracker) CurrentInstance(ctx context.Context) (trackerTypes.TrackedInstance, error) {
	return t.currentInstance, nil
}

func (t *k8sInstanceTracker) LiveInstances(ctx context.Context) ([]trackerTypes.TrackedInstance, error) {
	endpoints, err := t.cli.CoreV1().Endpoints(t.ns).Get(ctx, t.service, metav1.GetOptions{})
	if err != nil {
		return nil, err
	}

	instances := []trackerTypes.TrackedInstance{}
	lastUpdate := time.Now()

	defaultHTTPPort, defaultHTTPSPort, err := listenPorts()
	if err != nil {
		return nil, err
	}
	if defaultHTTPPort == "" {
		defaultHTTPPort = "80"
	}
	if defaultHTTPSPort == "" {
		defaultHTTPSPort = "443"
	}

	for _, subset := range endpoints.Subsets {
		httpPort := defaultHTTPPort
		httpsPort := defaultHTTPSPort

		for _, port := range subset.Ports {
			if port.Name == "http" {
				httpPort = strconv.Itoa(int(port.Port))
			}

			if port.Name == "https" {
				httpsPort = strconv.Itoa(int(port.Port))
			}
		}

		for _, address := range subset.Addresses {
			if address.TargetRef == nil {
				continue
			}
			instances = append(instances, trackerTypes.TrackedInstance{
				Name: address.TargetRef.Name,
				Addresses: []string{
					address.IP,
				},
				Port:       httpPort,
				TLSPort:    httpsPort,
				LastUpdate: lastUpdate,
			})
		}
	}

	return instances, nil
}

func newK8sInstanceTracker() (*k8sInstanceTracker, error) {
	inClusterCfg, err := rest.InClusterConfig()
	if err != nil {
		return nil, err
	}

	cli, err := kubernetes.NewForConfig(inClusterCfg)
	if err != nil {
		return nil, err
	}

	ns, err := config.GetString("tracker:kubernetes-namespace")
	if err != nil {
		return nil, err
	}

	service, err := config.GetString("tracker:kubernetes-service")
	if err != nil {
		return nil, err
	}

	currentInstance, err := createInstance()
	if err != nil {
		return nil, err
	}

	return &k8sInstanceTracker{
		cli:             cli,
		ns:              ns,
		service:         service,
		currentInstance: currentInstance,
	}, nil
}
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.521"
<commit_msg>fnserver: 0.3.522 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.522"
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.193"
<commit_msg>fn-server: 0.3.194 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.194"
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.248"
<commit_msg>fnserver: 0.3.249 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.249"
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.666"
<commit_msg>fnserver: 0.3.667 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.667"
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.218"
<commit_msg>fnserver: 0.3.219 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.219"
<|endoftext|>


<commit_before>package cloudwatch

import (
	"fmt"
	"sort"
	"time"

	"github.com/lucagrulla/cw/timeutil"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
	"github.com/fatih/color"
)

func cwClient() *cloudwatchlogs.CloudWatchLogs {
	sess := session.Must(session.NewSessionWithOptions(session.Options{
		SharedConfigState: session.SharedConfigEnable,
	}))
	return cloudwatchlogs.New(sess)
}

func params(logGroupName string, streamNames []*string, epochStartTime int64, epochEndTime int64, grep *string, follow *bool) *cloudwatchlogs.FilterLogEventsInput {
	startTimeInt64 := epochStartTime * 1000
	endTimeInt64 := epochEndTime * 1000
	params := &cloudwatchlogs.FilterLogEventsInput{
		LogGroupName: &logGroupName,
		Interleaved:  aws.Bool(true),
		StartTime:    &startTimeInt64}

	if *grep != "" {
		params.FilterPattern = grep
	}

	if streamNames != nil {
		params.LogStreamNames = streamNames
	}

	if !*follow && endTimeInt64 != 0 {
		params.EndTime = &endTimeInt64
	}
	return params
}

func Tail(logGroupName *string, logStreamName *string, follow *bool, startTime *time.Time, endTime *time.Time, grep *string, printTimestamp *bool, printStreamName *bool) <-chan *string {
	cwl := cwClient()

	startTimeEpoch := timeutil.ParseTime(startTime.Format(timeutil.TimeFormat)).Unix()
	lastSeenTimestamp := startTimeEpoch

	var endTimeEpoch int64
	if !endTime.IsZero() {
		endTimeEpoch = timeutil.ParseTime(endTime.Format(timeutil.TimeFormat)).Unix()
	}

	var ids []string

	ch := make(chan *string)
	pageHandler := func(res *cloudwatchlogs.FilterLogEventsOutput, lastPage bool) bool {
		if len(res.Events) == 0 {
			time.Sleep(2 * time.Second)
		} else {
			for _, event := range res.Events {
				eventTimestamp := *event.Timestamp / 1000
				if eventTimestamp != lastSeenTimestamp {
					ids = nil
					lastSeenTimestamp = eventTimestamp
				} else {
					sort.Strings(ids)
				}
				idx := sort.SearchStrings(ids, *event.EventId)
				if ids == nil || (idx == len(ids) || ids[idx] != *event.EventId) {
					d := timeutil.FormatTimestamp(eventTimestamp)
					var msg string
					if *printTimestamp {
						msg = fmt.Sprintf("%s - ", color.GreenString(d))
					}
					if *printStreamName {
						msg = fmt.Sprintf("%s%s - ", msg, color.BlueString(*event.LogStreamName))
					}
					msg = fmt.Sprintf("%s%s", msg, *event.Message)
					ch <- &msg

				}
				ids = append(ids, *event.EventId)
			}
		}
		if lastPage && !*follow {
			time.Sleep(1 * time.Second)
			close(ch)
		}
		return !lastPage
	}
	var streams []*string
	if *logStreamName != "*" {
		for stream := range LsStreams(logGroupName, logStreamName) {
			streams = append(streams, stream)
		}
		if len(streams) == 0 {
			panic("No such log stream.")
		}
	}
	go func() {
		for *follow || lastSeenTimestamp == startTimeEpoch {
			logParam := params(*logGroupName, streams, lastSeenTimestamp, endTimeEpoch, grep, follow)
			error := cwl.FilterLogEventsPages(logParam, pageHandler)
			if error != nil {
				panic(error)
			}
		}
	}()
	return ch
}

func LsGroups() <-chan *string {
	cwl := cwClient()
	ch := make(chan *string)
	params := &cloudwatchlogs.DescribeLogGroupsInput{
	//		LogGroupNamePrefix: aws.String("LogGroupName"),
	}

	handler := func(res *cloudwatchlogs.DescribeLogGroupsOutput, lastPage bool) bool {
		for _, logGroup := range res.LogGroups {
			ch <- logGroup.LogGroupName
		}
		if lastPage {
			close(ch)
		}
		return !lastPage
	}
	go func() {
		err := cwl.DescribeLogGroupsPages(params, handler)
		if err != nil {
			panic(err)
		}
	}()
	return ch
}

func LsStreams(groupName *string, streamName *string) <-chan *string {
	cwl := cwClient()
	ch := make(chan *string)

	params := &cloudwatchlogs.DescribeLogStreamsInput{
		LogGroupName: groupName}
	if streamName != nil {
		params.LogStreamNamePrefix = streamName
	}
	handler := func(res *cloudwatchlogs.DescribeLogStreamsOutput, lastPage bool) bool {
		for _, logStream := range res.LogStreams {
			ch <- logStream.LogStreamName
		}
		if lastPage {
			close(ch)
		}
		return !lastPage
	}

	go func() {
		err := cwl.DescribeLogStreamsPages(params, handler)
		if err != nil {
			panic(err)
		}
	}()
	return ch
}
<commit_msg>Better error handling<commit_after>package cloudwatch

import (
	"fmt"
	"os"
	"sort"
	"time"

	"github.com/lucagrulla/cw/timeutil"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
	"github.com/fatih/color"
)

func cwClient() *cloudwatchlogs.CloudWatchLogs {
	sess := session.Must(session.NewSessionWithOptions(session.Options{
		SharedConfigState: session.SharedConfigEnable,
	}))
	return cloudwatchlogs.New(sess)
}

func params(logGroupName string, streamNames []*string, epochStartTime int64, epochEndTime int64, grep *string, follow *bool) *cloudwatchlogs.FilterLogEventsInput {
	startTimeInt64 := epochStartTime * 1000
	endTimeInt64 := epochEndTime * 1000
	params := &cloudwatchlogs.FilterLogEventsInput{
		LogGroupName: &logGroupName,
		Interleaved:  aws.Bool(true),
		StartTime:    &startTimeInt64}

	if *grep != "" {
		params.FilterPattern = grep
	}

	if streamNames != nil {
		params.LogStreamNames = streamNames
	}

	if !*follow && endTimeInt64 != 0 {
		params.EndTime = &endTimeInt64
	}
	return params
}

func Tail(logGroupName *string, logStreamName *string, follow *bool, startTime *time.Time, endTime *time.Time, grep *string, printTimestamp *bool, printStreamName *bool) <-chan *string {
	cwl := cwClient()

	startTimeEpoch := timeutil.ParseTime(startTime.Format(timeutil.TimeFormat)).Unix()
	lastSeenTimestamp := startTimeEpoch

	var endTimeEpoch int64
	if !endTime.IsZero() {
		endTimeEpoch = timeutil.ParseTime(endTime.Format(timeutil.TimeFormat)).Unix()
	}

	var ids []string

	ch := make(chan *string)
	pageHandler := func(res *cloudwatchlogs.FilterLogEventsOutput, lastPage bool) bool {
		if len(res.Events) == 0 {
			time.Sleep(2 * time.Second)
		} else {
			for _, event := range res.Events {
				eventTimestamp := *event.Timestamp / 1000
				if eventTimestamp != lastSeenTimestamp {
					ids = nil
					lastSeenTimestamp = eventTimestamp
				} else {
					sort.Strings(ids)
				}
				idx := sort.SearchStrings(ids, *event.EventId)
				if ids == nil || (idx == len(ids) || ids[idx] != *event.EventId) {
					d := timeutil.FormatTimestamp(eventTimestamp)
					var msg string
					if *printTimestamp {
						msg = fmt.Sprintf("%s - ", color.GreenString(d))
					}
					if *printStreamName {
						msg = fmt.Sprintf("%s%s - ", msg, color.BlueString(*event.LogStreamName))
					}
					msg = fmt.Sprintf("%s%s", msg, *event.Message)
					ch <- &msg

				}
				ids = append(ids, *event.EventId)
			}
		}
		if lastPage && !*follow {
			time.Sleep(1 * time.Second)
			close(ch)
		}
		return !lastPage
	}
	var streams []*string
	if *logStreamName != "*" {
		for stream := range LsStreams(logGroupName, logStreamName) {
			streams = append(streams, stream)
		}
		if len(streams) == 0 {
			fmt.Println("No such log stream.")
			os.Exit(1)
		}
	}
	go func() {
		for *follow || lastSeenTimestamp == startTimeEpoch {
			logParam := params(*logGroupName, streams, lastSeenTimestamp, endTimeEpoch, grep, follow)
			error := cwl.FilterLogEventsPages(logParam, pageHandler)
			if error != nil {
				if awsErr, ok := error.(awserr.Error); ok {
					fmt.Println(awsErr.Message())
					os.Exit(1)
				}
			}
		}
	}()
	return ch
}

func LsGroups() <-chan *string {
	cwl := cwClient()
	ch := make(chan *string)
	params := &cloudwatchlogs.DescribeLogGroupsInput{
	//		LogGroupNamePrefix: aws.String("LogGroupName"),
	}

	handler := func(res *cloudwatchlogs.DescribeLogGroupsOutput, lastPage bool) bool {
		for _, logGroup := range res.LogGroups {
			ch <- logGroup.LogGroupName
		}
		if lastPage {
			close(ch)
		}
		return !lastPage
	}
	go func() {
		err := cwl.DescribeLogGroupsPages(params, handler)
		if err != nil {
			if awsErr, ok := err.(awserr.Error); ok {
				fmt.Println(awsErr.Message())
				os.Exit(1)
			}
		}
	}()
	return ch
}

func LsStreams(groupName *string, streamName *string) <-chan *string {
	cwl := cwClient()
	ch := make(chan *string)

	params := &cloudwatchlogs.DescribeLogStreamsInput{
		LogGroupName: groupName}
	if streamName != nil {
		params.LogStreamNamePrefix = streamName
	}
	handler := func(res *cloudwatchlogs.DescribeLogStreamsOutput, lastPage bool) bool {
		for _, logStream := range res.LogStreams {
			ch <- logStream.LogStreamName
		}
		if lastPage {
			close(ch)
		}
		return !lastPage
	}

	go func() {
		err := cwl.DescribeLogStreamsPages(params, handler)
		if err != nil {
			if awsErr, ok := err.(awserr.Error); ok {
				fmt.Println(awsErr.Message())
				os.Exit(1)
			}
		}
	}()
	return ch
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"time"
	"encoding/json"
	"net/http"
	"io/ioutil"

	"current/settings"
)

type DatabaseAPI struct {
	Id int64 `json:"id"`
	Name string `json:"name"`
	Environment string `json:"environment"`
	Project string `json:"project"`
	Team string `json:"team"`
	Created_At string `json:"created_at"`
	Quarantine_At string `json:"quarantine_dt"`
}

type LinkAPI struct {
	Next string `json:"next"`
}

type DatabaseListAPI struct {
	Link LinkAPI `json:"_links"`
	Databases []DatabaseAPI `json:"database"`
}

func main() {
	GetDatabases()
}

func GetDatabases() {
	url := settings.DBAAS_ENDPOINT + "/api/database/"
    for {
		body, err := GetJson(url)
    	if (err != nil) {
        	panic(err)
    	}

    	database_list := ParseResponse(body)
		for _, database := range database_list.Databases {
			fmt.Println(database);
		}
    	if database_list.Link.Next == "" {
    		break
    	}
    	url = database_list.Link.Next
    }
}

func GetJson(url string) ([]byte, error) {

	client := &http.Client{
		Timeout: 10 * time.Second,
	}

	request, err := http.NewRequest("GET", url, nil)
    if err != nil {
        return nil, err
    }

	request.SetBasicAuth(settings.DBAAS_USER, settings.DBAAS_PASSWORD)
	response, err := client.Do(request)
    if err != nil {
        return nil, err
    }

    body, err := ioutil.ReadAll(response.Body)
    if err != nil {
        return nil, err
    }

    return []byte(body), err
}

func ParseResponse(body []byte) (*DatabaseListAPI) {
    var database_list = new(DatabaseListAPI)
    err := json.Unmarshal(body, &database_list)
    if(err != nil) {
        fmt.Println("Error in parser", err)
    }
    return database_list
}<commit_msg>Removing import settings<commit_after>package main

import (
	"fmt"
	"time"
	"encoding/json"
	"net/http"
	"io/ioutil"

	"current/settings"
)

type DatabaseAPI struct {
	Id int64 `json:"id"`
	Name string `json:"name"`
	Environment string `json:"environment"`
	Project string `json:"project"`
	Team string `json:"team"`
	Created_At string `json:"created_at"`
	Quarantine_At string `json:"quarantine_dt"`
}

type LinkAPI struct {
	Next string `json:"next"`
}

type DatabaseListAPI struct {
	Link LinkAPI `json:"_links"`
	Databases []DatabaseAPI `json:"database"`
}

func GetDatabases() {
	url := settings.DBAAS_ENDPOINT + "/api/database/"
    for {
		body, err := GetJson(url)
    	if (err != nil) {
        	panic(err)
    	}

    	database_list := ParseResponse(body)
		for _, database := range database_list.Databases {
			fmt.Println(database);
		}
    	if database_list.Link.Next == "" {
    		break
    	}
    	url = database_list.Link.Next
    }
}

func GetJson(url string) ([]byte, error) {

	client := &http.Client{
		Timeout: 10 * time.Second,
	}

	request, err := http.NewRequest("GET", url, nil)
    if err != nil {
        return nil, err
    }

	request.SetBasicAuth(settings.DBAAS_USER, settings.DBAAS_PASSWORD)
	response, err := client.Do(request)
    if err != nil {
        return nil, err
    }

    body, err := ioutil.ReadAll(response.Body)
    if err != nil {
        return nil, err
    }

    return []byte(body), err
}

func ParseResponse(body []byte) (*DatabaseListAPI) {
    var database_list = new(DatabaseListAPI)
    err := json.Unmarshal(body, &database_list)
    if(err != nil) {
        fmt.Println("Error in parser", err)
    }
    return database_list
}<|endoftext|>


<commit_before>package restful

// Copyright 2013 Ernest Micklei. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.

import (
	"compress/zlib"
	"net/http"
)

var defaultRequestContentType string

// Request is a wrapper for a http Request that provides convenience methods
type Request struct {
	Request        *http.Request
	pathParameters map[string]string
	attributes     map[string]interface{} // for storing request-scoped values
	selectedRoute  *Route                 // is nil when no route was matched
}

func NewRequest(httpRequest *http.Request) *Request {
	return &Request{
		Request:        httpRequest,
		pathParameters: map[string]string{},
		attributes:     map[string]interface{}{},
	} // empty parameters, attributes
}

// If ContentType is missing or */* is given then fall back to this type, otherwise
// a "Unable to unmarshal content of type:" response is returned.
// Valid values are restful.MIME_JSON and restful.MIME_XML
// Example:
// 	restful.DefaultRequestContentType(restful.MIME_JSON)
func DefaultRequestContentType(mime string) {
	defaultRequestContentType = mime
}

// PathParameter accesses the Path parameter value by its name
func (r *Request) PathParameter(name string) string {
	return r.pathParameters[name]
}

// PathParameters accesses the Path parameter values
func (r *Request) PathParameters() map[string]string {
	return r.pathParameters
}

// QueryParameter returns the (first) Query parameter value by its name
func (r *Request) QueryParameter(name string) string {
	return r.Request.FormValue(name)
}

// QueryParameters returns the all the query parameters values by name
func (r *Request) QueryParameters(name string) []string {
	return r.Request.URL.Query()[name]
}

// BodyParameter parses the body of the request (once for typically a POST or a PUT) and returns the value of the given name or an error.
func (r *Request) BodyParameter(name string) (string, error) {
	err := r.Request.ParseForm()
	if err != nil {
		return "", err
	}
	return r.Request.PostFormValue(name), nil
}

// HeaderParameter returns the HTTP Header value of a Header name or empty if missing
func (r *Request) HeaderParameter(name string) string {
	return r.Request.Header.Get(name)
}

// ReadEntity checks the Accept header and reads the content into the entityPointer.
func (r *Request) ReadEntity(entityPointer interface{}) (err error) {
	contentType := r.Request.Header.Get(HEADER_ContentType)
	contentEncoding := r.Request.Header.Get(HEADER_ContentEncoding)

	// check if the request body needs decompression
	if ENCODING_GZIP == contentEncoding {
		gzipReader := currentCompressorProvider.AcquireGzipReader()
		defer currentCompressorProvider.ReleaseGzipReader(gzipReader)
		gzipReader.Reset(r.Request.Body)
		r.Request.Body = gzipReader
	} else if ENCODING_DEFLATE == contentEncoding {
		zlibReader, err := zlib.NewReader(r.Request.Body)
		if err != nil {
			return err
		}
		r.Request.Body = zlibReader
	}

	// lookup the EntityReader, use defaultRequestContentType if needed and provided
	entityReader, ok := entityAccessRegistry.accessorAt(contentType)
	if !ok {
		if len(defaultRequestContentType) != 0 {
			entityReader, ok = entityAccessRegistry.accessorAt(defaultRequestContentType)
		}
		if !ok {
			return NewError(http.StatusBadRequest, "Unable to unmarshal content of type:"+contentType)
		}
	}
	return entityReader.Read(r, entityPointer)
}

// SetAttribute adds or replaces the attribute with the given value.
func (r *Request) SetAttribute(name string, value interface{}) {
	r.attributes[name] = value
}

// Attribute returns the value associated to the given name. Returns nil if absent.
func (r Request) Attribute(name string) interface{} {
	return r.attributes[name]
}

// SelectedRoutePath root path + route path that matched the request, e.g. /meetings/{id}/attendees
// If no route was matched then return an empty string.
func (r Request) SelectedRoutePath() string {
	if r.selectedRoute == nil {
		return ""
	}
	// skip creating an accessor
	return r.selectedRoute.Path
}

// SelectedRoute returns a reader to access the selected Route by the container
// Returns nil if no route was matched.
func (r Request) SelectedRoute() RouteReader {
	if r.selectedRoute == nil {
		return nil
	}
	return routeAccessor{route: r.selectedRoute}
}
<commit_msg>change how to get query parameter issue #510<commit_after>package restful

// Copyright 2013 Ernest Micklei. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.

import (
	"compress/zlib"
	"net/http"
)

var defaultRequestContentType string

// Request is a wrapper for a http Request that provides convenience methods
type Request struct {
	Request        *http.Request
	pathParameters map[string]string
	attributes     map[string]interface{} // for storing request-scoped values
	selectedRoute  *Route                 // is nil when no route was matched
}

func NewRequest(httpRequest *http.Request) *Request {
	return &Request{
		Request:        httpRequest,
		pathParameters: map[string]string{},
		attributes:     map[string]interface{}{},
	} // empty parameters, attributes
}

// If ContentType is missing or */* is given then fall back to this type, otherwise
// a "Unable to unmarshal content of type:" response is returned.
// Valid values are restful.MIME_JSON and restful.MIME_XML
// Example:
//
//	restful.DefaultRequestContentType(restful.MIME_JSON)
func DefaultRequestContentType(mime string) {
	defaultRequestContentType = mime
}

// PathParameter accesses the Path parameter value by its name
func (r *Request) PathParameter(name string) string {
	return r.pathParameters[name]
}

// PathParameters accesses the Path parameter values
func (r *Request) PathParameters() map[string]string {
	return r.pathParameters
}

// QueryParameter returns the (first) Query parameter value by its name
func (r *Request) QueryParameter(name string) string {
	return r.Request.URL.Query().Get(name)
}

// QueryParameters returns the all the query parameters values by name
func (r *Request) QueryParameters(name string) []string {
	return r.Request.URL.Query()[name]
}

// BodyParameter parses the body of the request (once for typically a POST or a PUT) and returns the value of the given name or an error.
func (r *Request) BodyParameter(name string) (string, error) {
	err := r.Request.ParseForm()
	if err != nil {
		return "", err
	}
	return r.Request.PostFormValue(name), nil
}

// HeaderParameter returns the HTTP Header value of a Header name or empty if missing
func (r *Request) HeaderParameter(name string) string {
	return r.Request.Header.Get(name)
}

// ReadEntity checks the Accept header and reads the content into the entityPointer.
func (r *Request) ReadEntity(entityPointer interface{}) (err error) {
	contentType := r.Request.Header.Get(HEADER_ContentType)
	contentEncoding := r.Request.Header.Get(HEADER_ContentEncoding)

	// check if the request body needs decompression
	if ENCODING_GZIP == contentEncoding {
		gzipReader := currentCompressorProvider.AcquireGzipReader()
		defer currentCompressorProvider.ReleaseGzipReader(gzipReader)
		gzipReader.Reset(r.Request.Body)
		r.Request.Body = gzipReader
	} else if ENCODING_DEFLATE == contentEncoding {
		zlibReader, err := zlib.NewReader(r.Request.Body)
		if err != nil {
			return err
		}
		r.Request.Body = zlibReader
	}

	// lookup the EntityReader, use defaultRequestContentType if needed and provided
	entityReader, ok := entityAccessRegistry.accessorAt(contentType)
	if !ok {
		if len(defaultRequestContentType) != 0 {
			entityReader, ok = entityAccessRegistry.accessorAt(defaultRequestContentType)
		}
		if !ok {
			return NewError(http.StatusBadRequest, "Unable to unmarshal content of type:"+contentType)
		}
	}
	return entityReader.Read(r, entityPointer)
}

// SetAttribute adds or replaces the attribute with the given value.
func (r *Request) SetAttribute(name string, value interface{}) {
	r.attributes[name] = value
}

// Attribute returns the value associated to the given name. Returns nil if absent.
func (r Request) Attribute(name string) interface{} {
	return r.attributes[name]
}

// SelectedRoutePath root path + route path that matched the request, e.g. /meetings/{id}/attendees
// If no route was matched then return an empty string.
func (r Request) SelectedRoutePath() string {
	if r.selectedRoute == nil {
		return ""
	}
	// skip creating an accessor
	return r.selectedRoute.Path
}

// SelectedRoute returns a reader to access the selected Route by the container
// Returns nil if no route was matched.
func (r Request) SelectedRoute() RouteReader {
	if r.selectedRoute == nil {
		return nil
	}
	return routeAccessor{route: r.selectedRoute}
}
<|endoftext|>


<commit_before>package gottp

import (
	"compress/gzip"
	"compress/zlib"
	"log"
	"net/http"
	"strconv"
	"strings"

	"gopkg.in/simversity/gottp.v3/utils"
)

//*http.Request, rw ResponseWriter

const PAGE_SIZE = 30
const SKIP = 0
const serverUA = "Gottp Server"
const ERROR = `Oops! An Internal Error occured while performing that action.
Please try again later`

type WireSender interface {
	SendOverWire() utils.Q
}

type Paginator struct {
	Skip  int
	Limit int
	Wlt   string
	Wgt   string
	Wkey  string
	Ids   []string
}

func makeInt(val interface{}, fallback int) int {
	switch val.(type) {
	case int:
		return val.(int)
	case string:
		_val := val.(string)
		ret, err := strconv.Atoi(_val)
		if err == nil {
			return ret
		}
	case float64:
		return int(val.(float64))
	}
	return fallback
}

func makeString(val interface{}) (ret string) {
	switch val.(type) {
	case string:
		ret = val.(string)
	case int:
		val := val.(int)
		ret = strconv.Itoa(val)
	}
	return
}

type Request struct {
	Request    *http.Request
	Writer     http.ResponseWriter
	UrlArgs    *map[string]string
	pipeOutput chan<- *utils.Q
	pipeIndex  int
	params     *utils.Q
}

func (r *Request) makeUrlArgs() {
	original := *r.params
	if r.UrlArgs != nil {
		for key, value := range *r.UrlArgs {
			original[key] = value
		}
	}
}

func (r *Request) GetPaginator() *Paginator {
	p := Paginator{Limit: -1}
	qp := r.GetArguments()
	for key, value := range *qp {
		switch key {
		case "skip":
			p.Skip = makeInt(value, SKIP)
		case "limit":
			p.Limit = makeInt(value, PAGE_SIZE)
		case "wlt":
			p.Wlt = makeString(value)
		case "wgt":
			p.Wgt = makeString(value)
		case "wkey":
			p.Wkey = makeString(value)
		case "ids":
			ids, ok := value.([]string)
			if !ok {
				id := value.(string)
				p.Ids = []string{id}
			} else {
				p.Ids = ids
			}
		}
	}

	if p.Limit < 0 {
		p.Limit = PAGE_SIZE
	}

	return &p
}

func (r *Request) makeUrlParams() {
	original := *r.params
	r.Request.ParseForm()
	for key, value := range r.Request.Form {
		length := len(value)
		if length == 1 {
			original[key] = value[0]
		} else {
			original[key] = value
		}
	}
}

func (r *Request) makeBodyParams() {
	//if (r.Request.Method == "PUT" || r.Request.Method == "POST") {
	if r.Request.ContentLength != 0 {
		utils.DecodeStream(r.Request.Body, r.params)
	}
}

func (r *Request) GetArguments() *utils.Q {
	if r.params == nil {
		r.params = &utils.Q{}
		r.makeUrlArgs()
		r.makeBodyParams()
		r.makeUrlParams()
	}

	return r.params
}

func (r *Request) ConvertArguments(f interface{}) {
	utils.Convert(r.GetArguments(), f)
}

func (r *Request) GetArgument(key string) interface{} {
	args := *r.GetArguments()
	return args[key]
}

func (r *Request) ConvertArgument(key string, f interface{}) {
	args := *r.GetArguments()
	val := args[key]
	utils.Convert(val, f)
}

func (r *Request) Finish(data interface{}) []byte {
	r.Writer.Header().Set("Server", serverUA)
	r.Writer.Header().Set("Access-Control-Allow-Origin", "*")
	r.Writer.Header().Set("Content-Type", "application/json")
	return utils.Encoder(data)
}

func (r *Request) Redirect(url string, status int) {
	log.Println("Redirecting to", url)
	http.Redirect(r.Writer, r.Request, url, status)
	return
}

func (r *Request) Write(data interface{}) {
	var piped utils.Q

	if v, ok := data.(WireSender); ok {
		piped = v.SendOverWire()
	} else {
		piped = utils.Q{
			"data":    data,
			"status":  http.StatusOK,
			"message": "",
		}
	}

	if r.pipeOutput != nil {
		piped["index"] = r.pipeIndex
		r.pipeOutput <- &piped
	} else if strings.Contains(
		r.Request.Header.Get("Accept-Encoding"), "deflate",
	) {
		r.Writer.Header().Set("Content-Encoding", "deflate")

		gz := zlib.NewWriter(r.Writer)
		defer gz.Close()
		gz.Write(r.Finish(piped))

	} else if strings.Contains(
		r.Request.Header.Get("Accept-Encoding"), "gzip",
	) {
		r.Writer.Header().Set("Content-Encoding", "gzip")

		gz := gzip.NewWriter(r.Writer)
		defer gz.Close()
		gz.Write(r.Finish(piped))

	} else {
		r.Writer.Write(r.Finish(piped))
	}
}

func (r *Request) Raise(e HttpError) {
	r.Write(e)
}

func performRequest(handler Handler, p *Request) {
	method := (*p).Request.Method

	switch method {
	case "GET":
		handler.Get(p)
	case "POST":
		handler.Post(p)
	case "PUT":
		handler.Put(p)
	case "DELETE":
		handler.Delete(p)
	case "HEAD":
		handler.Head(p)
	case "OPTIONS":
		handler.Options(p)
	case "PATCH":
		handler.Patch(p)
	default:
		log.Println("Unsupported method", method)
	}
}

func doRequest(request *Request, availableUrls *[]*Url) {
	requestUrl := request.Request.URL.Path
	defer Tracer.Notify(getTracerExtra(request))

	for _, url := range *availableUrls {
		urlArgs, err := url.MakeUrlArgs(&requestUrl)
		if !err {
			request.UrlArgs = urlArgs
			performRequest(url.handler, request)
			return
		}
	}

	e := HttpError{404, requestUrl + " not Found"}
	request.Raise(e)
	return
}
<commit_msg>Do not use a shared data structure<commit_after>package gottp

import (
	"compress/gzip"
	"compress/zlib"
	"log"
	"net/http"
	"strconv"
	"strings"

	"gopkg.in/simversity/gottp.v3/utils"
)

//*http.Request, rw ResponseWriter

const PAGE_SIZE = 30
const SKIP = 0
const serverUA = "Gottp Server"
const ERROR = `Oops! An Internal Error occured while performing that action.
Please try again later`

type WireSender interface {
	SendOverWire() utils.Q
}

type Paginator struct {
	Skip  int
	Limit int
	Wlt   string
	Wgt   string
	Wkey  string
	Ids   []string
}

func makeInt(val interface{}, fallback int) int {
	switch val.(type) {
	case int:
		return val.(int)
	case string:
		_val := val.(string)
		ret, err := strconv.Atoi(_val)
		if err == nil {
			return ret
		}
	case float64:
		return int(val.(float64))
	}
	return fallback
}

func makeString(val interface{}) (ret string) {
	switch val.(type) {
	case string:
		ret = val.(string)
	case int:
		val := val.(int)
		ret = strconv.Itoa(val)
	}
	return
}

type Request struct {
	Request    *http.Request
	Writer     http.ResponseWriter
	UrlArgs    *map[string]string
	pipeOutput chan<- *utils.Q
	pipeIndex  int
	params     *utils.Q
}

func (r *Request) makeUrlArgs(params utils.Q) {
	if r.UrlArgs != nil {
		for key, value := range *r.UrlArgs {
			params[key] = value
		}
	}
}

func (r *Request) GetPaginator() *Paginator {
	p := Paginator{Limit: -1}
	qp := r.GetArguments()
	for key, value := range *qp {
		switch key {
		case "skip":
			p.Skip = makeInt(value, SKIP)
		case "limit":
			p.Limit = makeInt(value, PAGE_SIZE)
		case "wlt":
			p.Wlt = makeString(value)
		case "wgt":
			p.Wgt = makeString(value)
		case "wkey":
			p.Wkey = makeString(value)
		case "ids":
			ids, ok := value.([]string)
			if !ok {
				id := value.(string)
				p.Ids = []string{id}
			} else {
				p.Ids = ids
			}
		}
	}

	if p.Limit < 0 {
		p.Limit = PAGE_SIZE
	}

	return &p
}

func (r *Request) makeUrlParams(params utils.Q) {
	r.Request.ParseForm()
	for key, value := range r.Request.Form {
		length := len(value)
		if length == 1 {
			params[key] = value[0]
		} else {
			params[key] = value
		}
	}
}

func (r *Request) makeBodyParams(params utils.Q) {
	//if (r.Request.Method == "PUT" || r.Request.Method == "POST") {
	if r.Request.ContentLength != 0 {
		utils.DecodeStream(r.Request.Body, &params)
	}
}

func (r *Request) GetArguments() *utils.Q {
	if r.params == nil {
		params := utils.Q{}
		r.makeUrlArgs(params)
		r.makeBodyParams(params)
		r.makeUrlParams(params)
		r.params = &params
	}

	return r.params
}

func (r *Request) ConvertArguments(f interface{}) {
	utils.Convert(r.GetArguments(), f)
}

func (r *Request) GetArgument(key string) interface{} {
	args := *r.GetArguments()
	return args[key]
}

func (r *Request) ConvertArgument(key string, f interface{}) {
	args := *r.GetArguments()
	val := args[key]
	utils.Convert(val, f)
}

func (r *Request) Finish(data interface{}) []byte {
	r.Writer.Header().Set("Server", serverUA)
	r.Writer.Header().Set("Access-Control-Allow-Origin", "*")
	r.Writer.Header().Set("Content-Type", "application/json")
	return utils.Encoder(data)
}

func (r *Request) Redirect(url string, status int) {
	log.Println("Redirecting to", url)
	http.Redirect(r.Writer, r.Request, url, status)
	return
}

func (r *Request) Write(data interface{}) {
	var piped utils.Q

	if v, ok := data.(WireSender); ok {
		piped = v.SendOverWire()
	} else {
		piped = utils.Q{
			"data":    data,
			"status":  http.StatusOK,
			"message": "",
		}
	}

	if r.pipeOutput != nil {
		piped["index"] = r.pipeIndex
		r.pipeOutput <- &piped
	} else if strings.Contains(
		r.Request.Header.Get("Accept-Encoding"), "deflate",
	) {
		r.Writer.Header().Set("Content-Encoding", "deflate")

		gz := zlib.NewWriter(r.Writer)
		defer gz.Close()
		gz.Write(r.Finish(piped))

	} else if strings.Contains(
		r.Request.Header.Get("Accept-Encoding"), "gzip",
	) {
		r.Writer.Header().Set("Content-Encoding", "gzip")

		gz := gzip.NewWriter(r.Writer)
		defer gz.Close()
		gz.Write(r.Finish(piped))

	} else {
		r.Writer.Write(r.Finish(piped))
	}
}

func (r *Request) Raise(e HttpError) {
	r.Write(e)
}

func performRequest(handler Handler, p *Request) {
	method := (*p).Request.Method

	switch method {
	case "GET":
		handler.Get(p)
	case "POST":
		handler.Post(p)
	case "PUT":
		handler.Put(p)
	case "DELETE":
		handler.Delete(p)
	case "HEAD":
		handler.Head(p)
	case "OPTIONS":
		handler.Options(p)
	case "PATCH":
		handler.Patch(p)
	default:
		log.Println("Unsupported method", method)
	}
}

func doRequest(request *Request, availableUrls *[]*Url) {
	requestUrl := request.Request.URL.Path
	defer Tracer.Notify(getTracerExtra(request))

	for _, url := range *availableUrls {
		urlArgs, err := url.MakeUrlArgs(&requestUrl)
		if !err {
			request.UrlArgs = urlArgs
			performRequest(url.handler, request)
			return
		}
	}

	e := HttpError{404, requestUrl + " did not match any exposed urls."}
	request.Raise(e)
	return
}
<|endoftext|>