<commit_before>/*
 * Minio Client (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"encoding/json"
	"runtime"
	"strings"
	"time"

	"github.com/fatih/color"
	"github.com/minio/cli"
	"github.com/minio/mc/pkg/console"
	"github.com/minio/minio-xl/pkg/probe"
)

// Check for new software updates.
var updateCmd = cli.Command{
	Name:   "update",
	Usage:  "Check for new software updates.",
	Action: mainUpdate,
	CustomHelpTemplate: `Name:
   mc {{.Name}} - {{.Usage}}

USAGE:
   mc {{.Name}} release
   mc {{.Name}} experimental

EXAMPLES:
   1. Check for new official releases
      $ mc {{.Name}} release

   2. Check for new experimental releases
      $ mc {{.Name}} experimental
`,
}

// update URL endpoints.
const (
	mcUpdateURL       = "https://dl.minio.io:9000/updates/updates.json"
	mcExperimentalURL = "https://dl.minio.io:9000/updates/experimental.json"
)

// mcUpdates container to hold updates json.
type mcUpdates struct {
	BuildDate string
	Platforms map[string]string
}

// updateMessage container to hold update messages.
type updateMessage struct {
	Update   bool   `json:"update"`
	Download string `json:"downloadURL"`
	Version  string `json:"version"`
}

// String colorized update message.
func (u updateMessage) String() string {
	if u.Update {
		var msg string
		if runtime.GOOS == "windows" {
			msg = "mc.exe cp " + u.Download + " .\\mc.exe"
		} else {
			msg = "mc cp " + u.Download + " ./mc.new; chmod 755 ./mc.new"
		}
		msg, err := colorizeUpdateMessage(msg)
		fatalIf(err.Trace(msg), "Unable to colorize experimental update notification string ‘"+msg+"’.")
		return msg
	}
	return console.Colorize("UpdateMessage", "You are already running the most recent version of ‘mc’.")
}

// JSON jsonified update message.
func (u updateMessage) JSON() string {
	updateMessageJSONBytes, err := json.Marshal(u)
	fatalIf(probe.NewError(err), "Unable to marshal into JSON.")

	return string(updateMessageJSONBytes)
}

// check valid input arguments.
func checkUpdateSyntax(ctx *cli.Context) {
	if ctx.Args().First() == "help" || !ctx.Args().Present() {
		cli.ShowCommandHelpAndExit(ctx, "update", 1) // last argument is exit code
	}
	arg := strings.TrimSpace(ctx.Args().First())
	if arg != "release" && arg != "experimental" {
		fatalIf(errInvalidArgument().Trace(arg), "Unrecognized argument provided.")
	}
}

// main for update command.
func mainUpdate(ctx *cli.Context) {
	// check input arguments
	checkUpdateSyntax(ctx)

	// Additional command speific theme customization.
	console.SetColor("UpdateMessage", color.New(color.FgGreen, color.Bold))

	arg := strings.TrimSpace(ctx.Args().First())
	switch arg {
	case "release":
		getReleaseUpdate()
	case "experimental":
		getExperimentalUpdate()
	}
}

// verify new updates for experimentals.
func getExperimentalUpdate() {
	clnt, err := url2Client(mcExperimentalURL)
	fatalIf(err.Trace(mcExperimentalURL), "Unable to initalize experimental URL.")

	data, _, err := clnt.Get(0, 0)
	fatalIf(err.Trace(mcExperimentalURL), "Unable to read from experimental URL ‘"+mcExperimentalURL+"’.")

	if mcVersion == "UNOFFICIAL.GOGET" {
		fatalIf(errDummy().Trace(mcVersion), "Update mechanism is not supported for ‘go get’ based binary builds.  Please download official releases from https://minio.io/#mc")
	}

	current, e := time.Parse(time.RFC3339, mcVersion)
	fatalIf(probe.NewError(e), "Unable to parse Version string as time.")

	if current.IsZero() {
		fatalIf(errDummy().Trace(), "Experimental updates not supported for custom build. Version field is empty. Please download official releases from https://minio.io/#mc")
	}

	var experimentals mcUpdates
	decoder := json.NewDecoder(data)
	e = decoder.Decode(&experimentals)
	fatalIf(probe.NewError(e), "Unable to decode experimental update notification.")

	latest, e := time.Parse(time.RFC3339, experimentals.BuildDate)
	fatalIf(probe.NewError(e), "Unable to parse BuildDate.")

	if latest.IsZero() {
		fatalIf(errDummy().Trace(), "Unable to validate any experimental update available at this time. Please open an issue at https://github.com/minio/mc/issues")
	}

	mcExperimentalURLParse := clnt.GetURL()
	downloadURL := (mcExperimentalURLParse.Scheme +
		string(mcExperimentalURLParse.SchemeSeparator) +
		mcExperimentalURLParse.Host + string(mcExperimentalURLParse.Separator) +
		experimentals.Platforms[runtime.GOOS+"-"+runtime.GOARCH])

	updateMsg := updateMessage{
		Download: downloadURL,
		Version:  mcVersion,
	}
	if latest.After(current) {
		updateMsg.Update = true
	}
	printMsg(updateMsg)
}

// verify updates for releases.
func getReleaseUpdate() {
	clnt, err := url2Client(mcUpdateURL)
	fatalIf(err.Trace(mcUpdateURL), "Unable to initalize update URL.")

	data, _, err := clnt.Get(0, 0)
	fatalIf(err.Trace(mcUpdateURL), "Unable to read from update URL ‘"+mcUpdateURL+"’.")

	if mcVersion == "UNOFFICIAL.GOGET" {
		fatalIf(errDummy().Trace(mcVersion), "Update mechanism is not supported for ‘go get’ based binary builds.  Please download official releases from https://minio.io/#mc")
	}

	current, e := time.Parse(time.RFC3339, mcVersion)
	fatalIf(probe.NewError(e), "Unable to parse Version string as time.")

	if current.IsZero() {
		fatalIf(errDummy().Trace(), "Updates not supported for custom build. Version field is empty. Please download official releases from https://minio.io/#mc")
	}

	var updates mcUpdates
	decoder := json.NewDecoder(data)
	e = decoder.Decode(&updates)
	fatalIf(probe.NewError(e), "Unable to decode update notification.")

	latest, e := time.Parse(time.RFC3339, updates.BuildDate)
	fatalIf(probe.NewError(e), "Unable to parse BuildDate.")

	if latest.IsZero() {
		fatalIf(errDummy().Trace(), "Unable to validate any update available at this time. Please open an issue at https://github.com/minio/mc/issues")
	}

	mcUpdateURLParse := clnt.GetURL()
	downloadURL := mcUpdateURLParse.Scheme +
		string(mcUpdateURLParse.SchemeSeparator) +
		mcUpdateURLParse.Host + string(mcUpdateURLParse.Separator) +
		updates.Platforms[runtime.GOOS+"-"+runtime.GOARCH]

	updateMsg := updateMessage{
		Download: downloadURL,
		Version:  mcVersion,
	}
	if latest.After(current) {
		updateMsg.Update = true
	}
	printMsg(updateMsg)
}
<commit_msg>update: Support backward compatible date formate<commit_after>/*
 * Minio Client (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"encoding/json"
	"net/http"
	"runtime"
	"strings"
	"time"

	"github.com/fatih/color"
	"github.com/minio/cli"
	"github.com/minio/mc/pkg/console"
	"github.com/minio/minio-xl/pkg/probe"
)

// Check for new software updates.
var updateCmd = cli.Command{
	Name:   "update",
	Usage:  "Check for new software updates.",
	Action: mainUpdate,
	CustomHelpTemplate: `Name:
   mc {{.Name}} - {{.Usage}}

USAGE:
   mc {{.Name}} release
   mc {{.Name}} experimental

EXAMPLES:
   1. Check for new official releases
      $ mc {{.Name}} release

   2. Check for new experimental releases
      $ mc {{.Name}} experimental
`,
}

// update URL endpoints.
const (
	mcUpdateURL       = "https://dl.minio.io:9000/updates/updates.json"
	mcExperimentalURL = "https://dl.minio.io:9000/updates/experimental.json"
)

// mcUpdates container to hold updates json.
type mcUpdates struct {
	BuildDate string
	Platforms map[string]string
}

// updateMessage container to hold update messages.
type updateMessage struct {
	Update   bool   `json:"update"`
	Download string `json:"downloadURL"`
	Version  string `json:"version"`
}

// String colorized update message.
func (u updateMessage) String() string {
	if u.Update {
		var msg string
		if runtime.GOOS == "windows" {
			msg = "mc.exe cp " + u.Download + " .\\mc.exe"
		} else {
			msg = "mc cp " + u.Download + " ./mc.new; chmod 755 ./mc.new"
		}
		msg, err := colorizeUpdateMessage(msg)
		fatalIf(err.Trace(msg), "Unable to colorize experimental update notification string ‘"+msg+"’.")
		return msg
	}
	return console.Colorize("UpdateMessage", "You are already running the most recent version of ‘mc’.")
}

// JSON jsonified update message.
func (u updateMessage) JSON() string {
	updateMessageJSONBytes, err := json.Marshal(u)
	fatalIf(probe.NewError(err), "Unable to marshal into JSON.")

	return string(updateMessageJSONBytes)
}

// check valid input arguments.
func checkUpdateSyntax(ctx *cli.Context) {
	if ctx.Args().First() == "help" || !ctx.Args().Present() {
		cli.ShowCommandHelpAndExit(ctx, "update", 1) // last argument is exit code
	}
	arg := strings.TrimSpace(ctx.Args().First())
	if arg != "release" && arg != "experimental" {
		fatalIf(errInvalidArgument().Trace(arg), "Unrecognized argument provided.")
	}
}

// main for update command.
func mainUpdate(ctx *cli.Context) {
	// check input arguments
	checkUpdateSyntax(ctx)

	// Additional command speific theme customization.
	console.SetColor("UpdateMessage", color.New(color.FgGreen, color.Bold))

	arg := strings.TrimSpace(ctx.Args().First())
	switch arg {
	case "release":
		getReleaseUpdate()
	case "experimental":
		getExperimentalUpdate()
	}
}

// verify new updates for experimentals.
func getExperimentalUpdate() {
	clnt, err := url2Client(mcExperimentalURL)
	fatalIf(err.Trace(mcExperimentalURL), "Unable to initalize experimental URL.")

	data, _, err := clnt.Get(0, 0)
	fatalIf(err.Trace(mcExperimentalURL), "Unable to read from experimental URL ‘"+mcExperimentalURL+"’.")

	if mcVersion == "UNOFFICIAL.GOGET" {
		fatalIf(errDummy().Trace(mcVersion), "Update mechanism is not supported for ‘go get’ based binary builds.  Please download official releases from https://minio.io/#mc")
	}

	current, e := time.Parse(time.RFC3339, mcVersion)
	fatalIf(probe.NewError(e), "Unable to parse Version string as time.")

	if current.IsZero() {
		fatalIf(errDummy().Trace(), "Experimental updates not supported for custom build. Version field is empty. Please download official releases from https://minio.io/#mc")
	}

	var experimentals mcUpdates
	decoder := json.NewDecoder(data)
	e = decoder.Decode(&experimentals)
	fatalIf(probe.NewError(e), "Unable to decode experimental update notification.")

	latest, e := time.Parse(time.RFC3339, experimentals.BuildDate)
	if e != nil {
		latest, e = time.Parse(http.TimeFormat, experimentals.BuildDate)
		fatalIf(probe.NewError(e), "Unable to parse BuildDate.")
	}

	if latest.IsZero() {
		fatalIf(errDummy().Trace(), "Unable to validate any experimental update available at this time. Please open an issue at https://github.com/minio/mc/issues")
	}

	mcExperimentalURLParse := clnt.GetURL()
	downloadURL := (mcExperimentalURLParse.Scheme +
		string(mcExperimentalURLParse.SchemeSeparator) +
		mcExperimentalURLParse.Host + string(mcExperimentalURLParse.Separator) +
		experimentals.Platforms[runtime.GOOS+"-"+runtime.GOARCH])

	updateMsg := updateMessage{
		Download: downloadURL,
		Version:  mcVersion,
	}
	if latest.After(current) {
		updateMsg.Update = true
	}
	printMsg(updateMsg)
}

// verify updates for releases.
func getReleaseUpdate() {
	clnt, err := url2Client(mcUpdateURL)
	fatalIf(err.Trace(mcUpdateURL), "Unable to initalize update URL.")

	data, _, err := clnt.Get(0, 0)
	fatalIf(err.Trace(mcUpdateURL), "Unable to read from update URL ‘"+mcUpdateURL+"’.")

	if mcVersion == "UNOFFICIAL.GOGET" {
		fatalIf(errDummy().Trace(mcVersion), "Update mechanism is not supported for ‘go get’ based binary builds.  Please download official releases from https://minio.io/#mc")
	}

	current, e := time.Parse(time.RFC3339, mcVersion)
	fatalIf(probe.NewError(e), "Unable to parse Version string as time.")

	if current.IsZero() {
		fatalIf(errDummy().Trace(), "Updates not supported for custom build. Version field is empty. Please download official releases from https://minio.io/#mc")
	}

	var updates mcUpdates
	decoder := json.NewDecoder(data)
	e = decoder.Decode(&updates)
	fatalIf(probe.NewError(e), "Unable to decode update notification.")

	latest, e := time.Parse(time.RFC3339, updates.BuildDate)
	if e != nil {
		latest, e = time.Parse(http.TimeFormat, updates.BuildDate)
		fatalIf(probe.NewError(e), "Unable to parse BuildDate.")
	}

	if latest.IsZero() {
		fatalIf(errDummy().Trace(), "Unable to validate any update available at this time. Please open an issue at https://github.com/minio/mc/issues")
	}

	mcUpdateURLParse := clnt.GetURL()
	downloadURL := mcUpdateURLParse.Scheme +
		string(mcUpdateURLParse.SchemeSeparator) +
		mcUpdateURLParse.Host + string(mcUpdateURLParse.Separator) +
		updates.Platforms[runtime.GOOS+"-"+runtime.GOARCH]

	updateMsg := updateMessage{
		Download: downloadURL,
		Version:  mcVersion,
	}
	if latest.After(current) {
		updateMsg.Update = true
	}
	printMsg(updateMsg)
}
<|endoftext|>


<commit_before>package io

import (
	"encoding/json"
	"io/ioutil"

	d "github.com/appPlant/alpinepass/src/data"
	"github.com/appPlant/alpinepass/src/util"
)

func writeFile(data string, filename string) {
	err := ioutil.WriteFile(filename, []byte(data), 0644)
	util.CheckError(err)
}

func ConfigToJSON(config d.Config, pretty bool) {

}

func ConfigsToJSON(configs []d.Config, pretty bool) string {
	var configsJSON []byte
	var err error
	if pretty {
		configsJSON, err = json.MarshalIndent(configs, "", "    ")
	} else {
		configsJSON, err = json.Marshal(configs)
	}
	util.CheckError(err)
	return string(configsJSON)
}

//WriteJSON writes Config to a file in JSON format.
func WriteJSON(path string, configs []d.Config, pretty bool) {
	if path == "" {
		path = "output.json"
	}

	var configsJSON []byte
	var err error
	if pretty {
		configsJSON, err = json.MarshalIndent(configs, "", "    ")
	} else {
		configsJSON, err = json.Marshal(configs)
	}
	util.CheckError(err)
	writeFile(string(configsJSON), path)
}
<commit_msg>Remove obsolete code<commit_after>package io

import (
	"encoding/json"
	"io/ioutil"

	d "github.com/appPlant/alpinepass/src/data"
	"github.com/appPlant/alpinepass/src/util"
)

func writeFile(data string, filename string) {
	err := ioutil.WriteFile(filename, []byte(data), 0644)
	util.CheckError(err)
}

//WriteJSON writes Config to a file in JSON format.
func WriteJSON(path string, configs []d.Config, pretty bool) {
	if path == "" {
		path = "output.json"
	}

	var configsJSON []byte
	var err error
	if pretty {
		configsJSON, err = json.MarshalIndent(configs, "", "    ")
	} else {
		configsJSON, err = json.Marshal(configs)
	}
	util.CheckError(err)
	writeFile(string(configsJSON), path)
}
<|endoftext|>


<commit_before>package properties

import (
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"strconv"
	"strings"
)

// Parser 是 Properties 文件的解析器, 支持保留注释、空白行,
// 以及保留所有元素的顺序
type Parser struct {
	data      []rune
	tokenizer *Tokenizer
}

// NewParser 创建新的解析器
func NewParser(r io.Reader) (*Parser, error) {
	bytes, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, err
	}
	data := []rune(string(bytes))
	return &Parser{data: data, tokenizer: NewTokenizer(data)}, nil
}

// 获取下一个节点信息
func (p *Parser) Next() (Node, error) {
	token := p.tokenizer.Next()
	if token == nil {
		return nil, nil
	}

	if token.Type == HashToken {
		var comments bytes.Buffer
		for {
			token = p.tokenizer.Next()
			if token == nil || token.Type == NewLineToken {
				return &CommentNode{Text: comments.String()}, nil
			} else if token.Type == ContinueLineToken {
				comments.WriteString(string(p.data[token.Offset : token.Offset+token.Length]))
				return &CommentNode{Text: comments.String()}, nil
			} else {
				comments.WriteString(string(p.data[token.Offset : token.Offset+token.Length]))
			}
		}
	} else if token.Type == NewLineToken {
		return &BlankNode{}, nil
	} else if token.Type == ContinueLineToken {
		return nil, errors.New("invalid continuation characters")
	} else if token.Type == EqualsToken {
		return nil, errors.New("invalid = at the beginning of a line")
	} else {
		// 非换行、结束、= 令牌之前的字符都收集起来
		// get key
		var key bytes.Buffer
		key.WriteString(string(p.data[token.Offset : token.Offset+token.Length]))
		equalsExist := false
		for {
			token = p.tokenizer.Next()
			if token == nil || token.Type == NewLineToken {
				break
			} else if token.Type == EqualsToken {
				equalsExist = true
				break
			} else if token.Type == HashToken {
				return nil, errors.New("invalid hash character in key")
			} else if token.Type == ContinueLineToken {
				continue
			}
			key.WriteString(string(p.data[token.Offset : token.Offset+token.Length]))
		}
		if !equalsExist {
			return nil, errors.New("no value provide for " + key.String())
		}
		keyStr, err := strconv.Unquote(`"` + strings.TrimRight(key.String(), "\r\t\f ") + `"`)
		if err != nil {
			return nil, err
		}

		// get value
		var value bytes.Buffer
		for {
			token = p.tokenizer.Next()
			if token == nil || token.Type == NewLineToken {
				break
			}
			if token.Type == ContinueLineToken {
				continue
			}
			value.WriteString(string(p.data[token.Offset : token.Offset+token.Length]))
		}
		valueStr, err := strconv.Unquote(`"` + strings.TrimLeft(value.String(), "\r\t\f ") + `"`)
		if err != nil {
			return nil, err
		}
		return &PropertyNode{Key: keyStr, Value: valueStr}, nil
	}

}
<commit_msg>implement unescapse func<commit_after>package properties

import (
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"strconv"
	"strings"
)

// Parser 是 Properties 文件的解析器, 支持保留注释、空白行,
// 以及保留所有元素的顺序
type Parser struct {
	data      []rune
	tokenizer *Tokenizer
}

// NewParser 创建新的解析器
func NewParser(r io.Reader) (*Parser, error) {
	bytes, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, err
	}
	data := []rune(string(bytes))
	return &Parser{data: data, tokenizer: NewTokenizer(data)}, nil
}

// 获取下一个节点信息
func (p *Parser) Next() (Node, error) {
	token := p.tokenizer.Next()
	if token == nil {
		return nil, nil
	}

	if token.Type == HashToken {
		var comments bytes.Buffer
		for {
			token = p.tokenizer.Next()
			if token == nil || token.Type == NewLineToken {
				return &CommentNode{Text: comments.String()}, nil
			} else if token.Type == ContinueLineToken {
				comments.WriteString(string(p.data[token.Offset : token.Offset+token.Length]))
				return &CommentNode{Text: comments.String()}, nil
			} else {
				comments.WriteString(string(p.data[token.Offset : token.Offset+token.Length]))
			}
		}
	} else if token.Type == NewLineToken {
		return &BlankNode{}, nil
	} else if token.Type == ContinueLineToken {
		return nil, errors.New("invalid continuation characters")
	} else if token.Type == EqualsToken {
		return nil, errors.New("invalid = at the beginning of a line")
	} else {
		// 非换行、结束、= 令牌之前的字符都收集起来
		// get key
		var key bytes.Buffer
		key.WriteString(string(p.data[token.Offset : token.Offset+token.Length]))
		equalsExist := false
		for {
			token = p.tokenizer.Next()
			if token == nil || token.Type == NewLineToken {
				break
			} else if token.Type == EqualsToken {
				equalsExist = true
				break
			} else if token.Type == HashToken {
				return nil, errors.New("invalid hash character in key")
			} else if token.Type == ContinueLineToken {
				continue
			}
			key.WriteString(string(p.data[token.Offset : token.Offset+token.Length]))
		}
		if !equalsExist {
			return nil, errors.New("no value provide for " + key.String())
		}
		keyStr, err := unescape(strings.TrimRight(key.String(), "\r\t\f "))
		if err != nil {
			return nil, err
		}

		// get value
		var value bytes.Buffer
		for {
			token = p.tokenizer.Next()
			if token == nil || token.Type == NewLineToken {
				break
			}
			if token.Type == ContinueLineToken {
				continue
			}
			value.WriteString(string(p.data[token.Offset : token.Offset+token.Length]))
		}
		valueStr, err := unescape(strings.TrimLeft(value.String(), "\r\t\f "))
		if err != nil {
			return nil, err
		}
		return &PropertyNode{Key: keyStr, Value: valueStr}, nil
	}
}

func unescape(s string) (string, error) {
	r := []rune(s)
	var buf bytes.Buffer
	escaped := false
	position := 0
	for position < len(r) {
		c := r[position]
		position++
		if c == '\\' && !escaped {
			escaped = true
			continue
		}
		if !escaped {
			buf.WriteRune(c)
		} else {
			switch c {
			case 'r':
				buf.WriteRune('\r')
			case 'n':
				buf.WriteRune('\n')
			case 'f':
				buf.WriteRune('\f')
			case 't':
				buf.WriteRune('\t')
			case 'u':
				if position+4 > len(s) {
					return "", strconv.ErrSyntax
				}
				var v rune
				for i := 0; i < 4; i++ {
					x, ok := unhex(byte(r[position]))
					if !ok {
						return "", strconv.ErrSyntax
					}
					v = v<<4 | x
					position++

				}
				buf.WriteRune(v)
			default:
				buf.WriteRune(c)

			}
		}
		escaped = false
	}
	return buf.String(), nil
}

func unhex(b byte) (v rune, ok bool) {
	c := rune(b)
	switch {
	case '0' <= c && c <= '9':
		return c - '0', true
	case 'a' <= c && c <= 'f':
		return c - 'a' + 10, true
	case 'A' <= c && c <= 'F':
		return c - 'A' + 10, true
	}
	return
}
<|endoftext|>


<commit_before>package aws

import (
	"bufio"
	"errors"
	"fmt"
	"html/template"
	"math/rand"
	"os"
	"regexp"
	"strconv"

	"strings"

	"github.com/apprenda/kismatic-provision/provision/plan"
	garbler "github.com/michaelbironneau/garbler/lib"
	"github.com/spf13/cobra"
)

type AWSOpts struct {
	EtcdNodeCount   uint16
	MasterNodeCount uint16
	WorkerNodeCount uint16
	LeaveArtifacts  bool
	RunKismatic     bool
	NoPlan          bool
	ForceProvision  bool
	KeyPairName     string
	InstanceType    string
	OS              string
}

func Cmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "aws",
		Short: "Provision infrastructure on AWS.",
		Long: `Provision infrastructure on AWS.

In addition to the commands below, AWS relies on some environment variables and conventions:
Required:
  AWS_ACCESS_KEY_ID: [Required] Your AWS access key, required for all operations
  AWS_SECRET_ACCESS_KEY: [Required] Your AWS secret key, required for all operations

Conditional: (These may be omitted if the -f flag is used)
  AWS_SUBNET_ID: The ID of a subnet to try to place machines into. If this environment variable exists,
                 it must be a real subnet in the us-east-1 region or all commands will fail.
  AWS_SECURITY_GROUP_ID: The ID of a security group to place all new machines in. Must be a part of the
                         above subnet or commands will fail.
  AWS_KEY_NAME: The name of a Keypair in AWS to be used to create machines. If empty, we will attempt
                to use a key named 'kismatic-integration-testing' and fail if it does not exist.
  AWS_SSH_KEY_PATH: The absolute path to the private key associated with the Key Name above. If left blank,
                    we will attempt to use a key named 'kismaticuser.key' in the same directory as the
					provision tool. This key is important as part of provisioning is ensuring that your
					instance is online and is able to be reached via SSH.
`,
	}

	cmd.AddCommand(AWSCreateCmd())
	cmd.AddCommand(AWSCreateMinikubeCmd())
	cmd.AddCommand(AWSDeleteCmd())

	return cmd
}

func AWSCreateCmd() *cobra.Command {
	opts := AWSOpts{}
	cmd := &cobra.Command{
		Use:   "create",
		Short: "Creates infrastructure for a new cluster. For now, only the US East region is supported.",
		Long: `Creates infrastructure for a new cluster.

For now, only the US East region is supported.

Smallish instances will be created with public IP addresses. The command will not return until the instances are all online and accessible via SSH.`,
		RunE: func(cmd *cobra.Command, args []string) error {
			return makeInfra(opts)
		},
	}

	cmd.Flags().Uint16VarP(&opts.EtcdNodeCount, "etcdNodeCount", "e", 1, "Count of etcd nodes to produce.")
	cmd.Flags().Uint16VarP(&opts.MasterNodeCount, "masterdNodeCount", "m", 1, "Count of master nodes to produce.")
	cmd.Flags().Uint16VarP(&opts.WorkerNodeCount, "workerNodeCount", "w", 1, "Count of worker nodes to produce.")
	cmd.Flags().BoolVarP(&opts.NoPlan, "noplan", "n", false, "If present, foregoes generating a plan file in this directory referencing the newly created nodes")
	cmd.Flags().BoolVarP(&opts.ForceProvision, "force-provision", "f", false, "If present, generate anything needed to build a cluster including VPCs, keypairs, routes, subnets, & a very insecure security group.")
	cmd.Flags().StringVarP(&opts.InstanceType, "instance-type-blueprint", "i", "small", "A blueprint of instance type(s). Current options: micro (all t2 micros), small (t2 micros, workers are t2.medium), beefy (M4.large and xlarge)")
	cmd.Flags().StringVarP(&opts.OS, "operating system", "o", "ubuntu", "Which flavor of Linux to provision. Try ubuntu, centos or rhel.")

	return cmd
}

func AWSCreateMinikubeCmd() *cobra.Command {
	opts := AWSOpts{}
	cmd := &cobra.Command{
		Use:   "create-mini",
		Short: "Creates infrastructure for a single-node instance. For now, only the US East region is supported.",
		Long: `Creates infrastructure for a single-node instance.

For now, only the US East region is supported.

A smallish instance will be created with public IP addresses. The command will not return until the instance is online and accessible via SSH.`,
		RunE: func(cmd *cobra.Command, args []string) error {
			return makeInfraMinikube(opts)
		},
	}

	cmd.Flags().StringVarP(&opts.OS, "operating system", "o", "ubuntu", "Which flavor of Linux to provision. Try ubuntu, centos or rhel.")
	cmd.Flags().BoolVarP(&opts.NoPlan, "noplan", "n", false, "If present, foregoes generating a plan file in this directory referencing the newly created nodes")
	cmd.Flags().BoolVarP(&opts.ForceProvision, "force-provision", "f", false, "If present, generate anything needed to build a cluster including VPCs, keypairs, routes, subnets, & a very insecure security group.")
	cmd.Flags().StringVarP(&opts.InstanceType, "instance-type-blueprint", "i", "small", "A blueprint of instance type(s). Current options: micro (all t2 micros), small (t2 micros, workers are t2.medium), beefy (M4.large and xlarge)")

	return cmd
}

func AWSDeleteCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "delete-all",
		Short: "Deletes all objects tagged as created by this machine with this tool. This will destroy clusters. Be ready.",
		Long: `Deletes all objects tagged as CreatedBy this machine and ProvisionedBy kismatic.

This command destroys clusters.

It has no way of knowing that you had really important data on them. It is utterly remorseless.

Be ready.`,
		RunE: func(cmd *cobra.Command, args []string) error {
			return deleteInfra()
		},
	}

	return cmd
}

func checkAWSCredentials() error {
	c := CompositeError{}
	accessKeyID := os.Getenv("AWS_ACCESS_KEY_ID")
	if accessKeyID == "" {
		c.add(errors.New("Need AWS_ACCESS_KEY_ID env variable set to perform any AWS operations"))
	}
	secretAccessKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
	if secretAccessKey == "" {
		c.add(errors.New("Need AWS_SECRET_ACCESS_KEY env variable set to perform any AWS operations"))
	}
	if c.hasError() {
		return c
	}
	return nil
}

// An Error made up of many contributing errors that all have equal weight (e.g. do not form a stack)
type CompositeError struct {
	e []error
}

func (c CompositeError) Error() string {
	ret := ""
	for _, e := range c.e {
		ret = ret + fmt.Sprintf(" - %v\n", e)
	}
	return ret
}

func (c *CompositeError) add(woe error) {
	c.e = append(c.e, woe)
}

func (c *CompositeError) merge(c2 CompositeError) {
	for _, e := range c2.e {
		c.e = append(c.e, e)
	}
}

func (c *CompositeError) hasError() bool {
	return len(c.e) > 0
}

func checkAWSDeploymentEnvironment() error {
	c := CompositeError{}
	if os.Getenv("AWS_SUBNET_ID") == "" {
		c.add(errors.New("Need AWS_SUBNET_ID env variable set to perform this AWS operations"))
	}
	if os.Getenv("AWS_SECURITY_GROUP_ID") == "" {
		c.add(errors.New("Need AWS_SECURITY_GROUP_ID env variable set to perform this AWS operations"))
	}

	if c.hasError() {
		return c
	}
	return nil
}

func deleteInfra() error {
	if err := checkAWSCredentials(); err != nil {
		return err
	}

	awsClient, _ := AWSClientFromEnvironment()

	return awsClient.TerminateAllNodes()
}

func prepareToModifyAWS(forceProvision bool) error {
	if err := checkAWSCredentials(); err != nil {
		return err
	}

	awsClient, _ := AWSClientFromEnvironment()

	fmt.Printf("Using region %v\n", awsClient.client.Config.Region)

	if forceProvision {
		if err := awsClient.ForceProvision(); err != nil {
			return err
		}
	}

	if err := checkAWSDeploymentEnvironment(); err != nil {
		return err
	}

	s, err := os.Stat(awsClient.sshKey)
	if os.IsNotExist(err) {
		return err
	}

	if s.Mode().Perm()&0044 != 0000 {

		return fmt.Errorf("Set permissions of %v to 0600", awsClient.sshKey)
	}

	return nil
}

func assertOptions(opts AWSOpts) (NodeBlueprint, LinuxDistro, error) {
	blueprint, ok := NodeBlueprintMap[opts.InstanceType]
	if !ok {
		return NodeBlueprint{}, "", fmt.Errorf("%v is not valid option for instance type blueprint.", opts.InstanceType)
	}
	if err := prepareToModifyAWS(opts.ForceProvision); err != nil {
		return NodeBlueprint{}, "", err
	}

	distro := Ubuntu1604LTS
	switch strings.ToLower(opts.OS) {
	case "centos":
		distro = CentOS7
	case "ubuntu":
		distro = Ubuntu1604LTS
	case "rhel":
		distro = Redhat7
	default:
		return NodeBlueprint{}, "", fmt.Errorf("%v is not a known option for OS")
	}
	return blueprint, distro, nil
}

func makeInfraMinikube(opts AWSOpts) error {
	blueprint, distro, err := assertOptions(opts)
	if err != nil {
		return err
	}

	fmt.Print("Provisioning")
	awsClient, _ := AWSClientFromEnvironment()
	nodes, err := awsClient.ProvisionNodes(blueprint, NodeCount{
		Worker: 1,
	}, distro)

	if err != nil {
		return err
	}

	sshKey := awsClient.SSHKey()
	fmt.Print("Waiting for SSH")
	if err = WaitForSSH(nodes, sshKey); err != nil {
		return err
	}

	if opts.NoPlan {
		fmt.Println("Your instances are ready.\n")
		printRole("Minikube", &nodes.Worker)
	} else {
		return makePlan(&plan.Plan{
			AdminPassword:       generateAlphaNumericPassword(),
			Etcd:                []plan.Node{nodes.Worker[0]},
			Master:              []plan.Node{nodes.Worker[0]},
			Worker:              []plan.Node{nodes.Worker[0]},
			Ingress:             []plan.Node{nodes.Worker[0]},
			MasterNodeFQDN:      nodes.Worker[0].PublicIPv4,
			MasterNodeShortName: nodes.Worker[0].PrivateIPv4,
			SSHKeyFile:          sshKey,
			SSHUser:             nodes.Worker[0].SSHUser,
		})
	}
	return nil
}

func makeInfra(opts AWSOpts) error {
	blueprint, distro, err := assertOptions(opts)
	if err != nil {
		return err
	}

	fmt.Print("Provisioning")
	awsClient, _ := AWSClientFromEnvironment()
	nodes, err := awsClient.ProvisionNodes(blueprint, NodeCount{
		Etcd:   opts.EtcdNodeCount,
		Worker: opts.WorkerNodeCount,
		Master: opts.MasterNodeCount,
	}, distro)

	if err != nil {
		return err
	}

	sshKey := awsClient.SSHKey()
	fmt.Print("Waiting for SSH")
	if err = WaitForSSH(nodes, sshKey); err != nil {
		return err
	}

	if opts.NoPlan {
		fmt.Println("Your instances are ready.\n")
		printNodes(&nodes)
	} else {
		return makePlan(&plan.Plan{
			AdminPassword:       generateAlphaNumericPassword(),
			Etcd:                nodes.Etcd,
			Master:              nodes.Master,
			Worker:              nodes.Worker,
			Ingress:             []plan.Node{nodes.Worker[0]},
			MasterNodeFQDN:      nodes.Master[0].PublicIPv4,
			MasterNodeShortName: nodes.Master[0].PrivateIPv4,
			SSHKeyFile:          sshKey,
			SSHUser:             nodes.Master[0].SSHUser,
		})
	}
	return nil
}

func makePlan(pln *plan.Plan) error {
	template, err := template.New("planAWSOverlay").Parse(plan.OverlayNetworkPlan)
	if err != nil {
		return err
	}

	f, err := makeUniqueFile(0)
	if err != nil {
		return err
	}

	defer f.Close()
	w := bufio.NewWriter(f)

	if err = template.Execute(w, &pln); err != nil {
		return err
	}

	w.Flush()
	fmt.Println("To install your cluster, run:")
	fmt.Println("./kismatic install apply -f " + f.Name())

	return nil
}

func makeUniqueFile(count int) (*os.File, error) {
	filename := "kismatic-cluster"
	if count > 0 {
		filename = filename + "-" + strconv.Itoa(count)
	}
	filename = filename + ".yaml"

	if _, err := os.Stat(filename); os.IsNotExist(err) {
		return os.Create(filename)
	} else {
		return makeUniqueFile(count + 1)
	}
}

func printNodes(nodes *ProvisionedNodes) {
	printRole("Etcd", &nodes.Etcd)
	printRole("Master", &nodes.Master)
	printRole("Worker", &nodes.Worker)
}

func printRole(title string, nodes *[]plan.Node) {
	fmt.Printf("%v:\n", title)
	for _, node := range *nodes {
		fmt.Printf("  %v (%v, %v)\n", node.ID, node.PublicIPv4, node.PrivateIPv4)
	}
}

func generateAlphaNumericPassword() string {
	attempts := 0
	for {
		reqs := &garbler.PasswordStrengthRequirements{
			MinimumTotalLength: 16,
			Uppercase:          rand.Intn(6),
			Digits:             rand.Intn(6),
			Punctuation:        -1, // disable punctuation
		}
		pass, err := garbler.NewPassword(reqs)
		if err != nil {
			return "weakpassword"
		}
		// validate that the library actually returned an alphanumeric password
		re := regexp.MustCompile("^[a-zA-Z1-9]+$")
		if re.MatchString(pass) {
			return pass
		}
		if attempts == 50 {
			return "weakpassword"
		}
		attempts++
	}
}
<commit_msg>Fix aws error msg<commit_after>package aws

import (
	"bufio"
	"errors"
	"fmt"
	"html/template"
	"math/rand"
	"os"
	"regexp"
	"strconv"

	"strings"

	"github.com/apprenda/kismatic-provision/provision/plan"
	garbler "github.com/michaelbironneau/garbler/lib"
	"github.com/spf13/cobra"
)

type AWSOpts struct {
	EtcdNodeCount   uint16
	MasterNodeCount uint16
	WorkerNodeCount uint16
	LeaveArtifacts  bool
	RunKismatic     bool
	NoPlan          bool
	ForceProvision  bool
	KeyPairName     string
	InstanceType    string
	OS              string
}

func Cmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "aws",
		Short: "Provision infrastructure on AWS.",
		Long: `Provision infrastructure on AWS.

In addition to the commands below, AWS relies on some environment variables and conventions:
Required:
  AWS_ACCESS_KEY_ID: [Required] Your AWS access key, required for all operations
  AWS_SECRET_ACCESS_KEY: [Required] Your AWS secret key, required for all operations

Conditional: (These may be omitted if the -f flag is used)
  AWS_SUBNET_ID: The ID of a subnet to try to place machines into. If this environment variable exists,
                 it must be a real subnet in the us-east-1 region or all commands will fail.
  AWS_SECURITY_GROUP_ID: The ID of a security group to place all new machines in. Must be a part of the
                         above subnet or commands will fail.
  AWS_KEY_NAME: The name of a Keypair in AWS to be used to create machines. If empty, we will attempt
                to use a key named 'kismatic-integration-testing' and fail if it does not exist.
  AWS_SSH_KEY_PATH: The absolute path to the private key associated with the Key Name above. If left blank,
                    we will attempt to use a key named 'kismaticuser.key' in the same directory as the
					provision tool. This key is important as part of provisioning is ensuring that your
					instance is online and is able to be reached via SSH.
`,
	}

	cmd.AddCommand(AWSCreateCmd())
	cmd.AddCommand(AWSCreateMinikubeCmd())
	cmd.AddCommand(AWSDeleteCmd())

	return cmd
}

func AWSCreateCmd() *cobra.Command {
	opts := AWSOpts{}
	cmd := &cobra.Command{
		Use:   "create",
		Short: "Creates infrastructure for a new cluster. For now, only the US East region is supported.",
		Long: `Creates infrastructure for a new cluster.

For now, only the US East region is supported.

Smallish instances will be created with public IP addresses. The command will not return until the instances are all online and accessible via SSH.`,
		RunE: func(cmd *cobra.Command, args []string) error {
			return makeInfra(opts)
		},
	}

	cmd.Flags().Uint16VarP(&opts.EtcdNodeCount, "etcdNodeCount", "e", 1, "Count of etcd nodes to produce.")
	cmd.Flags().Uint16VarP(&opts.MasterNodeCount, "masterdNodeCount", "m", 1, "Count of master nodes to produce.")
	cmd.Flags().Uint16VarP(&opts.WorkerNodeCount, "workerNodeCount", "w", 1, "Count of worker nodes to produce.")
	cmd.Flags().BoolVarP(&opts.NoPlan, "noplan", "n", false, "If present, foregoes generating a plan file in this directory referencing the newly created nodes")
	cmd.Flags().BoolVarP(&opts.ForceProvision, "force-provision", "f", false, "If present, generate anything needed to build a cluster including VPCs, keypairs, routes, subnets, & a very insecure security group.")
	cmd.Flags().StringVarP(&opts.InstanceType, "instance-type-blueprint", "i", "small", "A blueprint of instance type(s). Current options: micro (all t2 micros), small (t2 micros, workers are t2.medium), beefy (M4.large and xlarge)")
	cmd.Flags().StringVarP(&opts.OS, "operating system", "o", "ubuntu", "Which flavor of Linux to provision. Try ubuntu, centos or rhel.")

	return cmd
}

func AWSCreateMinikubeCmd() *cobra.Command {
	opts := AWSOpts{}
	cmd := &cobra.Command{
		Use:   "create-mini",
		Short: "Creates infrastructure for a single-node instance. For now, only the US East region is supported.",
		Long: `Creates infrastructure for a single-node instance.

For now, only the US East region is supported.

A smallish instance will be created with public IP addresses. The command will not return until the instance is online and accessible via SSH.`,
		RunE: func(cmd *cobra.Command, args []string) error {
			return makeInfraMinikube(opts)
		},
	}

	cmd.Flags().StringVarP(&opts.OS, "operating system", "o", "ubuntu", "Which flavor of Linux to provision. Try ubuntu, centos or rhel.")
	cmd.Flags().BoolVarP(&opts.NoPlan, "noplan", "n", false, "If present, foregoes generating a plan file in this directory referencing the newly created nodes")
	cmd.Flags().BoolVarP(&opts.ForceProvision, "force-provision", "f", false, "If present, generate anything needed to build a cluster including VPCs, keypairs, routes, subnets, & a very insecure security group.")
	cmd.Flags().StringVarP(&opts.InstanceType, "instance-type-blueprint", "i", "small", "A blueprint of instance type(s). Current options: micro (all t2 micros), small (t2 micros, workers are t2.medium), beefy (M4.large and xlarge)")

	return cmd
}

func AWSDeleteCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "delete-all",
		Short: "Deletes all objects tagged as created by this machine with this tool. This will destroy clusters. Be ready.",
		Long: `Deletes all objects tagged as CreatedBy this machine and ProvisionedBy kismatic.

This command destroys clusters.

It has no way of knowing that you had really important data on them. It is utterly remorseless.

Be ready.`,
		RunE: func(cmd *cobra.Command, args []string) error {
			return deleteInfra()
		},
	}

	return cmd
}

func checkAWSCredentials() error {
	c := CompositeError{}
	accessKeyID := os.Getenv("AWS_ACCESS_KEY_ID")
	if accessKeyID == "" {
		c.add(errors.New("Need AWS_ACCESS_KEY_ID env variable set to perform any AWS operations"))
	}
	secretAccessKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
	if secretAccessKey == "" {
		c.add(errors.New("Need AWS_SECRET_ACCESS_KEY env variable set to perform any AWS operations"))
	}
	if c.hasError() {
		return c
	}
	return nil
}

// An Error made up of many contributing errors that all have equal weight (e.g. do not form a stack)
type CompositeError struct {
	e []error
}

func (c CompositeError) Error() string {
	ret := ""
	for _, e := range c.e {
		ret = ret + fmt.Sprintf(" - %v\n", e)
	}
	return ret
}

func (c *CompositeError) add(woe error) {
	c.e = append(c.e, woe)
}

func (c *CompositeError) merge(c2 CompositeError) {
	for _, e := range c2.e {
		c.e = append(c.e, e)
	}
}

func (c *CompositeError) hasError() bool {
	return len(c.e) > 0
}

func checkAWSDeploymentEnvironment() error {
	c := CompositeError{}
	if os.Getenv("AWS_SUBNET_ID") == "" {
		c.add(errors.New("Need AWS_SUBNET_ID env variable set to perform this AWS operations"))
	}
	if os.Getenv("AWS_SECURITY_GROUP_ID") == "" {
		c.add(errors.New("Need AWS_SECURITY_GROUP_ID env variable set to perform this AWS operations"))
	}

	if c.hasError() {
		return c
	}
	return nil
}

func deleteInfra() error {
	if err := checkAWSCredentials(); err != nil {
		return err
	}

	awsClient, _ := AWSClientFromEnvironment()

	return awsClient.TerminateAllNodes()
}

func prepareToModifyAWS(forceProvision bool) error {
	if err := checkAWSCredentials(); err != nil {
		return err
	}

	awsClient, _ := AWSClientFromEnvironment()

	fmt.Printf("Using region %v\n", awsClient.client.Config.Region)

	if forceProvision {
		if err := awsClient.ForceProvision(); err != nil {
			return err
		}
	}

	if err := checkAWSDeploymentEnvironment(); err != nil {
		return err
	}

	s, err := os.Stat(awsClient.sshKey)
	if os.IsNotExist(err) {
		return err
	}

	if s.Mode().Perm()&0044 != 0000 {

		return fmt.Errorf("Set permissions of %v to 0600", awsClient.sshKey)
	}

	return nil
}

func assertOptions(opts AWSOpts) (NodeBlueprint, LinuxDistro, error) {
	blueprint, ok := NodeBlueprintMap[opts.InstanceType]
	if !ok {
		return NodeBlueprint{}, "", fmt.Errorf("%v is not valid option for instance type blueprint.", opts.InstanceType)
	}
	if err := prepareToModifyAWS(opts.ForceProvision); err != nil {
		return NodeBlueprint{}, "", err
	}

	distro := Ubuntu1604LTS
	switch strings.ToLower(opts.OS) {
	case "centos":
		distro = CentOS7
	case "ubuntu":
		distro = Ubuntu1604LTS
	case "rhel":
		distro = Redhat7
	default:
		return NodeBlueprint{}, "", fmt.Errorf("%s is not a known option for OS", opts.OS)
	}
	return blueprint, distro, nil
}

func makeInfraMinikube(opts AWSOpts) error {
	blueprint, distro, err := assertOptions(opts)
	if err != nil {
		return err
	}

	fmt.Print("Provisioning")
	awsClient, _ := AWSClientFromEnvironment()
	nodes, err := awsClient.ProvisionNodes(blueprint, NodeCount{
		Worker: 1,
	}, distro)

	if err != nil {
		return err
	}

	sshKey := awsClient.SSHKey()
	fmt.Print("Waiting for SSH")
	if err = WaitForSSH(nodes, sshKey); err != nil {
		return err
	}

	if opts.NoPlan {
		fmt.Println("Your instances are ready.\n")
		printRole("Minikube", &nodes.Worker)
	} else {
		return makePlan(&plan.Plan{
			AdminPassword:       generateAlphaNumericPassword(),
			Etcd:                []plan.Node{nodes.Worker[0]},
			Master:              []plan.Node{nodes.Worker[0]},
			Worker:              []plan.Node{nodes.Worker[0]},
			Ingress:             []plan.Node{nodes.Worker[0]},
			MasterNodeFQDN:      nodes.Worker[0].PublicIPv4,
			MasterNodeShortName: nodes.Worker[0].PrivateIPv4,
			SSHKeyFile:          sshKey,
			SSHUser:             nodes.Worker[0].SSHUser,
		})
	}
	return nil
}

func makeInfra(opts AWSOpts) error {
	blueprint, distro, err := assertOptions(opts)
	if err != nil {
		return err
	}

	fmt.Print("Provisioning")
	awsClient, _ := AWSClientFromEnvironment()
	nodes, err := awsClient.ProvisionNodes(blueprint, NodeCount{
		Etcd:   opts.EtcdNodeCount,
		Worker: opts.WorkerNodeCount,
		Master: opts.MasterNodeCount,
	}, distro)

	if err != nil {
		return err
	}

	sshKey := awsClient.SSHKey()
	fmt.Print("Waiting for SSH")
	if err = WaitForSSH(nodes, sshKey); err != nil {
		return err
	}

	if opts.NoPlan {
		fmt.Println("Your instances are ready.\n")
		printNodes(&nodes)
	} else {
		return makePlan(&plan.Plan{
			AdminPassword:       generateAlphaNumericPassword(),
			Etcd:                nodes.Etcd,
			Master:              nodes.Master,
			Worker:              nodes.Worker,
			Ingress:             []plan.Node{nodes.Worker[0]},
			MasterNodeFQDN:      nodes.Master[0].PublicIPv4,
			MasterNodeShortName: nodes.Master[0].PrivateIPv4,
			SSHKeyFile:          sshKey,
			SSHUser:             nodes.Master[0].SSHUser,
		})
	}
	return nil
}

func makePlan(pln *plan.Plan) error {
	template, err := template.New("planAWSOverlay").Parse(plan.OverlayNetworkPlan)
	if err != nil {
		return err
	}

	f, err := makeUniqueFile(0)
	if err != nil {
		return err
	}

	defer f.Close()
	w := bufio.NewWriter(f)

	if err = template.Execute(w, &pln); err != nil {
		return err
	}

	w.Flush()
	fmt.Println("To install your cluster, run:")
	fmt.Println("./kismatic install apply -f " + f.Name())

	return nil
}

func makeUniqueFile(count int) (*os.File, error) {
	filename := "kismatic-cluster"
	if count > 0 {
		filename = filename + "-" + strconv.Itoa(count)
	}
	filename = filename + ".yaml"

	if _, err := os.Stat(filename); os.IsNotExist(err) {
		return os.Create(filename)
	} else {
		return makeUniqueFile(count + 1)
	}
}

func printNodes(nodes *ProvisionedNodes) {
	printRole("Etcd", &nodes.Etcd)
	printRole("Master", &nodes.Master)
	printRole("Worker", &nodes.Worker)
}

func printRole(title string, nodes *[]plan.Node) {
	fmt.Printf("%v:\n", title)
	for _, node := range *nodes {
		fmt.Printf("  %v (%v, %v)\n", node.ID, node.PublicIPv4, node.PrivateIPv4)
	}
}

func generateAlphaNumericPassword() string {
	attempts := 0
	for {
		reqs := &garbler.PasswordStrengthRequirements{
			MinimumTotalLength: 16,
			Uppercase:          rand.Intn(6),
			Digits:             rand.Intn(6),
			Punctuation:        -1, // disable punctuation
		}
		pass, err := garbler.NewPassword(reqs)
		if err != nil {
			return "weakpassword"
		}
		// validate that the library actually returned an alphanumeric password
		re := regexp.MustCompile("^[a-zA-Z1-9]+$")
		if re.MatchString(pass) {
			return pass
		}
		if attempts == 50 {
			return "weakpassword"
		}
		attempts++
	}
}
<|endoftext|>


<commit_before>package util

import "fmt"

const (
	// OtherError -- Other (maybe non HTTP-related) error
	OtherError = 0
	// BadRequestError -- Indicates an invalid request
	BadRequestError = 400
	// AuthenticationError -- Indicates invalid authentication credentials
	AuthenticationError = 401
	// ForbiddenError -- You're not allowed to access that resource
	ForbiddenError = 403
	// NotFoundError -- The resource that's been requested doesn't exist
	NotFoundError = 404
)

// APIError -- Custom error interface
type APIError interface {
	error
	Code() int
}

type apiErrorImpl struct {
	msg  string
	code int
}

func (e *apiErrorImpl) Error() string {
	return e.msg
}

func (e *apiErrorImpl) Code() int {
	return e.code
}

// NewAPIError -- Create an APIError instance
func NewAPIError(code int, msg ...interface{}) APIError {
	return &apiErrorImpl{code: code, msg: fmt.Sprint(msg...)}
}
<commit_msg>util/errors.go: using http.Status* instead of integer constants for error codes where possible, added 429 error code<commit_after>package util

import (
	"fmt"
	"net/http"
)

const (
	// OtherError -- Other (maybe non HTTP-related) error
	OtherError = 0
	// BadRequestError -- Indicates an invalid request (400)
	BadRequestError = http.StatusBadRequest
	// AuthenticationError -- Indicates invalid authentication credentials (401)
	AuthenticationError = http.StatusUnauthorized
	// ForbiddenError -- You're not allowed to access that resource (403)
	ForbiddenError = http.StatusForbidden
	// NotFoundError -- The resource that's been requested doesn't exist (404)
	NotFoundError = http.StatusNotFound
	// TooManyRequestsError -- You have been rate limited (429)
	TooManyRequestsError = http.StatusTooManyRequests
)

// APIError -- Custom error interface
type APIError interface {
	error
	Code() int
}

type apiErrorImpl struct {
	msg  string
	code int
}

func (e *apiErrorImpl) Error() string {
	return e.msg
}

func (e *apiErrorImpl) Code() int {
	return e.code
}

// NewAPIError -- Create an APIError instance
func NewAPIError(code int, msg ...interface{}) APIError {
	return &apiErrorImpl{code: code, msg: fmt.Sprint(msg...)}
}
<|endoftext|>


<commit_before>package utils

import (
	"fmt"
	"github.com/Jeffail/gabs"
	"io/ioutil"
	"net/http"
)

var (
	// Unknown is returned when the city or country name can't be resolved.
	Unknown = "<unknown>"
)

// GeoIP contains information about the geolocalization.
type GeoIP struct {
	data *gabs.Container
}

// NewGeoIP returns a new instance of the Geoip struct
func NewGeoIP(ip string) (*GeoIP, error) {
	var data *gabs.Container
	var err error

	if ip == "::1" || ip == "127.0.0.1" {
		data, err = lookupMyIP()
	} else {
		data, err = lookupIP(ip)
	}

	if err != nil {
		return nil, err
	}

	geoip := &GeoIP{
		data: data,
	}

	return geoip, nil
}

// City returns the name of the city
func (geoip *GeoIP) City() string {
	if geoip.data == nil {
		return Unknown
	}

	city, ok := geoip.data.Search("city").Data().(string)
	if !ok {
		return Unknown
	}

	return city
}

// Country returns the name of the country
func (geoip *GeoIP) Country() string {
	if geoip.data == nil {
		return Unknown
	}

	country, ok := geoip.data.Search("country", "name").Data().(string)
	if !ok {
		return Unknown
	}

	return country
}

func lookupMyIP() (*gabs.Container, error) {
	response, err := http.Get("https://geoip.nekudo.com/api")
	if err != nil {
		return nil, err
	}
	return parseGeoIPResponse(response)
}

func lookupIP(ip string) (*gabs.Container, error) {
	response, err := http.Get(fmt.Sprintf("https://geoip.nekudo.com/api/%s", ip))
	if err != nil {
		return nil, err
	}

	return parseGeoIPResponse(response)
}

// FormatIPAndLocation returns the ip with geo location for the given ip.
func FormatIPAndLocation(ip string) string {
	geoip, err := NewGeoIP(ip)

	if err != nil {
		return fmt.Sprintf("%s (%s, %s)", ip, geoip.City(), geoip.Country())
	}

	return ip
}

func parseGeoIPResponse(response *http.Response) (*gabs.Container, error) {
	defer response.Body.Close()
	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}

	return gabs.ParseJSON(body)
}
<commit_msg>Add more strict check for geoip failures.<commit_after>package utils

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"

	"github.com/Jeffail/gabs"
)

var (
	// Unknown is returned when the city or country name can't be resolved.
	Unknown = "<unknown>"

	errInvalidGeoip = errors.New("invalid geoip data")
)

// GeoIP contains information about the geolocalization.
type GeoIP struct {
	data *gabs.Container
}

// NewGeoIP returns a new instance of the Geoip struct
func NewGeoIP(ip string) (*GeoIP, error) {
	var data *gabs.Container
	var err error

	if ip == "::1" || ip == "127.0.0.1" {
		data, err = lookupMyIP()
	} else {
		data, err = lookupIP(ip)
	}

	if err != nil {
		return nil, err
	}

	if data == nil {
		return nil, errInvalidGeoip
	}

	geoip := &GeoIP{
		data: data,
	}

	return geoip, nil
}

// City returns the name of the city
func (geoip *GeoIP) City() string {
	if geoip.data == nil {
		return Unknown
	}

	city, ok := geoip.data.Search("city").Data().(string)
	if !ok {
		return Unknown
	}

	return city
}

// Country returns the name of the country
func (geoip *GeoIP) Country() string {
	if geoip.data == nil {
		return Unknown
	}

	country, ok := geoip.data.Search("country", "name").Data().(string)
	if !ok {
		return Unknown
	}

	return country
}

func lookupMyIP() (*gabs.Container, error) {
	response, err := http.Get("https://geoip.nekudo.com/api")
	if err != nil {
		return nil, err
	}
	return parseGeoIPResponse(response)
}

func lookupIP(ip string) (*gabs.Container, error) {
	response, err := http.Get(fmt.Sprintf("https://geoip.nekudo.com/api/%s", ip))
	if err != nil {
		return nil, err
	}

	return parseGeoIPResponse(response)
}

// FormatIPAndLocation returns the ip with geo location for the given ip.
func FormatIPAndLocation(ip string) string {
	geoip, err := NewGeoIP(ip)

	if err != nil {
		return fmt.Sprintf("%s (%s, %s)", ip, geoip.City(), geoip.Country())
	}

	return ip
}

func parseGeoIPResponse(response *http.Response) (*gabs.Container, error) {
	defer response.Body.Close()
	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}

	return gabs.ParseJSON(body)
}
<|endoftext|>


<commit_before>package utils

import (
	"bytes"
	"crypto/md5"
	"crypto/sha1"
	"database/sql"
	"encoding/hex"
	"errors"
	"fmt"
	"image"
	// gif support
	_ "image/gif"
	// jpeg support
	_ "image/jpeg"
	// png support
	_ "image/png"
	"io"
	"mime/multipart"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	shortid "github.com/ventu-io/go-shortid"

	"github.com/eirka/eirka-libs/amazon"
	"github.com/eirka/eirka-libs/config"
	"github.com/eirka/eirka-libs/db"

	local "github.com/eirka/eirka-post/config"
)

// Allowed chars for filenames
const fileChars = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"

// valid file extensions
var validExt = map[string]bool{
	".jpg":  true,
	".jpeg": true,
	".png":  true,
	".gif":  true,
	".webm": true,
}

func init() {

	var err error

	// test for ImageMagick
	_, err = exec.Command("convert", "--version").Output()
	if err != nil {
		panic("ImageMagick not found")
	}

}

// FileUploader defines the file processing functions
type FileUploader interface {
	// struct integrity
	IsValid() bool
	IsValidPost() bool

	// image processing
	SaveImage() (err error)
	checkReqExt() (err error)
	copyBytes() (err error)
	getHash() (err error)
	checkBanned() (err error)
	checkDuplicate() (err error)
	checkMagic() (err error)
	getStats() (err error)
	saveFile() (err error)
	makeFilenames()
	createThumbnail(maxwidth, maxheight int) (err error)
	copyToS3() (err error)

	// webm specific functions
	checkWebM() (err error)
	createWebMThumbnail() (err error)

	// avatar functions
	SaveAvatar() (err error)
	avatarToS3() (err error)
}

// ImageType defines an image and its metadata for processing
type ImageType struct {
	File        multipart.File
	Header      *multipart.FileHeader
	Ib          uint
	Filename    string
	Thumbnail   string
	Filepath    string
	Thumbpath   string
	Ext         string
	MD5         string
	SHA         string
	OrigWidth   int
	OrigHeight  int
	ThumbWidth  int
	ThumbHeight int
	image       *bytes.Buffer
	mime        string
	duration    int
	video       bool
	avatar      bool
}

var _ = FileUploader(&ImageType{})

// IsValid will check struct integrity
func (i *ImageType) IsValid() bool {

	if i.Filename == "" {
		return false
	}

	if i.Filepath == "" {
		return false
	}

	if i.Thumbnail == "" {
		return false
	}

	if i.Thumbpath == "" {
		return false
	}

	if i.Ib == 0 {
		return false
	}

	if i.Ext == "" {
		return false
	}

	if i.MD5 == "" {
		return false
	}

	if i.SHA == "" {
		return false
	}

	if i.mime == "" {
		return false
	}

	return true
}

// IsValidPost will check final struct integrity
func (i *ImageType) IsValidPost() bool {
	if i.OrigWidth == 0 {
		return false
	}

	if i.OrigHeight == 0 {
		return false
	}

	if i.ThumbWidth == 0 {
		return false
	}

	if i.ThumbHeight == 0 {
		return false
	}

	return true
}

// SaveImage runs the entire file processing pipeline
func (i *ImageType) SaveImage() (err error) {

	// check given file ext
	err = i.checkReqExt()
	if err != nil {
		return
	}

	// copy the multipart file into a buffer
	err = i.copyBytes()
	if err != nil {
		return
	}

	// get file md5
	err = i.getHash()
	if err != nil {
		return
	}

	// check to see if the file is banned
	err = i.checkBanned()
	if err != nil {
		return
	}

	// check to see if the file already exists
	err = i.checkDuplicate()
	if err != nil {
		return
	}

	// check file magic sig
	err = i.checkMagic()
	if err != nil {
		return
	}

	// check image stats
	err = i.getStats()
	if err != nil {
		return
	}

	// save the file to disk
	err = i.saveFile()
	if err != nil {
		return
	}

	// process a webm
	if i.video {
		// check the webm info
		err = i.checkWebM()
		if err != nil {
			return
		}

		// create thumbnail from webm
		err = i.createWebMThumbnail()
		if err != nil {
			return
		}

	}

	// create a thumbnail
	err = i.createThumbnail(config.Settings.Limits.ThumbnailMaxWidth, config.Settings.Limits.ThumbnailMaxHeight)
	if err != nil {
		return
	}

	// copy the file to s3
	err = i.copyToS3()
	if err != nil {
		return
	}

	// check final state
	if !i.IsValidPost() {
		return errors.New("ImageType is not valid")
	}

	return

}

// Get file extension from request header
func (i *ImageType) checkReqExt() (err error) {
	// Get ext from request header
	name := i.Header.Filename
	ext := filepath.Ext(name)

	if ext == "" {
		return errors.New("No file extension")
	}

	// Check to see if extension is allowed
	if !isAllowedExt(ext) {
		return errors.New("Format not supported")
	}

	return

}

// Check if file ext allowed
func isAllowedExt(ext string) bool {
	return validExt[strings.ToLower(ext)]
}

// Copy the multipart file into a bytes buffer
func (i *ImageType) copyBytes() (err error) {
	defer i.File.Close()

	i.image = new(bytes.Buffer)

	// Save file and also read into hasher for md5
	_, err = io.Copy(i.image, i.File)
	if err != nil {
		return errors.New("Problem copying file to buffer")
	}

	return
}

// Get image MD5 and write file into buffer
func (i *ImageType) getHash() (err error) {

	hasher := md5.New()

	sha := sha1.New()

	// read into hasher for md5
	_, err = io.Copy(hasher, bytes.NewReader(i.image.Bytes()))
	if err != nil {
		return errors.New("Problem creating MD5 hash")
	}

	// Set md5sum from hasher
	i.MD5 = hex.EncodeToString(hasher.Sum(nil))

	// read into hasher for md5
	_, err = io.Copy(sha, bytes.NewReader(i.image.Bytes()))
	if err != nil {
		return errors.New("Problem creating SHA1 hash")
	}

	// Set sha1
	i.SHA = hex.EncodeToString(sha.Sum(nil))

	return

}

// check if the md5 is a banned file
func (i *ImageType) checkBanned() (err error) {

	// Get Database handle
	dbase, err := db.GetDb()
	if err != nil {
		return
	}

	if i.MD5 == "" {
		return errors.New("No hash set on file banned check")
	}

	var check bool

	err = dbase.QueryRow(`SELECT count(*) FROM banned_files WHERE ban_hash = ?`, i.MD5).Scan(&check)
	if err != nil {
		return
	}

	// return error if it exists
	if check {
		return fmt.Errorf("File is banned")
	}

	return
}

// check if the md5 is already in the database
func (i *ImageType) checkDuplicate() (err error) {

	// Get Database handle
	dbase, err := db.GetDb()
	if err != nil {
		return
	}

	if i.MD5 == "" {
		return errors.New("No hash set on duplicate check")
	}

	if i.Ib == 0 {
		return errors.New("No imageboard set on duplicate check")
	}

	var check bool
	var thread, post sql.NullInt64

	err = dbase.QueryRow(`select count(1),posts.post_num,threads.thread_id from threads
	LEFT JOIN posts on threads.thread_id = posts.thread_id
	LEFT JOIN images on posts.post_id = images.post_id
	WHERE image_hash = ? AND ib_id = ?`, i.MD5, i.Ib).Scan(&check, &post, &thread)
	if err != nil {
		return
	}

	// return error if it exists
	if check {
		return fmt.Errorf("Image has already been posted. Thread: %d Post: %d", thread.Int64, post.Int64)
	}

	return
}

func (i *ImageType) checkMagic() (err error) {

	// detect the mime type
	i.mime = http.DetectContentType(i.image.Bytes())

	switch i.mime {
	case "image/png":
		i.Ext = ".png"
	case "image/jpeg":
		i.Ext = ".jpg"
	case "image/gif":
		i.Ext = ".gif"
	case "video/webm":
		i.Ext = ".webm"
		i.video = true
	default:
		return errors.New("Unknown file type")
	}

	// Check to see if extension is allowed
	if !isAllowedExt(i.Ext) {
		return errors.New("Format not supported")
	}

	return

}

func (i *ImageType) getStats() (err error) {

	// skip if its a video since we cant decode it
	if i.video {
		return
	}

	// decode image config
	img, _, err := image.DecodeConfig(bytes.NewReader(i.image.Bytes()))
	if err != nil {
		return errors.New("Problem decoding image")
	}

	// set original width
	i.OrigWidth = img.Width
	// set original height
	i.OrigHeight = img.Height

	// Check against maximum sizes
	switch {
	case i.OrigWidth > config.Settings.Limits.ImageMaxWidth:
		return fmt.Errorf("Image width too large. Max: %dpx", config.Settings.Limits.ImageMaxWidth)
	case img.Width < config.Settings.Limits.ImageMinWidth:
		return fmt.Errorf("Image width too small. Min: %dpx", config.Settings.Limits.ImageMinWidth)
	case i.OrigHeight > config.Settings.Limits.ImageMaxHeight:
		return fmt.Errorf("Image height too large. Max: %dpx", config.Settings.Limits.ImageMaxHeight)
	case img.Height < config.Settings.Limits.ImageMinHeight:
		return fmt.Errorf("Image height too small. Min: %dpx", config.Settings.Limits.ImageMinHeight)
	case i.image.Len() > config.Settings.Limits.ImageMaxSize:
		return fmt.Errorf("Image filesize too large. Max: %dMB", (config.Settings.Limits.ImageMaxSize/1024)/1024)
	}

	return

}

func (i *ImageType) saveFile() (err error) {

	defer i.image.Reset()

	i.makeFilenames()

	// avatar filename is the users id
	if i.avatar {
		i.Thumbnail = fmt.Sprintf("%d.png", i.Ib)
		i.Thumbpath = filepath.Join(local.Settings.Directories.ThumbnailDir, i.Thumbnail)
	}

	if !i.IsValid() {
		return errors.New("ImageType is not valid")
	}

	imagefile := filepath.Join(local.Settings.Directories.ImageDir, i.Filename)

	image, err := os.Create(imagefile)
	if err != nil {
		return errors.New("Problem saving file")
	}
	defer image.Close()

	_, err = io.Copy(image, bytes.NewReader(i.image.Bytes()))
	if err != nil {
		return errors.New("Problem saving file")
	}

	return

}

// Make a random unix time filename
func (i *ImageType) makeFilenames() {

	// get new short id generator
	sid := shortid.MustNew(1, fileChars, 9001)

	// generate filename
	filename := sid.MustGenerate()

	// Append ext to filename
	i.Filename = fmt.Sprintf("%s%s", filename, i.Ext)

	// Append jpg to thumbnail name because it is always a jpg
	i.Thumbnail = fmt.Sprintf("%ss.jpg", filename)

	// set the full file path
	i.Filepath = filepath.Join(local.Settings.Directories.ImageDir, i.Filename)

	// set the full thumbnail path
	i.Thumbpath = filepath.Join(local.Settings.Directories.ThumbnailDir, i.Thumbnail)

	return

}

func (i *ImageType) createThumbnail(maxwidth, maxheight int) (err error) {

	var imagef string

	if i.video {
		imagef = fmt.Sprintf("%s[0]", i.Thumbpath)
	} else {
		imagef = fmt.Sprintf("%s[0]", i.Filepath)
	}

	originalDimensions := fmt.Sprintf("%dx%d", i.OrigWidth, i.OrigHeight)

	var args []string

	// different options for avatars
	if i.avatar {
		args = []string{
			"-size",
			originalDimensions,
			imagef,
			"-background",
			"none",
			"-thumbnail",
			fmt.Sprintf("%dx%d^", maxwidth, maxheight),
			"-gravity",
			"center",
			"-extent",
			fmt.Sprintf("%dx%d", maxwidth, maxheight),
			i.Thumbpath,
		}
	} else {
		args = []string{
			"-background",
			"white",
			"-flatten",
			"-size",
			originalDimensions,
			"-resize",
			fmt.Sprintf("%dx%d>", maxwidth, maxheight),
			"-quality",
			"90",
			imagef,
			i.Thumbpath,
		}
	}

	_, err = exec.Command("convert", args...).Output()
	if err != nil {
		return errors.New("Problem making thumbnail")
	}

	thumb, err := os.Open(i.Thumbpath)
	if err != nil {
		return errors.New("Problem creating thumbnail file")
	}
	defer thumb.Close()

	img, _, err := image.DecodeConfig(thumb)
	if err != nil {
		return errors.New("Problem decoding thumbnail")
	}

	i.ThumbWidth = img.Width
	i.ThumbHeight = img.Height

	return

}

func (i *ImageType) copyToS3() (err error) {

	s3 := amazon.New()

	err = s3.Save(i.Filepath, fmt.Sprintf("src/%s", i.Filename), i.mime, false)
	if err != nil {
		return
	}

	err = s3.Save(i.Thumbpath, fmt.Sprintf("thumb/%s", i.Thumbnail), "image/jpeg", false)
	if err != nil {
		return
	}

	return
}
<commit_msg>try new filename generator<commit_after>package utils

import (
	"bytes"
	"crypto/md5"
	"crypto/sha1"
	"database/sql"
	"encoding/hex"
	"errors"
	"fmt"
	"image"
	// gif support
	_ "image/gif"
	// jpeg support
	_ "image/jpeg"
	// png support
	_ "image/png"
	"io"
	"mime/multipart"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	shortid "github.com/ventu-io/go-shortid"

	"github.com/eirka/eirka-libs/amazon"
	"github.com/eirka/eirka-libs/config"
	"github.com/eirka/eirka-libs/db"

	local "github.com/eirka/eirka-post/config"
)

// valid file extensions
var validExt = map[string]bool{
	".jpg":  true,
	".jpeg": true,
	".png":  true,
	".gif":  true,
	".webm": true,
}

func init() {

	var err error

	// test for ImageMagick
	_, err = exec.Command("convert", "--version").Output()
	if err != nil {
		panic("ImageMagick not found")
	}

}

// FileUploader defines the file processing functions
type FileUploader interface {
	// struct integrity
	IsValid() bool
	IsValidPost() bool

	// image processing
	SaveImage() (err error)
	checkReqExt() (err error)
	copyBytes() (err error)
	getHash() (err error)
	checkBanned() (err error)
	checkDuplicate() (err error)
	checkMagic() (err error)
	getStats() (err error)
	saveFile() (err error)
	makeFilenames()
	createThumbnail(maxwidth, maxheight int) (err error)
	copyToS3() (err error)

	// webm specific functions
	checkWebM() (err error)
	createWebMThumbnail() (err error)

	// avatar functions
	SaveAvatar() (err error)
	avatarToS3() (err error)
}

// ImageType defines an image and its metadata for processing
type ImageType struct {
	File        multipart.File
	Header      *multipart.FileHeader
	Ib          uint
	Filename    string
	Thumbnail   string
	Filepath    string
	Thumbpath   string
	Ext         string
	MD5         string
	SHA         string
	OrigWidth   int
	OrigHeight  int
	ThumbWidth  int
	ThumbHeight int
	image       *bytes.Buffer
	mime        string
	duration    int
	video       bool
	avatar      bool
}

var _ = FileUploader(&ImageType{})

// IsValid will check struct integrity
func (i *ImageType) IsValid() bool {

	if i.Filename == "" {
		return false
	}

	if i.Filepath == "" {
		return false
	}

	if i.Thumbnail == "" {
		return false
	}

	if i.Thumbpath == "" {
		return false
	}

	if i.Ib == 0 {
		return false
	}

	if i.Ext == "" {
		return false
	}

	if i.MD5 == "" {
		return false
	}

	if i.SHA == "" {
		return false
	}

	if i.mime == "" {
		return false
	}

	return true
}

// IsValidPost will check final struct integrity
func (i *ImageType) IsValidPost() bool {
	if i.OrigWidth == 0 {
		return false
	}

	if i.OrigHeight == 0 {
		return false
	}

	if i.ThumbWidth == 0 {
		return false
	}

	if i.ThumbHeight == 0 {
		return false
	}

	return true
}

// SaveImage runs the entire file processing pipeline
func (i *ImageType) SaveImage() (err error) {

	// check given file ext
	err = i.checkReqExt()
	if err != nil {
		return
	}

	// copy the multipart file into a buffer
	err = i.copyBytes()
	if err != nil {
		return
	}

	// get file md5
	err = i.getHash()
	if err != nil {
		return
	}

	// check to see if the file is banned
	err = i.checkBanned()
	if err != nil {
		return
	}

	// check to see if the file already exists
	err = i.checkDuplicate()
	if err != nil {
		return
	}

	// check file magic sig
	err = i.checkMagic()
	if err != nil {
		return
	}

	// check image stats
	err = i.getStats()
	if err != nil {
		return
	}

	// save the file to disk
	err = i.saveFile()
	if err != nil {
		return
	}

	// process a webm
	if i.video {
		// check the webm info
		err = i.checkWebM()
		if err != nil {
			return
		}

		// create thumbnail from webm
		err = i.createWebMThumbnail()
		if err != nil {
			return
		}

	}

	// create a thumbnail
	err = i.createThumbnail(config.Settings.Limits.ThumbnailMaxWidth, config.Settings.Limits.ThumbnailMaxHeight)
	if err != nil {
		return
	}

	// copy the file to s3
	err = i.copyToS3()
	if err != nil {
		return
	}

	// check final state
	if !i.IsValidPost() {
		return errors.New("ImageType is not valid")
	}

	return

}

// Get file extension from request header
func (i *ImageType) checkReqExt() (err error) {
	// Get ext from request header
	name := i.Header.Filename
	ext := filepath.Ext(name)

	if ext == "" {
		return errors.New("No file extension")
	}

	// Check to see if extension is allowed
	if !isAllowedExt(ext) {
		return errors.New("Format not supported")
	}

	return

}

// Check if file ext allowed
func isAllowedExt(ext string) bool {
	return validExt[strings.ToLower(ext)]
}

// Copy the multipart file into a bytes buffer
func (i *ImageType) copyBytes() (err error) {
	defer i.File.Close()

	i.image = new(bytes.Buffer)

	// Save file and also read into hasher for md5
	_, err = io.Copy(i.image, i.File)
	if err != nil {
		return errors.New("Problem copying file to buffer")
	}

	return
}

// Get image MD5 and write file into buffer
func (i *ImageType) getHash() (err error) {

	hasher := md5.New()

	sha := sha1.New()

	// read into hasher for md5
	_, err = io.Copy(hasher, bytes.NewReader(i.image.Bytes()))
	if err != nil {
		return errors.New("Problem creating MD5 hash")
	}

	// Set md5sum from hasher
	i.MD5 = hex.EncodeToString(hasher.Sum(nil))

	// read into hasher for md5
	_, err = io.Copy(sha, bytes.NewReader(i.image.Bytes()))
	if err != nil {
		return errors.New("Problem creating SHA1 hash")
	}

	// Set sha1
	i.SHA = hex.EncodeToString(sha.Sum(nil))

	return

}

// check if the md5 is a banned file
func (i *ImageType) checkBanned() (err error) {

	// Get Database handle
	dbase, err := db.GetDb()
	if err != nil {
		return
	}

	if i.MD5 == "" {
		return errors.New("No hash set on file banned check")
	}

	var check bool

	err = dbase.QueryRow(`SELECT count(*) FROM banned_files WHERE ban_hash = ?`, i.MD5).Scan(&check)
	if err != nil {
		return
	}

	// return error if it exists
	if check {
		return fmt.Errorf("File is banned")
	}

	return
}

// check if the md5 is already in the database
func (i *ImageType) checkDuplicate() (err error) {

	// Get Database handle
	dbase, err := db.GetDb()
	if err != nil {
		return
	}

	if i.MD5 == "" {
		return errors.New("No hash set on duplicate check")
	}

	if i.Ib == 0 {
		return errors.New("No imageboard set on duplicate check")
	}

	var check bool
	var thread, post sql.NullInt64

	err = dbase.QueryRow(`select count(1),posts.post_num,threads.thread_id from threads
	LEFT JOIN posts on threads.thread_id = posts.thread_id
	LEFT JOIN images on posts.post_id = images.post_id
	WHERE image_hash = ? AND ib_id = ?`, i.MD5, i.Ib).Scan(&check, &post, &thread)
	if err != nil {
		return
	}

	// return error if it exists
	if check {
		return fmt.Errorf("Image has already been posted. Thread: %d Post: %d", thread.Int64, post.Int64)
	}

	return
}

func (i *ImageType) checkMagic() (err error) {

	// detect the mime type
	i.mime = http.DetectContentType(i.image.Bytes())

	switch i.mime {
	case "image/png":
		i.Ext = ".png"
	case "image/jpeg":
		i.Ext = ".jpg"
	case "image/gif":
		i.Ext = ".gif"
	case "video/webm":
		i.Ext = ".webm"
		i.video = true
	default:
		return errors.New("Unknown file type")
	}

	// Check to see if extension is allowed
	if !isAllowedExt(i.Ext) {
		return errors.New("Format not supported")
	}

	return

}

func (i *ImageType) getStats() (err error) {

	// skip if its a video since we cant decode it
	if i.video {
		return
	}

	// decode image config
	img, _, err := image.DecodeConfig(bytes.NewReader(i.image.Bytes()))
	if err != nil {
		return errors.New("Problem decoding image")
	}

	// set original width
	i.OrigWidth = img.Width
	// set original height
	i.OrigHeight = img.Height

	// Check against maximum sizes
	switch {
	case i.OrigWidth > config.Settings.Limits.ImageMaxWidth:
		return fmt.Errorf("Image width too large. Max: %dpx", config.Settings.Limits.ImageMaxWidth)
	case img.Width < config.Settings.Limits.ImageMinWidth:
		return fmt.Errorf("Image width too small. Min: %dpx", config.Settings.Limits.ImageMinWidth)
	case i.OrigHeight > config.Settings.Limits.ImageMaxHeight:
		return fmt.Errorf("Image height too large. Max: %dpx", config.Settings.Limits.ImageMaxHeight)
	case img.Height < config.Settings.Limits.ImageMinHeight:
		return fmt.Errorf("Image height too small. Min: %dpx", config.Settings.Limits.ImageMinHeight)
	case i.image.Len() > config.Settings.Limits.ImageMaxSize:
		return fmt.Errorf("Image filesize too large. Max: %dMB", (config.Settings.Limits.ImageMaxSize/1024)/1024)
	}

	return

}

func (i *ImageType) saveFile() (err error) {

	defer i.image.Reset()

	i.makeFilenames()

	// avatar filename is the users id
	if i.avatar {
		i.Thumbnail = fmt.Sprintf("%d.png", i.Ib)
		i.Thumbpath = filepath.Join(local.Settings.Directories.ThumbnailDir, i.Thumbnail)
	}

	if !i.IsValid() {
		return errors.New("ImageType is not valid")
	}

	imagefile := filepath.Join(local.Settings.Directories.ImageDir, i.Filename)

	image, err := os.Create(imagefile)
	if err != nil {
		return errors.New("Problem saving file")
	}
	defer image.Close()

	_, err = io.Copy(image, bytes.NewReader(i.image.Bytes()))
	if err != nil {
		return errors.New("Problem saving file")
	}

	return

}

// Make a random unix time filename
func (i *ImageType) makeFilenames() {

	// get new short id generator
	sid := shortid.MustNew(1, shortid.DefaultABC, 9001)

	// generate filename
	filename := sid.MustGenerate()

	// Append ext to filename
	i.Filename = fmt.Sprintf("%s%s", filename, i.Ext)

	// Append jpg to thumbnail name because it is always a jpg
	i.Thumbnail = fmt.Sprintf("%ss.jpg", filename)

	// set the full file path
	i.Filepath = filepath.Join(local.Settings.Directories.ImageDir, i.Filename)

	// set the full thumbnail path
	i.Thumbpath = filepath.Join(local.Settings.Directories.ThumbnailDir, i.Thumbnail)

	return

}

func (i *ImageType) createThumbnail(maxwidth, maxheight int) (err error) {

	var imagef string

	if i.video {
		imagef = fmt.Sprintf("%s[0]", i.Thumbpath)
	} else {
		imagef = fmt.Sprintf("%s[0]", i.Filepath)
	}

	originalDimensions := fmt.Sprintf("%dx%d", i.OrigWidth, i.OrigHeight)

	var args []string

	// different options for avatars
	if i.avatar {
		args = []string{
			"-size",
			originalDimensions,
			imagef,
			"-background",
			"none",
			"-thumbnail",
			fmt.Sprintf("%dx%d^", maxwidth, maxheight),
			"-gravity",
			"center",
			"-extent",
			fmt.Sprintf("%dx%d", maxwidth, maxheight),
			i.Thumbpath,
		}
	} else {
		args = []string{
			"-background",
			"white",
			"-flatten",
			"-size",
			originalDimensions,
			"-resize",
			fmt.Sprintf("%dx%d>", maxwidth, maxheight),
			"-quality",
			"90",
			imagef,
			i.Thumbpath,
		}
	}

	_, err = exec.Command("convert", args...).Output()
	if err != nil {
		return errors.New("Problem making thumbnail")
	}

	thumb, err := os.Open(i.Thumbpath)
	if err != nil {
		return errors.New("Problem creating thumbnail file")
	}
	defer thumb.Close()

	img, _, err := image.DecodeConfig(thumb)
	if err != nil {
		return errors.New("Problem decoding thumbnail")
	}

	i.ThumbWidth = img.Width
	i.ThumbHeight = img.Height

	return

}

func (i *ImageType) copyToS3() (err error) {

	s3 := amazon.New()

	err = s3.Save(i.Filepath, fmt.Sprintf("src/%s", i.Filename), i.mime, false)
	if err != nil {
		return
	}

	err = s3.Save(i.Thumbpath, fmt.Sprintf("thumb/%s", i.Thumbnail), "image/jpeg", false)
	if err != nil {
		return
	}

	return
}
<|endoftext|>


<commit_before>// Copyright 2017 Jeff Foley. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package utils

import (
	"sync"
)

type queueNode struct {
	Next *queueNode
	Data interface{}
}

// Queue implements a FIFO data structure.
type Queue struct {
	sync.Mutex
	size       int
	head, tail *queueNode
}

// Append adds the data to the end of the Queue.
func (q *Queue) Append(data interface{}) {
	element := new(queueNode)

	q.Lock()
	defer q.Unlock()

	q.size++
	if q.head == nil {
		q.head = element
	}

	end := q.tail
	if end != nil {
		end.Next = element
	}
	q.tail = element
	element.Data = data
}

// Next returns the data at the front of the Queue.
func (q *Queue) Next() (interface{}, bool) {
	q.Lock()
	defer q.Unlock()

	if q.head == nil {
		return nil, false
	}

	q.size--
	element := q.head
	q.head = element.Next
	if q.tail == element {
		q.tail = nil
	}
	element.Next = nil
	return element.Data, true
}

// Empty return true if the Queue is empty.
func (q *Queue) Empty() bool {
	q.Lock()
	defer q.Unlock()

	if q.head == nil {
		return true
	}
	return false
}

// Len returns the current length of the Queue
func (q *Queue) Len() int {
	return q.size
}
<commit_msg>updated comments for queue implementation<commit_after>// Copyright 2017 Jeff Foley. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package utils

import (
	"sync"
)

type queueNode struct {
	Next *queueNode
	Data interface{}
}

// Queue implements a FIFO data structure.
type Queue struct {
	sync.Mutex
	size       int
	head, tail *queueNode
}

// Append adds the data to the end of the Queue.
func (q *Queue) Append(data interface{}) {
	element := new(queueNode)

	q.Lock()
	defer q.Unlock()

	q.size++
	if q.head == nil {
		q.head = element
	}

	end := q.tail
	if end != nil {
		end.Next = element
	}
	q.tail = element
	element.Data = data
}

// Next returns the data at the front of the Queue.
func (q *Queue) Next() (interface{}, bool) {
	q.Lock()
	defer q.Unlock()

	if q.head == nil {
		return nil, false
	}

	q.size--
	element := q.head
	q.head = element.Next
	if q.tail == element {
		q.tail = nil
	}
	element.Next = nil
	return element.Data, true
}

// Empty returns true if the Queue is empty.
func (q *Queue) Empty() bool {
	q.Lock()
	defer q.Unlock()

	if q.head == nil {
		return true
	}
	return false
}

// Len returns the current length of the Queue
func (q *Queue) Len() int {
	return q.size
}
<|endoftext|>


<commit_before>package utils

import (
	"bytes"
	"crypto/md5"
	"crypto/sha1"
	"encoding/binary"
	"encoding/gob"
	"encoding/hex"
	"encoding/json"
	"errors"
	"hash"
	"io"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"

	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
)

var (
	exitListening bool
	exitHandlers  []func()
)

func CatchExit(handler func()) {
	if handler == nil {
		return
	}

	exitHandlers = append(exitHandlers, handler)

	if exitListening {
		return
	}

	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)

		for {
			<-c
			for _, handler := range exitHandlers {
				handler()
			}
			os.Exit(1)
		}
	}()
	exitListening = true
}

func Contains(items interface{}, item interface{}) (ok bool) {
	switch a := items.(type) {
	case string:
		if len(a) == 0 {
			return
		}
		sep, yes := item.(string)
		ok = yes && strings.Index(a, sep) > -1
		return
	case []string:
		if len(a) == 0 {
			return
		}
		s, yes := item.(string)
		if yes {
			for _, c := range a {
				if c == s {
					ok = true
					return
				}
			}
		}
		return
	case []int:
		if len(a) == 0 {
			return
		}
		i, yes := item.(int)
		if yes {
			for _, c := range a {
				if c == i {
					return true
				}
			}
		}
		return
	default:
		return
	}
}

func CopyFile(src, dst string) (n int64, err error) {
	if src == dst {
		return
	}

	_, err = os.Lstat(dst)
	if err != nil && !os.IsNotExist(err) {
		return
	}

	sf, err := os.Open(src)
	if err != nil {
		return
	}
	defer sf.Close()

	df, err := os.Create(dst)
	if err != nil {
		return
	}
	defer df.Close()

	return io.Copy(df, sf)
}

func HashString(hasher, input interface{}) string {
	var h hash.Hash
	switch v := hasher.(type) {
	case hash.Hash:
		h = v
	case string:
		switch v {
		case "sha1":
			h = sha1.New()
		case "md5":
			h = md5.New()
		}
	}
	if h == nil {
		h = md5.New()
	}
	switch v := input.(type) {
	case []byte:
		h.Write(v)
	case string:
		h.Write([]byte(v))
	case io.Reader:
		io.Copy(h, v)
	}
	return hex.EncodeToString(h.Sum(nil))
}

func UnmarshalJSONFile(filename string, v interface{}) (err error) {
	f, err := os.Open(filename)
	if err != nil {
		return
	}
	defer f.Close()
	return json.NewDecoder(f).Decode(v)
}

func MarshalJSONFile(filename string, v interface{}) (err error) {
	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
	if err != nil {
		return
	}
	defer f.Close()
	return json.NewEncoder(f).Encode(v)
}

func GetHttpJSON(url string, v interface{}) (err error) {
	resp, err := http.Get(url)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		err = errors.New(resp.Status)
		return
	}

	err = json.NewDecoder(resp.Body).Decode(&v)
	return
}

func UnmarshalGobFile(filename string, v interface{}) (err error) {
	f, err := os.Open(filename)
	if err != nil {
		return
	}
	defer f.Close()
	return gob.NewDecoder(f).Decode(v)
}

func MarshalGobFile(filename string, v interface{}) (err error) {
	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
	if err != nil {
		return
	}
	defer f.Close()
	return gob.NewEncoder(f).Encode(v)
}

func SplitByFirstByte(s string, c byte) (string, string) {
	for i, l := 0, len(s); i < l; i++ {
		if s[i] == c {
			return s[:i], s[i+1:]
		}
	}
	return s, ""
}

func SplitByLastByte(s string, c byte) (string, string) {
	for i := len(s) - 1; i >= 0; i-- {
		if s[i] == c {
			return s[:i], s[i+1:]
		}
	}
	return s, ""
}

func FileExt(filename string) (ext string) {
	for i := len(filename) - 1; i > 0; i-- {
		if filename[i] == '.' {
			ext = filename[i+1:]
			break
		}
	}
	return
}

func ToLines(s string) (lines []string) {
	for i, j, l := 0, 0, len(s); i < l; i++ {
		switch s[i] {
		case '\r', '\n':
			if i == j {
				lines = append(lines, "")
			} else if i > j {
				lines = append(lines, s[j:i])
			}
			j = i + 1
			if s[i] == '\r' && i+1 < l && s[i+1] == '\n' {
				i++
				j++
			}
		default:
			if i == l-1 && j < l {
				lines = append(lines, s[j:])
			}
		}
	}
	return
}

func ToNumber(v interface{}) (f float64, ok bool) {
	ok = true
	switch i := v.(type) {
	case string:
		i64, err := strconv.ParseInt(i, 10, 64)
		if err != nil {
			ok = false
		} else {
			f = float64(i64)
		}
	case int:
		f = float64(i)
	case int8:
		f = float64(i)
	case int16:
		f = float64(i)
	case int32:
		f = float64(i)
	case int64:
		f = float64(i)
	case uint:
		f = float64(i)
	case uint8:
		f = float64(i)
	case uint16:
		f = float64(i)
	case uint32:
		f = float64(i)
	case uint64:
		f = float64(i)
	case float32:
		f = float64(i)
	case float64:
		f = i
	default:
		ok = false
	}
	return
}

func Html2Text(reader io.Reader) (text string, err error) {
	buf := bytes.NewBuffer(nil)
	doc, err := html.Parse(reader)
	if err != nil {
		return
	}
	extractNode(doc, buf)
	text = buf.String()
	return
}

func extractNode(node *html.Node, buf *bytes.Buffer) {
	if node.Type == html.TextNode {
		data := strings.Trim(node.Data, "\r\n ")
		if len(data) > 0 {
			buf.WriteString(data)
			switch node.Parent.DataAtom {
			case atom.A, atom.Image, atom.Span, atom.Strong, atom.B, atom.I, atom.Em, atom.Small, atom.Big, atom.Ins, atom.Del, atom.Button:
			default:
				buf.WriteByte('\n')
			}
		}
	}
	for c := node.FirstChild; c != nil; c = c.NextSibling {
		extractNode(c, buf)
	}
}

// PathClean has the same function with path.Clean(strings.Replace(strings.TrimSpace(s), "\\", "/", -1)),
// but it's faster!
func CleanPath(path string, toLower bool) string {
	pl := len(path)
	if pl == 0 {
		return "."
	}
	var n int
	var c byte
	var root bool
	var newpath = make([]byte, pl)
	for i := 0; i < pl; i++ {
		switch c = path[i]; c {
		case ' ':
			if n == 0 {
				continue
			}
			newpath[n] = ' '
			n++
		case '/', '\\':
			if n > 0 {
				if newpath[n-1] == '/' {
					continue
				} else if newpath[n-1] == '.' && n > 1 && newpath[n-2] == '/' {
					n--
					continue
				}
			}
			if n == 0 {
				root = true
			}
			newpath[n] = '/'
			n++
		case '.':
			if n > 1 && newpath[n-1] == '.' && newpath[n-2] == '/' {
				if n = n - 2; n > 0 {
					for n = n - 1; n > 0; n-- {
						if newpath[n] == '/' {
							break
						}
					}
				}
				continue
			}
			newpath[n] = '.'
			n++
		default:
			newpath[n] = c
			n++
		}
	}
	// trim right spaces
	if n > 0 && newpath[n-1] == ' ' {
		for n > 0 && newpath[n-1] == ' ' {
			n--
		}
	}
	if n > 1 && newpath[n-1] == '.' && newpath[n-2] == '/' {
		n--
	}
	if n > 0 && newpath[n-1] == '/' && (!root || n > 1) {
		n--
	}
	if n == 0 {
		return "."
	}
	return string(newpath[:n])
}

func Ipv4ToLong(ipStr string) uint32 {
	ip := net.ParseIP(ipStr)
	if ip == nil {
		return 0
	}
	ip = ip.To4()
	return binary.BigEndian.Uint32(ip)
}

func LongToIpv4(ipLong uint32) string {
	ipByte := make([]byte, 4)
	binary.BigEndian.PutUint32(ipByte, ipLong)
	ip := net.IP(ipByte)
	return ip.String()
}

func GetLocalIp() (ip string) {
	addrs, err := net.InterfaceAddrs()
	if err != nil {
		return
	}

	for _, addr := range addrs {
		if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
			if ipnet.IP.To4() != nil {
				ip = ipnet.IP.String()
				break
			}
		}
	}
	return
}
<commit_msg>rewrite CatchExit function of utils package<commit_after>package utils

import (
	"bytes"
	"crypto/md5"
	"crypto/sha1"
	"encoding/binary"
	"encoding/gob"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"hash"
	"io"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"

	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
)

var (
	exitCallbacks []func()
)

func CatchExit(callback func(), wait bool) {
	if callback == nil {
		return
	}
	exitCallbacks = append(exitCallbacks, callback)

	if !wait {
		return
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)

	fmt.Println("exit signal catched:", <-c)
	for _, callback := range exitCallbacks {
		callback()
	}
}

func Contains(items interface{}, item interface{}) (ok bool) {
	switch a := items.(type) {
	case string:
		if len(a) == 0 {
			return
		}
		sep, yes := item.(string)
		ok = yes && strings.Index(a, sep) > -1
		return
	case []string:
		if len(a) == 0 {
			return
		}
		s, yes := item.(string)
		if yes {
			for _, c := range a {
				if c == s {
					ok = true
					return
				}
			}
		}
		return
	case []int:
		if len(a) == 0 {
			return
		}
		i, yes := item.(int)
		if yes {
			for _, c := range a {
				if c == i {
					return true
				}
			}
		}
		return
	default:
		return
	}
}

func CopyFile(src, dst string) (n int64, err error) {
	if src == dst {
		return
	}

	_, err = os.Lstat(dst)
	if err != nil && !os.IsNotExist(err) {
		return
	}

	sf, err := os.Open(src)
	if err != nil {
		return
	}
	defer sf.Close()

	df, err := os.Create(dst)
	if err != nil {
		return
	}
	defer df.Close()

	return io.Copy(df, sf)
}

func HashString(hasher, input interface{}) string {
	var h hash.Hash
	switch v := hasher.(type) {
	case hash.Hash:
		h = v
	case string:
		switch v {
		case "sha1":
			h = sha1.New()
		case "md5":
			h = md5.New()
		}
	}
	if h == nil {
		h = md5.New()
	}
	switch v := input.(type) {
	case []byte:
		h.Write(v)
	case string:
		h.Write([]byte(v))
	case io.Reader:
		io.Copy(h, v)
	}
	return hex.EncodeToString(h.Sum(nil))
}

func UnmarshalJSONFile(filename string, v interface{}) (err error) {
	f, err := os.Open(filename)
	if err != nil {
		return
	}
	defer f.Close()
	return json.NewDecoder(f).Decode(v)
}

func MarshalJSONFile(filename string, v interface{}) (err error) {
	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
	if err != nil {
		return
	}
	defer f.Close()
	return json.NewEncoder(f).Encode(v)
}

func GetHttpJSON(url string, v interface{}) (err error) {
	resp, err := http.Get(url)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		err = errors.New(resp.Status)
		return
	}

	err = json.NewDecoder(resp.Body).Decode(&v)
	return
}

func UnmarshalGobFile(filename string, v interface{}) (err error) {
	f, err := os.Open(filename)
	if err != nil {
		return
	}
	defer f.Close()
	return gob.NewDecoder(f).Decode(v)
}

func MarshalGobFile(filename string, v interface{}) (err error) {
	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
	if err != nil {
		return
	}
	defer f.Close()
	return gob.NewEncoder(f).Encode(v)
}

func SplitByFirstByte(s string, c byte) (string, string) {
	for i, l := 0, len(s); i < l; i++ {
		if s[i] == c {
			return s[:i], s[i+1:]
		}
	}
	return s, ""
}

func SplitByLastByte(s string, c byte) (string, string) {
	for i := len(s) - 1; i >= 0; i-- {
		if s[i] == c {
			return s[:i], s[i+1:]
		}
	}
	return s, ""
}

func FileExt(filename string) (ext string) {
	for i := len(filename) - 1; i > 0; i-- {
		if filename[i] == '.' {
			ext = filename[i+1:]
			break
		}
	}
	return
}

func ToLines(s string) (lines []string) {
	for i, j, l := 0, 0, len(s); i < l; i++ {
		switch s[i] {
		case '\r', '\n':
			if i == j {
				lines = append(lines, "")
			} else if i > j {
				lines = append(lines, s[j:i])
			}
			j = i + 1
			if s[i] == '\r' && i+1 < l && s[i+1] == '\n' {
				i++
				j++
			}
		default:
			if i == l-1 && j < l {
				lines = append(lines, s[j:])
			}
		}
	}
	return
}

func ToNumber(v interface{}) (f float64, ok bool) {
	ok = true
	switch i := v.(type) {
	case string:
		i64, err := strconv.ParseInt(i, 10, 64)
		if err != nil {
			ok = false
		} else {
			f = float64(i64)
		}
	case int:
		f = float64(i)
	case int8:
		f = float64(i)
	case int16:
		f = float64(i)
	case int32:
		f = float64(i)
	case int64:
		f = float64(i)
	case uint:
		f = float64(i)
	case uint8:
		f = float64(i)
	case uint16:
		f = float64(i)
	case uint32:
		f = float64(i)
	case uint64:
		f = float64(i)
	case float32:
		f = float64(i)
	case float64:
		f = i
	default:
		ok = false
	}
	return
}

func Html2Text(reader io.Reader) (text string, err error) {
	buf := bytes.NewBuffer(nil)
	doc, err := html.Parse(reader)
	if err != nil {
		return
	}
	extractNode(doc, buf)
	text = buf.String()
	return
}

func extractNode(node *html.Node, buf *bytes.Buffer) {
	if node.Type == html.TextNode {
		data := strings.Trim(node.Data, "\r\n ")
		if len(data) > 0 {
			buf.WriteString(data)
			switch node.Parent.DataAtom {
			case atom.A, atom.Image, atom.Span, atom.Strong, atom.B, atom.I, atom.Em, atom.Small, atom.Big, atom.Ins, atom.Del, atom.Button:
			default:
				buf.WriteByte('\n')
			}
		}
	}
	for c := node.FirstChild; c != nil; c = c.NextSibling {
		extractNode(c, buf)
	}
}

// PathClean has the same function with path.Clean(strings.Replace(strings.TrimSpace(s), "\\", "/", -1)),
// but it's faster!
func CleanPath(path string, toLower bool) string {
	pl := len(path)
	if pl == 0 {
		return "."
	}
	var n int
	var c byte
	var root bool
	var newpath = make([]byte, pl)
	for i := 0; i < pl; i++ {
		switch c = path[i]; c {
		case ' ':
			if n == 0 {
				continue
			}
			newpath[n] = ' '
			n++
		case '/', '\\':
			if n > 0 {
				if newpath[n-1] == '/' {
					continue
				} else if newpath[n-1] == '.' && n > 1 && newpath[n-2] == '/' {
					n--
					continue
				}
			}
			if n == 0 {
				root = true
			}
			newpath[n] = '/'
			n++
		case '.':
			if n > 1 && newpath[n-1] == '.' && newpath[n-2] == '/' {
				if n = n - 2; n > 0 {
					for n = n - 1; n > 0; n-- {
						if newpath[n] == '/' {
							break
						}
					}
				}
				continue
			}
			newpath[n] = '.'
			n++
		default:
			newpath[n] = c
			n++
		}
	}
	// trim right spaces
	if n > 0 && newpath[n-1] == ' ' {
		for n > 0 && newpath[n-1] == ' ' {
			n--
		}
	}
	if n > 1 && newpath[n-1] == '.' && newpath[n-2] == '/' {
		n--
	}
	if n > 0 && newpath[n-1] == '/' && (!root || n > 1) {
		n--
	}
	if n == 0 {
		return "."
	}
	return string(newpath[:n])
}

func Ipv4ToLong(ipStr string) uint32 {
	ip := net.ParseIP(ipStr)
	if ip == nil {
		return 0
	}
	ip = ip.To4()
	return binary.BigEndian.Uint32(ip)
}

func LongToIpv4(ipLong uint32) string {
	ipByte := make([]byte, 4)
	binary.BigEndian.PutUint32(ipByte, ipLong)
	ip := net.IP(ipByte)
	return ip.String()
}

func GetLocalIp() (ip string) {
	addrs, err := net.InterfaceAddrs()
	if err != nil {
		return
	}

	for _, addr := range addrs {
		if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
			if ipnet.IP.To4() != nil {
				ip = ipnet.IP.String()
				break
			}
		}
	}
	return
}
<|endoftext|>