<commit_before>package main

import (
	"fmt"
	"github.com/gin-gonic/gin"
	"runtime"

	"github.com/techjanitor/pram-post/config"
	c "github.com/techjanitor/pram-post/controllers"
	m "github.com/techjanitor/pram-post/middleware"
	u "github.com/techjanitor/pram-post/utils"
)

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	config.Print()

	// Set up DB connection
	u.NewDb()

	// Set up Redis connection
	u.NewRedisCache()
}

func main() {
	r := gin.Default()

	r.Use(gin.ForwardedFor("127.0.0.1/32"))
	// Checks for antispam cookie
	r.Use(m.GetAntiSpamCookie())

	r.POST("/thread/new", c.ThreadController)
	r.POST("/thread/reply", c.ReplyController)
	r.POST("/tag/new", c.NewTagController)
	r.POST("/tag/add", c.AddTagController)
	r.NoRoute(c.ErrorController)

	r.Run(fmt.Sprintf("%s:%d", config.Settings.General.Address, config.Settings.General.Port))

}
<commit_msg>add graceful shutdown<commit_after>package main

import (
	"fmt"
	"github.com/facebookgo/grace/gracehttp"
	"github.com/gin-gonic/gin"
	"runtime"

	"github.com/techjanitor/pram-post/config"
	c "github.com/techjanitor/pram-post/controllers"
	m "github.com/techjanitor/pram-post/middleware"
	u "github.com/techjanitor/pram-post/utils"
)

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	config.Print()

	// Set up DB connection
	u.NewDb()

	// Set up Redis connection
	u.NewRedisCache()
}

func main() {
	r := gin.Default()

	r.Use(gin.ForwardedFor("127.0.0.1/32"))
	// Checks for antispam cookie
	r.Use(m.GetAntiSpamCookie())

	r.POST("/thread/new", c.ThreadController)
	r.POST("/thread/reply", c.ReplyController)
	r.POST("/tag/new", c.NewTagController)
	r.POST("/tag/add", c.AddTagController)
	r.NoRoute(c.ErrorController)

	s := &http.Server{
		Addr:    fmt.Sprintf("%s:%d", config.Settings.General.Address, config.Settings.General.Port),
		Handler: r,
	}

	gracehttp.Serve(s)

}
<|endoftext|>


<commit_before>// Copyright 2013 Joe Walnes and the websocketd team.
// All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"net/http"
	"os"
	"strconv"
	"strings"

	"github.com/joewalnes/websocketd/libwebsocketd"
)

func log(l *libwebsocketd.LogScope, level libwebsocketd.LogLevel, levelName string, category string, msg string, args ...interface{}) {
	if level < l.MinLevel {
		return
	}
	fullMsg := fmt.Sprintf(msg, args...)

	assocDump := ""
	for index, pair := range l.Associated {
		if index > 0 {
			assocDump += " "
		}
		assocDump += fmt.Sprintf("%s:'%s'", pair.Key, pair.Value)
	}

	l.Mutex.Lock()
	fmt.Printf("%s | %-6s | %-10s | %s | %s\n", libwebsocketd.Timestamp(), levelName, category, assocDump, fullMsg)
	l.Mutex.Unlock()
}

func main() {
	config := parseCommandLine()

	log := libwebsocketd.RootLogScope(config.LogLevel, log)

	if config.DevConsole {
		if config.StaticDir != "" {
			log.Fatal("server", "Invalid parameters: --devconsole cannot be used with --staticdir. Pick one.")
			os.Exit(4)
		}
		if config.CgiDir != "" {
			log.Fatal("server", "Invalid parameters: --devconsole cannot be used with --cgidir. Pick one.")
			os.Exit(4)
		}
	}

	os.Clearenv() // it's ok to wipe it clean, we already read env variables from passenv into config
	handler := libwebsocketd.NewWebsocketdServer(config.Config, log, config.MaxForks)
	http.Handle("/", handler)

	if config.UsingScriptDir {
		log.Info("server", "Serving from directory      : %s", config.ScriptDir)
	} else if config.CommandName != "" {
		log.Info("server", "Serving using application   : %s %s", config.CommandName, strings.Join(config.CommandArgs, " "))
	}
	if config.StaticDir != "" {
		log.Info("server", "Serving static content from : %s", config.StaticDir)
	}
	if config.CgiDir != "" {
		log.Info("server", "Serving CGI scripts from    : %s", config.CgiDir)
	}

	rejects := make(chan error, 1)
	for _, addrSingle := range config.Addr {
		log.Info("server", "Starting WebSocket server   : %s", handler.TellURL("ws", addrSingle, "/"))
		if config.DevConsole {
			log.Info("server", "Developer console enabled   : %s", handler.TellURL("http", addrSingle, "/"))
		} else if config.StaticDir != "" || config.CgiDir != "" {
			log.Info("server", "Serving CGI or static files : %s", handler.TellURL("http", addrSingle, "/"))
		}
		// ListenAndServe is blocking function. Let's run it in
		// go routine, reporting result to control channel.
		// Since it's blocking it'll never return non-error.

		go func(addr string) {
			if config.Ssl {
				rejects <- http.ListenAndServeTLS(addr, config.CertFile, config.KeyFile, nil)
			} else {
				rejects <- http.ListenAndServe(addr, nil)
			}
			fmt.Println(config.RedirPort)
		}(addrSingle)

		if config.RedirPort != 0 {
			go func(addr string) {
				pos := strings.IndexByte(addr, ':')
				rediraddr := addr[:pos] + ":" + strconv.Itoa(config.RedirPort) // it would be silly to optimize this one
				redir := &http.Server{Addr: rediraddr, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

					// redirect to same hostname as in request but different port and probably schema
					uri := "https://"
					if !config.Ssl {
						uri = "http://"
					}
					uri += r.Host[:strings.IndexByte(r.Host, ':')] + addr[pos:] + "/"

					http.Redirect(w, r, uri, http.StatusMovedPermanently)
				})}
				log.Info("server", "Starting redirect server   : http://%s/", rediraddr)
				rejects <- redir.ListenAndServe()
			}(addrSingle)
		}
	}
	select {
	case err := <-rejects:
		log.Fatal("server", "Can't start server: %s", err)
		os.Exit(3)
	}
}
<commit_msg>Removed debug code<commit_after>// Copyright 2013 Joe Walnes and the websocketd team.
// All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"net/http"
	"os"
	"strconv"
	"strings"

	"github.com/joewalnes/websocketd/libwebsocketd"
)

func log(l *libwebsocketd.LogScope, level libwebsocketd.LogLevel, levelName string, category string, msg string, args ...interface{}) {
	if level < l.MinLevel {
		return
	}
	fullMsg := fmt.Sprintf(msg, args...)

	assocDump := ""
	for index, pair := range l.Associated {
		if index > 0 {
			assocDump += " "
		}
		assocDump += fmt.Sprintf("%s:'%s'", pair.Key, pair.Value)
	}

	l.Mutex.Lock()
	fmt.Printf("%s | %-6s | %-10s | %s | %s\n", libwebsocketd.Timestamp(), levelName, category, assocDump, fullMsg)
	l.Mutex.Unlock()
}

func main() {
	config := parseCommandLine()

	log := libwebsocketd.RootLogScope(config.LogLevel, log)

	if config.DevConsole {
		if config.StaticDir != "" {
			log.Fatal("server", "Invalid parameters: --devconsole cannot be used with --staticdir. Pick one.")
			os.Exit(4)
		}
		if config.CgiDir != "" {
			log.Fatal("server", "Invalid parameters: --devconsole cannot be used with --cgidir. Pick one.")
			os.Exit(4)
		}
	}

	os.Clearenv() // it's ok to wipe it clean, we already read env variables from passenv into config
	handler := libwebsocketd.NewWebsocketdServer(config.Config, log, config.MaxForks)
	http.Handle("/", handler)

	if config.UsingScriptDir {
		log.Info("server", "Serving from directory      : %s", config.ScriptDir)
	} else if config.CommandName != "" {
		log.Info("server", "Serving using application   : %s %s", config.CommandName, strings.Join(config.CommandArgs, " "))
	}
	if config.StaticDir != "" {
		log.Info("server", "Serving static content from : %s", config.StaticDir)
	}
	if config.CgiDir != "" {
		log.Info("server", "Serving CGI scripts from    : %s", config.CgiDir)
	}

	rejects := make(chan error, 1)
	for _, addrSingle := range config.Addr {
		log.Info("server", "Starting WebSocket server   : %s", handler.TellURL("ws", addrSingle, "/"))
		if config.DevConsole {
			log.Info("server", "Developer console enabled   : %s", handler.TellURL("http", addrSingle, "/"))
		} else if config.StaticDir != "" || config.CgiDir != "" {
			log.Info("server", "Serving CGI or static files : %s", handler.TellURL("http", addrSingle, "/"))
		}
		// ListenAndServe is blocking function. Let's run it in
		// go routine, reporting result to control channel.
		// Since it's blocking it'll never return non-error.

		go func(addr string) {
			if config.Ssl {
				rejects <- http.ListenAndServeTLS(addr, config.CertFile, config.KeyFile, nil)
			} else {
				rejects <- http.ListenAndServe(addr, nil)
			}
		}(addrSingle)

		if config.RedirPort != 0 {
			go func(addr string) {
				pos := strings.IndexByte(addr, ':')
				rediraddr := addr[:pos] + ":" + strconv.Itoa(config.RedirPort) // it would be silly to optimize this one
				redir := &http.Server{Addr: rediraddr, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

					// redirect to same hostname as in request but different port and probably schema
					uri := "https://"
					if !config.Ssl {
						uri = "http://"
					}
					uri += r.Host[:strings.IndexByte(r.Host, ':')] + addr[pos:] + "/"

					http.Redirect(w, r, uri, http.StatusMovedPermanently)
				})}
				log.Info("server", "Starting redirect server   : http://%s/", rediraddr)
				rejects <- redir.ListenAndServe()
			}(addrSingle)
		}
	}
	select {
	case err := <-rejects:
		log.Fatal("server", "Can't start server: %s", err)
		os.Exit(3)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"github.com/drone/drone-plugin-go/plugin"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"time"
)

var debug bool

func doRequest(param ReqEnvelope) (bool, error) {
	if debug {
		log.Println("doRequest ")
	}
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	client := &http.Client{Transport: tr}
	var req *http.Request
	var err error
	// post payload to each artifact
	if param.Json == nil {
		req, err = http.NewRequest(param.Verb, param.Url, nil)
	} else {
		req, err = http.NewRequest(param.Verb, param.Url, bytes.NewBuffer(param.Json))
	}

	if param.Verb == "PATCH" {
		req.Header.Set("Content-Type", "application/strategic-merge-patch+json ")
	} else {
		req.Header.Set("Content-Type", "application/json ")
	}

	if debug {
		log.Println("HTTP Request %s", param.Verb)
		log.Println("HTTP Request %s", param.Url)
		log.Println("HTTP Request %s", string(param.Json))
	}

	req.Header.Set("Authorization", "Bearer "+param.Token)
	response, err := client.Do(req)
	if debug {
		contents, err := ioutil.ReadAll(response.Body)
		if err != nil {
			log.Fatal(err)
		}
		log.Printf("%s\n", string(contents))
	}

	if err != nil {
		log.Fatal(err)
	} else {
		defer response.Body.Close()

		if response.StatusCode == 200 {
			return true, err
		}
	}
	return false, err
}

func readArtifactFromFile(workspace string, artifactFile string, apiserver string, namespace string) (Artifact, error) {
	artifactFilename := workspace + "/" + artifactFile
	if debug {
		log.Println("readArtifactFromFile " + artifactFilename)
	}
	file, err := ioutil.ReadFile(artifactFilename)
	if err != nil {
		log.Fatal(err)
	}
	artifact := Artifact{}
	if strings.HasSuffix(artifactFilename, ".yaml") {
		file = yaml2Json(file)
	}

	json.Unmarshal(file, &artifact)
	artifact.Data = file

	if artifact.Kind == "ReplicationController" {
		artifact.Url = fmt.Sprintf("%s/api/v1/namespaces/%s/replicationcontrollers", apiserver, namespace)
	}
	if artifact.Kind == "Service" {
		artifact.Url = fmt.Sprintf("%s/api/v1/namespaces/%s/services", apiserver, namespace)
	}

	return artifact, err
}

func makeTimestamp() int64 {
	return time.Now().UnixNano() / int64(time.Millisecond)
}

func sendWebhook(wh *WebHook) {

	jwh, err := json.Marshal(wh)
	if err != nil {
		log.Panic(err)
		return
	}
	req := ReqEnvelope{
		Verb:  "POST",
		Token: wh.Token,
		Url:   wh.Url,
		Json:  []byte(jwh),
	}
	doRequest(req)
}

var deployments []string

func main() {
	var vargs = struct {
		ReplicationControllers []string `json:replicationcontrollers`
		Services               []string `json:services`
		ApiServer              string   `json:apiserver`
		Token                  string   `json:token`
		Namespace              string   `json:namespace`
		Debug                  string   `json:debug`
		Source                 string   `json:source`
	}{}

	workspace := plugin.Workspace{}
	plugin.Param("workspace", &workspace)
	plugin.Param("vargs", &vargs)
	plugin.Parse()
	debug = true
	if vargs.Debug == "true" {
		debug = true
	}

	if debug {
		log.Println("Workspace Root: " + workspace.Root)
		log.Println("Workspace Path: " + workspace.Path)
	}

	// Iterate over rcs and svcs
	for _, rc := range vargs.ReplicationControllers {
		artifact, err := readArtifactFromFile(workspace.Path, rc, vargs.ApiServer, vargs.Namespace)
		if err != nil {
			log.Fatal(err)
		}
		if debug {
			log.Println("Artifact loaded: " + artifact.Url)
		}
		if b, _ := existsArtifact(artifact, vargs.Token); b {
			deleteArtifact(artifact, vargs.Token)
			time.Sleep(time.Second * 5)
		}
		createArtifact(artifact, vargs.Token)
	}
	for _, rc := range vargs.Services {
		artifact, err := readArtifactFromFile(workspace.Path, rc, vargs.ApiServer, vargs.Namespace)
		if err != nil {
			log.Fatal(err)
		}
		createArtifact(artifact, vargs.Token)
	}
	// if vargs.Webhook != "" {
	// 	wh := &WebHook{
	// 		Timestamp: makeTimestamp(),
	// 		Images:    deployments,
	// 		Namespace: vargs.Namespace,
	// 		Source:    vargs.Source,
	// 		Target:    vargs.ApiServer,
	// 		Url:       vargs.Webhook,
	// 		Token:     vargs.WebHookToken,
	// 	}
	// 	sendWebhook(wh)
	// }
}
<commit_msg>debug print env vars<commit_after>package main

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"github.com/drone/drone-plugin-go/plugin"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
	"time"
)

var debug bool

func doRequest(param ReqEnvelope) (bool, error) {
	if debug {
		log.Println("doRequest ")
	}
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	client := &http.Client{Transport: tr}
	var req *http.Request
	var err error
	// post payload to each artifact
	if param.Json == nil {
		req, err = http.NewRequest(param.Verb, param.Url, nil)
	} else {
		req, err = http.NewRequest(param.Verb, param.Url, bytes.NewBuffer(param.Json))
	}

	if param.Verb == "PATCH" {
		req.Header.Set("Content-Type", "application/strategic-merge-patch+json ")
	} else {
		req.Header.Set("Content-Type", "application/json ")
	}

	if debug {
		log.Println("HTTP Request %s", param.Verb)
		log.Println("HTTP Request %s", param.Url)
		log.Println("HTTP Request %s", string(param.Json))
	}

	req.Header.Set("Authorization", "Bearer "+param.Token)
	response, err := client.Do(req)
	if debug {
		contents, err := ioutil.ReadAll(response.Body)
		if err != nil {
			log.Fatal(err)
		}
		log.Printf("%s\n", string(contents))
	}

	if err != nil {
		log.Fatal(err)
	} else {
		defer response.Body.Close()

		if response.StatusCode == 200 {
			return true, err
		}
	}
	return false, err
}

func readArtifactFromFile(workspace string, artifactFile string, apiserver string, namespace string) (Artifact, error) {
	artifactFilename := workspace + "/" + artifactFile
	if debug {
		log.Println("readArtifactFromFile " + artifactFilename)
	}
	file, err := ioutil.ReadFile(artifactFilename)
	if err != nil {
		log.Fatal(err)
	}
	artifact := Artifact{}
	if strings.HasSuffix(artifactFilename, ".yaml") {
		file = yaml2Json(file)
	}

	json.Unmarshal(file, &artifact)
	artifact.Data = file

	if artifact.Kind == "ReplicationController" {
		artifact.Url = fmt.Sprintf("%s/api/v1/namespaces/%s/replicationcontrollers", apiserver, namespace)
	}
	if artifact.Kind == "Service" {
		artifact.Url = fmt.Sprintf("%s/api/v1/namespaces/%s/services", apiserver, namespace)
	}

	return artifact, err
}

func makeTimestamp() int64 {
	return time.Now().UnixNano() / int64(time.Millisecond)
}

func sendWebhook(wh *WebHook) {

	jwh, err := json.Marshal(wh)
	if err != nil {
		log.Panic(err)
		return
	}
	req := ReqEnvelope{
		Verb:  "POST",
		Token: wh.Token,
		Url:   wh.Url,
		Json:  []byte(jwh),
	}
	doRequest(req)
}

var deployments []string

func main() {
	var vargs = struct {
		ReplicationControllers []string `json:replicationcontrollers`
		Services               []string `json:services`
		ApiServer              string   `json:apiserver`
		Token                  string   `json:token`
		Namespace              string   `json:namespace`
		Debug                  string   `json:debug`
		Source                 string   `json:source`
	}{}

	workspace := plugin.Workspace{}
	plugin.Param("workspace", &workspace)
	plugin.Param("vargs", &vargs)
	plugin.Parse()
	debug = true
	if vargs.Debug == "true" {
		debug = true
	}

	if debug {
		log.Println("Workspace Root: " + workspace.Root)
		log.Println("Workspace Path: " + workspace.Path)

		for _, e := range os.Environ() {
			pair := strings.Split(e, "=")
			log.Println(pair[0])
		}
	}

	// Iterate over rcs and svcs
	for _, rc := range vargs.ReplicationControllers {
		artifact, err := readArtifactFromFile(workspace.Path, rc, vargs.ApiServer, vargs.Namespace)
		if err != nil {
			log.Fatal(err)
		}
		if debug {
			log.Println("Artifact loaded: " + artifact.Url)
		}
		if b, _ := existsArtifact(artifact, vargs.Token); b {
			deleteArtifact(artifact, vargs.Token)
			time.Sleep(time.Second * 5)
		}
		createArtifact(artifact, vargs.Token)
	}
	for _, rc := range vargs.Services {
		artifact, err := readArtifactFromFile(workspace.Path, rc, vargs.ApiServer, vargs.Namespace)
		if err != nil {
			log.Fatal(err)
		}
		createArtifact(artifact, vargs.Token)
	}
	// if vargs.Webhook != "" {
	// 	wh := &WebHook{
	// 		Timestamp: makeTimestamp(),
	// 		Images:    deployments,
	// 		Namespace: vargs.Namespace,
	// 		Source:    vargs.Source,
	// 		Target:    vargs.ApiServer,
	// 		Url:       vargs.Webhook,
	// 		Token:     vargs.WebHookToken,
	// 	}
	// 	sendWebhook(wh)
	// }
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"

	jd "github.com/josephburnett/jd/lib"
)

var patch = flag.Bool("p", false, "Patch mode")
var output = flag.String("o", "", "Output file")
var set = flag.Bool("set", false, "Arrays as sets")
var mset = flag.Bool("mset", false, "Arrays as multisets")
var setkeys = flag.String("setkeys", "", "Keys to identify set objects")

func main() {
	flag.Parse()
	var a, b string
	switch len(flag.Args()) {
	case 1:
		a = readFile(flag.Arg(0))
		b = readStdin()
	case 2:
		a = readFile(flag.Arg(0))
		b = readFile(flag.Arg(1))
	default:
		printUsageAndExit()
	}
	if *patch {
		patchJson(a, b)
	} else {
		diffJson(a, b)
	}
}

func printUsageAndExit() {
	for _, line := range []string{
		``,
		`Usage: jd [OPTION]... FILE1 [FILE2]`,
		`Diff and patch JSON files.`,
		``,
		`Prints the diff of FILE1 and FILE2 to STDOUT.`,
		`When FILE2 is omitted the second input is read from STDIN.`,
		`When patching (-p) FILE1 is a diff.`,
		``,
		`Options:`,
		`  -p        Apply patch FILE1 to FILE2 or STDIN.`,
		`  -o=FILE3  Write to FILE3 instead of STDOUT.`,
		`  -set      Treat arrays as sets.`,
		`  -mset     Treat arrays as multisets (bags).`,
		``,
		`Examples:`,
		`  jd a.json b.json`,
		`  cat b.json | jd a.json`,
		`  jd -o patch a.json b.json; jd patch a.json`,
		`  jd -set a.json b.json`,
		``,
	} {
		fmt.Println(line)
	}
	os.Exit(1)
}

func diffJson(a, b string) {
	aNode, err := readJsonString(a)
	if err != nil {
		log.Fatalf(err.Error())
	}
	bNode, err := readJsonString(b)
	if err != nil {
		log.Fatalf(err.Error())
	}
	diff := aNode.Diff(bNode)
	if *output == "" {
		fmt.Print(diff.Render())
	} else {
		ioutil.WriteFile(*output, []byte(diff.Render()), 0644)
	}
}

func patchJson(p, a string) {
	diff, err := readDiffString(p)
	if err != nil {
		log.Fatalf(err.Error())
	}
	aNode, err := readJsonString(a)
	if err != nil {
		log.Fatalf(err.Error())
	}
	bNode, err := aNode.Patch(diff)
	if err != nil {
		log.Fatalf(err.Error())
	}
	if *output == "" {
		fmt.Print(bNode.Json())
	} else {
		ioutil.WriteFile(*output, []byte(bNode.Json()), 0644)
	}
}

func readJsonString(s string) (jd.JsonNode, error) {
	if *set {
		return jd.ReadJsonString(s, jd.SET)
	}
	if *mset {
		return jd.ReadJsonString(s, jd.MULTISET)
	}
	return jd.ReadJsonString(s)
}

func readDiffString(s string) (jd.Diff, error) {
	if *set {
		return jd.ReadDiffString(s, jd.SET)
	}
	if *mset {
		return jd.ReadDiffString(s, jd.MULTISET)
	}
	return jd.ReadDiffString(s)
}

func readFile(filename string) string {
	bytes, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Fatalf(err.Error())
	}
	return string(bytes)
}

func readStdin() string {
	r := bufio.NewReader(os.Stdin)
	bytes, err := ioutil.ReadAll(r)
	if err != nil {
		log.Fatalf(err.Error())
	}
	return string(bytes)
}
<commit_msg>Pass setkeys option when unmarshalling.<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"

	jd "github.com/josephburnett/jd/lib"
)

var patch = flag.Bool("p", false, "Patch mode")
var output = flag.String("o", "", "Output file")
var set = flag.Bool("set", false, "Arrays as sets")
var mset = flag.Bool("mset", false, "Arrays as multisets")
var setkeys = flag.String("setkeys", "", "Keys to identify set objects")

func main() {
	flag.Parse()
	options, err := parseOptions()
	if err != nil {
		log.Fatalf(err.Error())
	}
	var a, b string
	switch len(flag.Args()) {
	case 1:
		a = readFile(flag.Arg(0))
		b = readStdin()
	case 2:
		a = readFile(flag.Arg(0))
		b = readFile(flag.Arg(1))
	default:
		printUsageAndExit()
	}
	if *patch {
		patchJson(a, b, options)
	} else {
		diffJson(a, b, options)
	}
}

func parseOptions() ([]jd.Option, error) {
	options := make([]jd.Option, 0)
	if *set {
		options = append(options, jd.SET)
	}
	if *mset {
		options = append(options, jd.MULTISET)
	}
	if *setkeys != "" {
		keys := make([]string, 0)
		ks := strings.Split(*setkeys, ",")
		for _, k := range ks {
			trimmed := strings.TrimSpace(k)
			if trimmed == "" {
				return nil, fmt.Errorf("Invalid set key: %v", k)
			}
			keys = append(keys, trimmed)
		}
		options = append(options, jd.SetkeysOption(keys))
	}
	return options, nil
}

func printUsageAndExit() {
	for _, line := range []string{
		``,
		`Usage: jd [OPTION]... FILE1 [FILE2]`,
		`Diff and patch JSON files.`,
		``,
		`Prints the diff of FILE1 and FILE2 to STDOUT.`,
		`When FILE2 is omitted the second input is read from STDIN.`,
		`When patching (-p) FILE1 is a diff.`,
		``,
		`Options:`,
		`  -p        Apply patch FILE1 to FILE2 or STDIN.`,
		`  -o=FILE3  Write to FILE3 instead of STDOUT.`,
		`  -set      Treat arrays as sets.`,
		`  -mset     Treat arrays as multisets (bags).`,
		``,
		`Examples:`,
		`  jd a.json b.json`,
		`  cat b.json | jd a.json`,
		`  jd -o patch a.json b.json; jd patch a.json`,
		`  jd -set a.json b.json`,
		``,
	} {
		fmt.Println(line)
	}
	os.Exit(1)
}

func diffJson(a, b string, options []jd.Option) {
	aNode, err := jd.ReadJsonString(a, options...)
	if err != nil {
		log.Fatalf(err.Error())
	}
	bNode, err := jd.ReadJsonString(b, options...)
	if err != nil {
		log.Fatalf(err.Error())
	}
	diff := aNode.Diff(bNode)
	if *output == "" {
		fmt.Print(diff.Render())
	} else {
		ioutil.WriteFile(*output, []byte(diff.Render()), 0644)
	}
}

func patchJson(p, a string, options []jd.Option) {
	diff, err := jd.ReadDiffString(p, options...)
	if err != nil {
		log.Fatalf(err.Error())
	}
	aNode, err := jd.ReadJsonString(a, options...)
	if err != nil {
		log.Fatalf(err.Error())
	}
	bNode, err := aNode.Patch(diff)
	if err != nil {
		log.Fatalf(err.Error())
	}
	if *output == "" {
		fmt.Print(bNode.Json())
	} else {
		ioutil.WriteFile(*output, []byte(bNode.Json()), 0644)
	}
}

func readFile(filename string) string {
	bytes, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Fatalf(err.Error())
	}
	return string(bytes)
}

func readStdin() string {
	r := bufio.NewReader(os.Stdin)
	bytes, err := ioutil.ReadAll(r)
	if err != nil {
		log.Fatalf(err.Error())
	}
	return string(bytes)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/tls"
	"fmt"
	"github.com/drone/drone-plugin-go/plugin"
	"io/ioutil"
	"net/http"
	"os"
)

var kubeConf *KubeConf

type KubeConf struct {
	ApiServer string `json:apiserver`
	Token     string `json:token`
	Namespace string `json:namespace`
}

type Artifact struct {
	Name      string
	Update    string // overwrite, rolling-update
	Namespace string
	Type      string // rcs, svcs
}

func (a Artifact) Exists() bool {
	return true
}

func createArtifact(artifact string, url string, token string, workspace string) {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}

	file, e := ioutil.ReadFile(workspace + "/" + artifact)
	fmt.Println(string(file))
	if e != nil {
		fmt.Println(e)
		os.Exit(1)
	}
	// post payload to each artifact
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(file))
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Authorization", "Bearer "+token)
	client := &http.Client{Transport: tr}
	resp, err := client.Do(req)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()
	contents, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		os.Exit(1)
	}
	fmt.Printf("%s\n", string(contents))

}

func main() {
	var vargs = struct {
		ReplicationControllers []string `json:"replicationcontrollers"`
		Services               []string `json:"services"`
		ApiServer              string   `json:apiserver`
		Token                  string   `json:token`
		Namespace              string   `json:namespace`
	}{}

	workspace := plugin.Workspace{}
	plugin.Param("workspace", &workspace)
	plugin.Param("vargs", &vargs)
	plugin.Parse()

	rc_url := fmt.Sprintf("%s/api/v1/namespaces/%s/replicationcontrollers", vargs.ApiServer, vargs.Namespace)
	svc_url := fmt.Sprintf("%s/api/v1/namespaces/%s/services", vargs.ApiServer, vargs.Namespace)

	// Iterate over rcs and svcs
	for _, rc := range vargs.ReplicationControllers {
		createArtifact(rc, rc_url, vargs.Token, workspace.Path)
	}
	for _, rc := range vargs.Services {
		createArtifact(rc, svc_url, vargs.Token, workspace.Path)
	}
}
<commit_msg>check before create to delete<commit_after>package main

import (
	"bytes"
	"crypto/tls"
	"fmt"
	"github.com/drone/drone-plugin-go/plugin"
	"io/ioutil"
	"net/http"
	"os"
)

type Artifact struct {
	ApiVersion string
	Kind       string
	Data       []byte
	Metadata   struct {
		Name string
	}
	Url string
}

// Kubernetes API doesn't delete pods when deleting an RC
// to cleanly remove the rc we have to set `replicas=0`
// and then delete the RC
func deleteArtifact(artifact Artifact, token string) (bool, error) {
	url := fmt.Sprintf("%s/%s", artifact.Url, artifact.Metadata.Name)
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	client := &http.Client{Transport: tr}
	// post payload to each artifact
	req, err := http.NewRequest("DELETE", url, nil)
	req.Header.Set("Authorization", "Bearer "+token)
	response, err := client.Do(req)
	if err != nil {
		fmt.Printf("%s", err)
		os.Exit(1)
	} else {
		defer response.Body.Close()
		contents, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			os.Exit(1)
		}
		fmt.Printf("%s\n", string(contents))
		if response.StatusCode == 200 {
			return true, err
		}
	}
	return false, err
}

func existsArtifact(artifact Artifact, token string) (bool, error) {
	url := fmt.Sprintf("%s/%s", artifact.Url, artifact.Metadata.Name)
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	client := &http.Client{Transport: tr}
	// post payload to each artifact
	req, err := http.NewRequest("GET", url, nil)
	req.Header.Set("Authorization", "Bearer "+token)
	response, err := client.Do(req)
	if err != nil {
		fmt.Printf("%s", err)
		os.Exit(1)
	} else {
		defer response.Body.Close()
		if response.StatusCode == 200 {
			return true, err
		}
	}
	return false, err
}

func createArtifact(artifact Artifact, token string) {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}

	// post payload to each artifact
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(artifact.Data))
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Authorization", "Bearer "+token)
	client := &http.Client{Transport: tr}
	resp, err := client.Do(req)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()
	contents, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		os.Exit(1)
	}
	fmt.Printf("%s\n", string(contents))

}

func readArtifactFromFile(workspace string, artifact string, apiserver string, namespace string) (Artifact, error) {
	file, e := ioutil.ReadFile(workspace + "/" + artifact)
	// fmt.Println(string(file))
	if e != nil {
		fmt.Println(e)
		os.Exit(1)
	}
	artifact := Artifact{}
	json.Unmarshal(file, &artifact)
	artifact.Data = file
	if artifact.Kind == "ReplicationController" {
		artifact.Url = fmt.Sprintf("%s/api/v1/namespaces/%s/replicationcontrollers", apiserver, namespace)
	}
	if artifact.Kind == "Service" {
		artifact.Url = fmt.Sprintf("%s/api/v1/namespaces/%s/services", apiserver, namespace)
	}

	return artifact, e
}

func main() {
	var vargs = struct {
		ReplicationControllers []string `json:replicationcontrollers`
		Services               []string `json:services`
		ApiServer              string   `json:apiserver`
		Token                  string   `json:token`
		Namespace              string   `json:namespace`
	}{}

	workspace := plugin.Workspace{}
	plugin.Param("workspace", &workspace)
	plugin.Param("vargs", &vargs)
	plugin.Parse()

	// Iterate over rcs and svcs
	for _, rc := range vargs.ReplicationControllers {
		artifact, e := readArtifactFromFile(workspace, rc, vargs.ApiServer, vargs.Namespace)
		if b, _ := existsArtifact(artifact, token); b {
			deleteArtifact(artifact, token)
		}
		createArtifact(artifact, vargs.Token)
	}
	for _, rc := range vargs.Services {
		createArtifact(artifact, vargs.Token)
	}
}
<|endoftext|>


<commit_before>/*
   Copyright 2009-2013 Phil Pennock

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package sks_spider

import (
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"os/signal"
	"runtime"
	"sync"
	"syscall"
	"time"
)

var (
	flSpiderStartHost    = flag.String("spider-start-host", "sks-peer.spodhuis.org", "Host to query to start things rolling")
	flListen             = flag.String("listen", "localhost:8001", "port to listen on with web-server")
	flMaintEmail         = flag.String("maint-email", "webmaster@spodhuis.org", "Email address of local maintainer")
	flHostname           = flag.String("hostname", "sks.spodhuis.org", "Hostname to use in generated pages")
	flMyStylesheet       = flag.String("stylesheet", "/styles/sks-peers.css", "CSS Style sheet to use")
	flSksMembershipFile  = flag.String("sks-membership-file", "/var/sks/membership", "SKS Membership file")
	flSksPortRecon       = flag.Int("sks-port-recon", 11370, "Default SKS recon port")
	flSksPortHkp         = flag.Int("sks-port-hkp", 11371, "Default SKS HKP port")
	flTimeoutStatsFetch  = flag.Int("timeout-stats-fetch", 30, "Timeout for fetching stats from a remote server")
	flCountriesZone      = flag.String("countries-zone", "zz.countries.nerd.dk.", "DNS zone for determining IP locations")
	flKeysSanityMin      = flag.Int("keys-sanity-min", 3100000, "Minimum number of keys that's sane, or we're broken")
	flKeysDailyJitter    = flag.Int("keys-daily-jitter", 500, "Max daily jitter in key count")
	flScanIntervalSecs   = flag.Int("scan-interval", 3600*8, "How often to trigger a scan")
	flScanIntervalJitter = flag.Int("scan-interval-jitter", 120, "Jitter in scan interval")
	flLogFile            = flag.String("log-file", "sksdaemon.log", "Where to write logfiles")
	flLogStdout          = flag.Bool("log-stdout", false, "Log to stdout instead of log-file")
	flJsonDump           = flag.String("json-dump", "", "File to dump JSON of spidered hosts to")
	flJsonLoad           = flag.String("json-load", "", "File to load JSON hosts from instead of spidering")
	flJsonPersistPath    = flag.String("json-persist", "", "File to load at startup if exists, and write to at SIGUSR1")
	flStartedFlagfile    = flag.String("started-file", "", "Create this file after started and running")
	flHttpFetchTimeout   = flag.Duration("http-fetch-timeout", 2*time.Minute, "Timeout for HTTP fetch from SKS servers")
)

var serverHeadersNative = map[string]bool{
	"sks_www": true,
	"gnuks":   true,
}
var defaultSoftware = "SKS"

// People put dumb things in their membership files
var blacklistedQueryHosts = []string{
	"localhost",
	"127.0.0.1",
	"::1",
}

var Log *log.Logger

func setupLogging() {
	if *flLogStdout {
		Log = log.New(os.Stdout, "", log.LstdFlags|log.Lshortfile)
		return
	}
	fh, err := os.OpenFile(*flLogFile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to open logfile \"%s\": %s\n", *flLogFile, err)
		os.Exit(1)
	}
	Log = log.New(fh, "", log.LstdFlags|log.Lshortfile)
}

type PersistedHostInfo struct {
	HostMap      HostMap
	AliasMap     AliasMap
	IPCountryMap IPCountryMap
	Sorted       []string
	DepthSorted  []string
	Graph        *HostGraph
	Timestamp    time.Time
}

var (
	currentHostInfo    *PersistedHostInfo
	currentHostMapLock sync.RWMutex
)

func GetCurrentPersisted() *PersistedHostInfo {
	currentHostMapLock.RLock()
	defer currentHostMapLock.RUnlock()
	return currentHostInfo
}

func GetCurrentHosts() HostMap {
	currentHostMapLock.RLock()
	defer currentHostMapLock.RUnlock()
	if currentHostInfo == nil {
		return nil
	}
	return currentHostInfo.HostMap
}

func GetCurrentHostlist() []string {
	currentHostMapLock.RLock()
	defer currentHostMapLock.RUnlock()
	if currentHostInfo == nil {
		return nil
	}
	return currentHostInfo.Sorted
}

func SetCurrentPersisted(p *PersistedHostInfo) {
	p.Timestamp = time.Now()
	p.LogInformation()
	currentHostMapLock.Lock()
	defer currentHostMapLock.Unlock()
	currentHostInfo = p
}

func normaliseMeshAndSet(spider *Spider, dumpJson bool) {
	go func(s *Spider) {
		persisted := GeneratePersistedInformation(s)
		SetCurrentPersisted(persisted)
		persisted.UpdateStatsCounters(spider)
		runtime.GC()
		if dumpJson && *flJsonDump != "" {
			Log.Printf("Saving JSON to \"%s\"", *flJsonDump)
			err := persisted.HostMap.DumpJSONToFile(*flJsonDump)
			if err != nil {
				Log.Printf("Error saving JSON to \"%s\": %s", *flJsonDump, err)
				// continue anyway
			}
			runtime.GC()
		}
	}(spider)
}

func respiderPeriodically() {
	for {
		var delay time.Duration = time.Duration(*flScanIntervalSecs) * time.Second
		if *flScanIntervalJitter > 0 {
			jitter := rand.Int63n(int64(*flScanIntervalJitter) * int64(time.Second))
			jitter -= int64(*flScanIntervalJitter) * int64(time.Second) / 2
			delay += time.Duration(jitter)
		}
		minDelay := time.Minute * 30
		if delay < minDelay {
			Log.Printf("respider period too low, capping %d up to %d", delay, minDelay)
			delay = minDelay
		}
		Log.Printf("Sleeping %s before next respider", delay)
		time.Sleep(delay)
		Log.Printf("Awoken!  Time to spider.")
		var spider *Spider
		func() {
			spider = StartSpider()
			defer func(sp *Spider) {
				if r := recover(); r != nil {
					Log.Printf("Spider paniced: %s", r)
				}
				sp.Terminate()
			}(spider)
			spider.AddHost(*flSpiderStartHost, 0)
			spider.Wait()
		}()
		normaliseMeshAndSet(spider, false)
	}
}

var httpServing sync.WaitGroup

func startHttpServing() {
	Log.Printf("Will Listen on <%s>", *flListen)
	server := setupHttpServer(*flListen)
	err := server.ListenAndServe()
	if err != nil {
		Log.Printf("ListenAndServe(%s): %s", *flListen, err)
	}
	httpServing.Done()
}

func shutdownRunner(ch <-chan os.Signal) {
	signal, ok := <-ch
	if !ok {
		return
	}
	persisted := GetCurrentPersisted()
	if persisted != nil {
		Log.Printf("Received signal %s; saving JSON to \"%s\"", signal, *flJsonPersistPath)
		err := persisted.HostMap.DumpJSONToFile(*flJsonPersistPath)
		if err != nil {
			Log.Printf("Error saving shutdown JSON: %s", err)
		} else {
			Log.Print("Wrote shutdown JSON")
		}
	}
	httpServing.Done()
}

func Main() {
	flag.Parse()

	if *flScanIntervalJitter < 0 {
		fmt.Fprintf(os.Stderr, "Bad jitter, must be >= 0 [got: %d]\n", *flScanIntervalJitter)
		os.Exit(1)
	}

	setupLogging()
	Log.Printf("started")

	httpServing.Add(1)
	go startHttpServing()

	if *flJsonPersistPath != "" {
		if _, err := os.Stat(*flJsonPersistPath); err == nil {
			if *flJsonLoad == "" {
				*flJsonLoad = *flJsonPersistPath
			}
		}
	}

	var doneRespider bool

	if *flJsonLoad != "" {
		Log.Printf("Loading hosts from \"%s\" instead of spidering", *flJsonLoad)
		hostmap, err := LoadJSONFromFile(*flJsonLoad)
		if err != nil {
			Log.Fatalf("Failed to load JSON from \"%s\": %s", *flJsonLoad, err)
		}
		Log.Printf("Loaded %d hosts from JSON", len(hostmap))
		hostnames := GenerateHostlistSorted(hostmap)
		countryMap := GetFreshCountryForHostmap(hostmap)
		aliasMap := GetAliasMapForHostmap(hostmap)
		SetCurrentPersisted(&PersistedHostInfo{
			HostMap:      hostmap,
			AliasMap:     aliasMap,
			IPCountryMap: countryMap,
			Sorted:       hostnames,
			DepthSorted:  GenerateDepthSorted(hostmap),
			Graph:        GenerateGraph(hostnames, hostmap, aliasMap),
		})
	} else {
		spider := StartSpider()
		spider.AddHost(*flSpiderStartHost, 0)
		spider.Wait()
		spider.Terminate()
		Log.Printf("Spidering complete")
		normaliseMeshAndSet(spider, true)
		go respiderPeriodically()
		doneRespider = true
	}

	if *flJsonPersistPath != "" {
		signalChan := make(chan os.Signal)
		if !doneRespider {
			go respiderPeriodically()
		}
		go shutdownRunner(signalChan)
		// Warning: Unix-specific, need to figure out how to make this signal-handling
		// replacable with another notification mechanism which is system-local and easily
		// triggered from an rc script
		signal.Notify(signalChan, syscall.SIGUSR1)
	}

	if *flStartedFlagfile != "" {
		fh, err := os.Create(*flStartedFlagfile)
		if err == nil {
			fmt.Fprintf(fh, "Started %s\n", os.Args[0])
			err = fh.Close()
			if err != nil {
				Log.Printf("Error in close(%s): %s", *flStartedFlagfile, err)
			}
		} else {
			Log.Printf("Failed to create -started-file: %s", err)
		}
	}

	httpServing.Wait()
}
<commit_msg>Clarify start-up log message.<commit_after>/*
   Copyright 2009-2013 Phil Pennock

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package sks_spider

import (
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"os/signal"
	"runtime"
	"sync"
	"syscall"
	"time"
)

var (
	flSpiderStartHost    = flag.String("spider-start-host", "sks-peer.spodhuis.org", "Host to query to start things rolling")
	flListen             = flag.String("listen", "localhost:8001", "port to listen on with web-server")
	flMaintEmail         = flag.String("maint-email", "webmaster@spodhuis.org", "Email address of local maintainer")
	flHostname           = flag.String("hostname", "sks.spodhuis.org", "Hostname to use in generated pages")
	flMyStylesheet       = flag.String("stylesheet", "/styles/sks-peers.css", "CSS Style sheet to use")
	flSksMembershipFile  = flag.String("sks-membership-file", "/var/sks/membership", "SKS Membership file")
	flSksPortRecon       = flag.Int("sks-port-recon", 11370, "Default SKS recon port")
	flSksPortHkp         = flag.Int("sks-port-hkp", 11371, "Default SKS HKP port")
	flTimeoutStatsFetch  = flag.Int("timeout-stats-fetch", 30, "Timeout for fetching stats from a remote server")
	flCountriesZone      = flag.String("countries-zone", "zz.countries.nerd.dk.", "DNS zone for determining IP locations")
	flKeysSanityMin      = flag.Int("keys-sanity-min", 3100000, "Minimum number of keys that's sane, or we're broken")
	flKeysDailyJitter    = flag.Int("keys-daily-jitter", 500, "Max daily jitter in key count")
	flScanIntervalSecs   = flag.Int("scan-interval", 3600*8, "How often to trigger a scan")
	flScanIntervalJitter = flag.Int("scan-interval-jitter", 120, "Jitter in scan interval")
	flLogFile            = flag.String("log-file", "sksdaemon.log", "Where to write logfiles")
	flLogStdout          = flag.Bool("log-stdout", false, "Log to stdout instead of log-file")
	flJsonDump           = flag.String("json-dump", "", "File to dump JSON of spidered hosts to")
	flJsonLoad           = flag.String("json-load", "", "File to load JSON hosts from instead of spidering")
	flJsonPersistPath    = flag.String("json-persist", "", "File to load at startup if exists, and write to at SIGUSR1")
	flStartedFlagfile    = flag.String("started-file", "", "Create this file after started and running")
	flHttpFetchTimeout   = flag.Duration("http-fetch-timeout", 2*time.Minute, "Timeout for HTTP fetch from SKS servers")
)

var serverHeadersNative = map[string]bool{
	"sks_www": true,
	"gnuks":   true,
}
var defaultSoftware = "SKS"

// People put dumb things in their membership files
var blacklistedQueryHosts = []string{
	"localhost",
	"127.0.0.1",
	"::1",
}

var Log *log.Logger

func setupLogging() {
	if *flLogStdout {
		Log = log.New(os.Stdout, "", log.LstdFlags|log.Lshortfile)
		return
	}
	fh, err := os.OpenFile(*flLogFile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to open logfile \"%s\": %s\n", *flLogFile, err)
		os.Exit(1)
	}
	Log = log.New(fh, "", log.LstdFlags|log.Lshortfile)
}

type PersistedHostInfo struct {
	HostMap      HostMap
	AliasMap     AliasMap
	IPCountryMap IPCountryMap
	Sorted       []string
	DepthSorted  []string
	Graph        *HostGraph
	Timestamp    time.Time
}

var (
	currentHostInfo    *PersistedHostInfo
	currentHostMapLock sync.RWMutex
)

func GetCurrentPersisted() *PersistedHostInfo {
	currentHostMapLock.RLock()
	defer currentHostMapLock.RUnlock()
	return currentHostInfo
}

func GetCurrentHosts() HostMap {
	currentHostMapLock.RLock()
	defer currentHostMapLock.RUnlock()
	if currentHostInfo == nil {
		return nil
	}
	return currentHostInfo.HostMap
}

func GetCurrentHostlist() []string {
	currentHostMapLock.RLock()
	defer currentHostMapLock.RUnlock()
	if currentHostInfo == nil {
		return nil
	}
	return currentHostInfo.Sorted
}

func SetCurrentPersisted(p *PersistedHostInfo) {
	p.Timestamp = time.Now()
	p.LogInformation()
	currentHostMapLock.Lock()
	defer currentHostMapLock.Unlock()
	currentHostInfo = p
}

func normaliseMeshAndSet(spider *Spider, dumpJson bool) {
	go func(s *Spider) {
		persisted := GeneratePersistedInformation(s)
		SetCurrentPersisted(persisted)
		persisted.UpdateStatsCounters(spider)
		runtime.GC()
		if dumpJson && *flJsonDump != "" {
			Log.Printf("Saving JSON to \"%s\"", *flJsonDump)
			err := persisted.HostMap.DumpJSONToFile(*flJsonDump)
			if err != nil {
				Log.Printf("Error saving JSON to \"%s\": %s", *flJsonDump, err)
				// continue anyway
			}
			runtime.GC()
		}
	}(spider)
}

func respiderPeriodically() {
	for {
		var delay time.Duration = time.Duration(*flScanIntervalSecs) * time.Second
		if *flScanIntervalJitter > 0 {
			jitter := rand.Int63n(int64(*flScanIntervalJitter) * int64(time.Second))
			jitter -= int64(*flScanIntervalJitter) * int64(time.Second) / 2
			delay += time.Duration(jitter)
		}
		minDelay := time.Minute * 30
		if delay < minDelay {
			Log.Printf("respider period too low, capping %d up to %d", delay, minDelay)
			delay = minDelay
		}
		Log.Printf("Sleeping %s before next respider", delay)
		time.Sleep(delay)
		Log.Printf("Awoken!  Time to spider.")
		var spider *Spider
		func() {
			spider = StartSpider()
			defer func(sp *Spider) {
				if r := recover(); r != nil {
					Log.Printf("Spider paniced: %s", r)
				}
				sp.Terminate()
			}(spider)
			spider.AddHost(*flSpiderStartHost, 0)
			spider.Wait()
		}()
		normaliseMeshAndSet(spider, false)
	}
}

var httpServing sync.WaitGroup

func startHttpServing() {
	Log.Printf("Will Listen on <%s>", *flListen)
	server := setupHttpServer(*flListen)
	err := server.ListenAndServe()
	if err != nil {
		Log.Printf("ListenAndServe(%s): %s", *flListen, err)
	}
	httpServing.Done()
}

func shutdownRunner(ch <-chan os.Signal) {
	signal, ok := <-ch
	if !ok {
		return
	}
	persisted := GetCurrentPersisted()
	if persisted != nil {
		Log.Printf("Received signal %s; saving JSON to \"%s\"", signal, *flJsonPersistPath)
		err := persisted.HostMap.DumpJSONToFile(*flJsonPersistPath)
		if err != nil {
			Log.Printf("Error saving shutdown JSON: %s", err)
		} else {
			Log.Print("Wrote shutdown JSON")
		}
	}
	httpServing.Done()
}

func Main() {
	flag.Parse()

	if *flScanIntervalJitter < 0 {
		fmt.Fprintf(os.Stderr, "Bad jitter, must be >= 0 [got: %d]\n", *flScanIntervalJitter)
		os.Exit(1)
	}

	setupLogging()
	Log.Printf("started")

	httpServing.Add(1)
	go startHttpServing()

	if *flJsonPersistPath != "" {
		if _, err := os.Stat(*flJsonPersistPath); err == nil {
			if *flJsonLoad == "" {
				*flJsonLoad = *flJsonPersistPath
			}
		}
	}

	var doneRespider bool

	if *flJsonLoad != "" {
		Log.Printf("Loading hosts from \"%s\" instead of spidering", *flJsonLoad)
		hostmap, err := LoadJSONFromFile(*flJsonLoad)
		if err != nil {
			Log.Fatalf("Failed to load JSON from \"%s\": %s", *flJsonLoad, err)
		}
		Log.Printf("Loaded %d hosts from JSON", len(hostmap))
		hostnames := GenerateHostlistSorted(hostmap)
		countryMap := GetFreshCountryForHostmap(hostmap)
		aliasMap := GetAliasMapForHostmap(hostmap)
		SetCurrentPersisted(&PersistedHostInfo{
			HostMap:      hostmap,
			AliasMap:     aliasMap,
			IPCountryMap: countryMap,
			Sorted:       hostnames,
			DepthSorted:  GenerateDepthSorted(hostmap),
			Graph:        GenerateGraph(hostnames, hostmap, aliasMap),
		})
	} else {
		spider := StartSpider()
		spider.AddHost(*flSpiderStartHost, 0)
		spider.Wait()
		spider.Terminate()
		Log.Printf("Start-up initial spidering complete")
		normaliseMeshAndSet(spider, true)
		go respiderPeriodically()
		doneRespider = true
	}

	if *flJsonPersistPath != "" {
		signalChan := make(chan os.Signal)
		if !doneRespider {
			go respiderPeriodically()
		}
		go shutdownRunner(signalChan)
		// Warning: Unix-specific, need to figure out how to make this signal-handling
		// replacable with another notification mechanism which is system-local and easily
		// triggered from an rc script
		signal.Notify(signalChan, syscall.SIGUSR1)
	}

	if *flStartedFlagfile != "" {
		fh, err := os.Create(*flStartedFlagfile)
		if err == nil {
			fmt.Fprintf(fh, "Started %s\n", os.Args[0])
			err = fh.Close()
			if err != nil {
				Log.Printf("Error in close(%s): %s", *flStartedFlagfile, err)
			}
		} else {
			Log.Printf("Failed to create -started-file: %s", err)
		}
	}

	httpServing.Wait()
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"strings"
	"syscall"
)

var address = flag.String("address", ":8080", "address to listen on")
var repositories = flag.String("repositories", "scraperwiki/tang", "colon separated list of repositories to watch")
var allowedPushers = flag.String("allowed-pushers", "drj11:pwaller", "list of people allowed")

var allowedPushersSet = map[string]bool{}

func init() {
	flag.Parse()
	for _, who := range strings.Split(*allowedPushers, ":") {
		allowedPushersSet[who] = true
	}
}

func check(err error) {
	if err != nil {
		panic(err)
	}
}

func main() {

	go func() {
		http.HandleFunc("/hook", handleHook)
		log.Println("Listening on:", *address)
		log.Fatal(http.ListenAndServe(*address, nil))
	}()

	configureHooks()

	sig := make(chan os.Signal)
	signal.Notify(sig, syscall.SIGHUP, syscall.SIGINT)
	<-sig
	signal.Stop(sig)

	log.Print("HUPPING!")

	exe, err := os.Readlink("/proc/self/exe")
	check(err)

	err = syscall.Exec(exe, os.Args, os.Environ())
	check(err)
}

func configureHooks() {

	if *repositories == "" {
		return
	}

	github_user := os.Getenv("GITHUB_USER")
	github_password := os.Getenv("GITHUB_PASSWORD")

	json := `{
	"name": "web",
	"config": {"url": "http://services.scraperwiki.com/hook",
		"content_type": "json"},
	"events": ["push", "issues", "issue_comment",
		"commit_comment", "create", "delete",
		"pull_request", "pull_request_review_comment",
		"gollum", "watch", "release", "fork", "member",
		"public", "team_add", "status"],
	"active": true
	}`

	endpoint := "https://" + github_user + ":" + github_password + "@" + "api.github.com"

	repos := strings.Split(*repositories, ":")

	for _, repo := range repos {
		log.Print("Repo: ", repo)

		buffer := strings.NewReader(json)
		resp, err := http.Post(endpoint+"/repos/"+repo+"/hooks", "application/json", buffer)
		check(err)

		log.Println("Rate Limit:", resp.Header["X-Ratelimit-Remaining"][0])

		switch resp.StatusCode {
		default:
			response, err := ioutil.ReadAll(resp.Body)
			check(err)

			log.Print(string(response))

		case 422:
			log.Println("Already hooked for", repo)
		}
	}

}

type Repository struct {
	Name         string `json:"name"`
	Url          string `json:"url"`
	Organization string `json:"organization"`
}

type Pusher struct {
	Name string `json:"name"`
}

type PushEvent struct {
	Ref        string     `json:"ref"`
	Repository Repository `json:"repository"`
	After      string     `json:"after"`
	Pusher     Pusher     `json:"pusher"`
}

func handleEvent(eventType string, document []byte) (err error) {

	log.Println("Incoming request:", string(document))

	switch eventType {
	case "push":
		var event PushEvent
		err = json.Unmarshal(document, &event)
		if err != nil {
			return
		}

		err = eventPush(event)
		if err != nil {
			return
		}

	default:
		log.Println("Unhandled event:", eventType)
	}

	return
}

func handleHook(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "OK\n")

	request, err := ioutil.ReadAll(r.Body)
	check(err)

	var buf bytes.Buffer
	r.Header.Write(&buf)
	log.Println("Incoming request headers: ", string(buf.Bytes()))

	buf.Reset()
	err = json.Indent(&buf, request, "", "  ")
	check(err)

	eventType := r.Header["X-Github-Event"][0]
	data := buf.Bytes()

	err = handleEvent(eventType, data)
	check(err)
}

func Command(workdir, command string, args ...string) *exec.Cmd {
	// log.Printf("wd = %s cmd = %s, args = %q", workdir, command, append([]string{}, args...))
	cmd := exec.Command(command, args...)
	cmd.Dir = workdir
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd
}

const GIT_BASE_DIR = "repo"

var (
	ErrEmptyRepoName         = errors.New("Empty repository name")
	ErrEmptyRepoOrganization = errors.New("Empty repository organization")
	ErrUserNotAllowed        = errors.New("User not in the allowed set")
)

// Creates or updates a mirror of `url` at `git_dir` using `git clone --mirror`
func gitLocalMirror(url, git_dir string) (err error) {

	err = os.MkdirAll(git_dir, 0777)
	if err != nil {
		return
	}

	err = Command(".", "git", "clone", "-q", "--mirror", url, git_dir).Run()

	if err == nil {
		log.Println("Cloned", url)

	} else if _, ok := err.(*exec.ExitError); ok {

		// Try "git remote update"
		err = Command(git_dir, "git", "fetch").Run()

		if err != nil {
			// git fetch where there is no update is exit status 1.
			if err.Error() != "exit status 1" {
				return
			}
		}

		log.Println("Remote updated", url)

	} else {
		return
	}

	return
}

func gitCheckout(git_dir, checkout_dir, ref string) (err error) {

	err = os.MkdirAll(path.Join(git_dir, checkout_dir), 0777)
	if err != nil {
		return
	}

	log.Println("Populating", checkout_dir)

	args := []string{"--work-tree", checkout_dir, "checkout", ref, "."}
	err = Command(git_dir, "git", args...).Run()
	if err != nil {
		return
	}

	return
}

func runTang(path string) (err error) {
	return Command(path, "./tang.hook").Run()
}

func eventPush(event PushEvent) (err error) {
	if event.Repository.Name == "" {
		return ErrEmptyRepoName
	}

	if event.Repository.Organization == "" {
		return ErrEmptyRepoOrganization
	}

	if _, ok := allowedPushersSet[event.Pusher.Name]; !ok {
		log.Printf("Ignoring %q, not allowed", event.Pusher.Name)
		return ErrUserNotAllowed
	}

	ref := event.Ref
	url := event.Repository.Url
	after := event.After

	log.Println("Push to", url, ref, "after", after)

	// The name of the subdirectory where the git
	// mirror is (or will appear, if it hasn't been
	// cloned yet).
	git_dir := path.Join(GIT_BASE_DIR, event.Repository.Organization,
		event.Repository.Name)
	err = gitLocalMirror(url, git_dir)
	if err != nil {
		return
	}

	checkout_dir := path.Join("checkout", after)
	err = gitCheckout(git_dir, checkout_dir, after)
	if err != nil {
		return
	}
	log.Println("Created", checkout_dir)

	err = runTang(path.Join(git_dir, checkout_dir))
	if err != nil {
		return
	}

	return
}
<commit_msg>Re-enable command logging<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"strings"
	"syscall"
)

var address = flag.String("address", ":8080", "address to listen on")
var repositories = flag.String("repositories", "scraperwiki/tang", "colon separated list of repositories to watch")
var allowedPushers = flag.String("allowed-pushers", "drj11:pwaller", "list of people allowed")

var allowedPushersSet = map[string]bool{}

func init() {
	flag.Parse()
	for _, who := range strings.Split(*allowedPushers, ":") {
		allowedPushersSet[who] = true
	}
}

func check(err error) {
	if err != nil {
		panic(err)
	}
}

func main() {

	go func() {
		http.HandleFunc("/hook", handleHook)
		log.Println("Listening on:", *address)
		log.Fatal(http.ListenAndServe(*address, nil))
	}()

	configureHooks()

	sig := make(chan os.Signal)
	signal.Notify(sig, syscall.SIGHUP, syscall.SIGINT)
	<-sig
	signal.Stop(sig)

	log.Print("HUPPING!")

	exe, err := os.Readlink("/proc/self/exe")
	check(err)

	err = syscall.Exec(exe, os.Args, os.Environ())
	check(err)
}

func configureHooks() {

	if *repositories == "" {
		return
	}

	github_user := os.Getenv("GITHUB_USER")
	github_password := os.Getenv("GITHUB_PASSWORD")

	json := `{
	"name": "web",
	"config": {"url": "http://services.scraperwiki.com/hook",
		"content_type": "json"},
	"events": ["push", "issues", "issue_comment",
		"commit_comment", "create", "delete",
		"pull_request", "pull_request_review_comment",
		"gollum", "watch", "release", "fork", "member",
		"public", "team_add", "status"],
	"active": true
	}`

	endpoint := "https://" + github_user + ":" + github_password + "@" + "api.github.com"

	repos := strings.Split(*repositories, ":")

	for _, repo := range repos {
		log.Print("Repo: ", repo)

		buffer := strings.NewReader(json)
		resp, err := http.Post(endpoint+"/repos/"+repo+"/hooks", "application/json", buffer)
		check(err)

		log.Println("Rate Limit:", resp.Header["X-Ratelimit-Remaining"][0])

		switch resp.StatusCode {
		default:
			response, err := ioutil.ReadAll(resp.Body)
			check(err)

			log.Print(string(response))

		case 422:
			log.Println("Already hooked for", repo)
		}
	}

}

type Repository struct {
	Name         string `json:"name"`
	Url          string `json:"url"`
	Organization string `json:"organization"`
}

type Pusher struct {
	Name string `json:"name"`
}

type PushEvent struct {
	Ref        string     `json:"ref"`
	Repository Repository `json:"repository"`
	After      string     `json:"after"`
	Pusher     Pusher     `json:"pusher"`
}

func handleEvent(eventType string, document []byte) (err error) {

	log.Println("Incoming request:", string(document))

	switch eventType {
	case "push":
		var event PushEvent
		err = json.Unmarshal(document, &event)
		if err != nil {
			return
		}

		err = eventPush(event)
		if err != nil {
			return
		}

	default:
		log.Println("Unhandled event:", eventType)
	}

	return
}

func handleHook(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "OK\n")

	request, err := ioutil.ReadAll(r.Body)
	check(err)

	var buf bytes.Buffer
	r.Header.Write(&buf)
	log.Println("Incoming request headers: ", string(buf.Bytes()))

	buf.Reset()
	err = json.Indent(&buf, request, "", "  ")
	check(err)

	eventType := r.Header["X-Github-Event"][0]
	data := buf.Bytes()

	err = handleEvent(eventType, data)
	check(err)
}

func Command(workdir, command string, args ...string) *exec.Cmd {
	log.Printf("wd = %s cmd = %s, args = %q", workdir, command, append([]string{}, args...))
	cmd := exec.Command(command, args...)
	cmd.Dir = workdir
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd
}

const GIT_BASE_DIR = "repo"

var (
	ErrEmptyRepoName         = errors.New("Empty repository name")
	ErrEmptyRepoOrganization = errors.New("Empty repository organization")
	ErrUserNotAllowed        = errors.New("User not in the allowed set")
)

// Creates or updates a mirror of `url` at `git_dir` using `git clone --mirror`
func gitLocalMirror(url, git_dir string) (err error) {

	err = os.MkdirAll(git_dir, 0777)
	if err != nil {
		return
	}

	err = Command(".", "git", "clone", "-q", "--mirror", url, git_dir).Run()

	if err == nil {
		log.Println("Cloned", url)

	} else if _, ok := err.(*exec.ExitError); ok {

		// Try "git remote update"
		err = Command(git_dir, "git", "fetch").Run()

		if err != nil {
			// git fetch where there is no update is exit status 1.
			if err.Error() != "exit status 1" {
				return
			}
		}

		log.Println("Remote updated", url)

	} else {
		return
	}

	return
}

func gitCheckout(git_dir, checkout_dir, ref string) (err error) {

	err = os.MkdirAll(path.Join(git_dir, checkout_dir), 0777)
	if err != nil {
		return
	}

	log.Println("Populating", checkout_dir)

	args := []string{"--work-tree", checkout_dir, "checkout", ref, "."}
	err = Command(git_dir, "git", args...).Run()
	if err != nil {
		return
	}

	return
}

func runTang(path string) (err error) {
	return Command(path, "./tang.hook").Run()
}

func eventPush(event PushEvent) (err error) {
	if event.Repository.Name == "" {
		return ErrEmptyRepoName
	}

	if event.Repository.Organization == "" {
		return ErrEmptyRepoOrganization
	}

	if _, ok := allowedPushersSet[event.Pusher.Name]; !ok {
		log.Printf("Ignoring %q, not allowed", event.Pusher.Name)
		return ErrUserNotAllowed
	}

	ref := event.Ref
	url := event.Repository.Url
	after := event.After

	log.Println("Push to", url, ref, "after", after)

	// The name of the subdirectory where the git
	// mirror is (or will appear, if it hasn't been
	// cloned yet).
	git_dir := path.Join(GIT_BASE_DIR, event.Repository.Organization,
		event.Repository.Name)
	err = gitLocalMirror(url, git_dir)
	if err != nil {
		return
	}

	checkout_dir := path.Join("checkout", after)
	err = gitCheckout(git_dir, checkout_dir, after)
	if err != nil {
		return
	}
	log.Println("Created", checkout_dir)

	err = runTang(path.Join(git_dir, checkout_dir))
	if err != nil {
		return
	}

	return
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/cbednarski/mkdeb/deb"
)

func main() {
	args := os.Args

	if len(args) < 2 {
		showUsage()
	}

	switch args[1] {
	case "archs":
		showArchs()
	case "build":
		buildCommand := flag.NewFlagSet("build", flag.ExitOnError)
		version := buildCommand.String("version", "1.0", "Package version")
		target := buildCommand.String("target", "", "Target folder with generated filename")
		buildCommand.Parse(args[2:])
		build(checkConfig(buildCommand.Args()), *version, *target)
	case "init":
		initialize()
	case "validate":
		commandArgs := flag.Args()

		validate(checkConfig(commandArgs))
	default:
		showUsage()
	}
	os.Exit(0)
}

func checkConfig(args []string) string {
	if len(args) < 1 {
		fmt.Printf("Missing config file\n")
		os.Exit(1)
	}
	if len(args) > 1 {
		fmt.Printf("Too many arguments\n")
		os.Exit(1)
	}
	return args[0]
}

// getAbsPaths takes a relative path to a file and returns both the containing
// directory and the absolute path to the file.
//
// Example: cat -> /bin /bin/cat
func getAbsPaths(filename string) (string, string) {
	path, err := filepath.Abs(filename)
	if err != nil {
		fmt.Printf("Can't find %q", filename)
		os.Exit(1)
	}
	dir, _ := filepath.Split(path)
	return dir, path
}

func showArchs() {
	fmt.Printf("mkdeb supported architectures: %s\n", strings.Join(deb.SupportedArchitectures(), ", "))
}

// initialize creates a new mkdeb config. This function is not called init()
// because that has a special meaning in Go.
func initialize() {
	// Get abs path to PWD
	workdir, err := os.Getwd()
	handleError(err)
	workdir, err = filepath.Abs(workdir)
	handleError(err)

	// Get config file name
	target := path.Join(workdir, "mkdeb.json")
	if deb.FileExists(target) {
		handleError(fmt.Errorf("mkdir.json already exists in this directory"))
	}

	// Create config file
	file, err := os.Create(target)
	handleError(err)
	defer file.Close()

	// Create config struct
	projectName := filepath.Base(workdir)
	p := deb.DefaultPackageSpec()
	p.Package = projectName
	p.Maintainer = "Your Name <you@example.com>"
	p.Architecture = "amd64"
	p.Description = projectName + " is an awsome project for..."
	p.Homepage = "https://www.example.com/project"
	p.Files = map[string]string{projectName: "/usr/local/bin/" + projectName}

	data, err := json.MarshalIndent(p, "", "  ")
	handleError(err)

	_, err = file.Write(data)
	handleError(err)
}

func validate(config string) {
	// Change to config path
	back, err := os.Getwd()
	handleError(err)
	workdir, filename := getAbsPaths(config)
	err = os.Chdir(workdir)
	handleError(err)
	defer os.Chdir(back)

	// Validate
	p, err := deb.NewPackageSpecFromFile(filename)
	handleError(err)
	handleError(p.Validate(false))
}

func build(config, version, target string) {
	// Change to config path
	back, err := os.Getwd()
	handleError(err)

	// Get the working directory to cd into and the absolute path to the file
	workdir, abspath := getAbsPaths(config)
	err = os.Chdir(workdir)
	handleError(err)
	defer os.Chdir(back)

	p, err := deb.NewPackageSpecFromFile(abspath)
	handleError(err)

	// Set version
	p.Version = version

	// Set target filename
	if target == "" {
		target = workdir
	} else {
		if !isDir(target) {
			handleError(fmt.Errorf("%q is not a directory", target))
		}
	}

	// Validate
	handleError(p.Validate(true))

	// Build
	handleError(p.Build(target))
	fmt.Printf("Built package %s\n", path.Join(target, p.Filename()))
}

func isDir(path string) bool {
	info, err := os.Stat(path)
	return err == nil && info.IsDir()
}

func handleError(err error) {
	if err != nil {
		fmt.Printf("Error: %s\n", err)
		os.Exit(1)
	}
}

func showUsage() {
	fmt.Print(usage)
	os.Exit(1)
}

const usage = `ABOUT

  mkdeb is a tool for building debian packages

  Report issues or get updates from https://github.com/cbednarski/mkdeb

COMMANDS

  build       Build a package based on the specified config file
  init        Create a new mkdeb config file in the current directory
  archs       List supported CPU architectures
  validate    Validate your config file

BUILD COMMAND

  mkdeb build -version=1.2.0 config.json

  Options:

    -version (required) Package version

    -target (optional) output artifact to this path

    -targetDir (optional) output artifact to this folder, use generated filename

  By default the build artifact

  The build command will change to the directory where the config file is
  located, so paths should always be specified relative to the config file.

PACKAGING CONFIGURATION

  Required Fields

  - package: The name of your package
  - version: Must adhere to debian version syntax.
  - architecture: CPU arch for your binaries, or "all"
  - maintainer: Your Name <email@example.com>
  - description: Brief explanation of your package

  Optional Fields

  - depends: Other packages you depend on. E.g: "python" or "curl (>= 7.0.0)"
  - conflicts: Packages your package are not compatible with
  - breaks: Packages your package breaks
  - replaces: Packages your package replaces
  - homepage: URL to your project homepage or source repository, if you have one

  For more details on how to specify various config options, refer to the
  debian package specification:

  - https://www.debian.org/doc/debian-policy/ch-controlfields.html
  - https://www.debian.org/doc/manuals/debian-faq/ch-pkg_basics.en.html

PACKAGING LAYOUT

  autoPath

  mkdeb will automatically include any files deb-pkg, the default autoPath
  directory. For example, the following files will be automatically included and
  installed to their corresponding paths:

    deb-pkg/etc/mysqld/my.conf  -> /etc/mysqld/my.conf
    deb-pkg/usr/bin/mysqld      -> /usr/bin/mysqld

  You can override this behavior by setting autoPath to - (dash character) and /
  or by using the Files map to create a custom source -> dest mapping.

  Control Scripts

  Control scripts allow you to take action at various stages of your package's
  lifecycle. These are commonly used to create users, start or stop services, or
  perform cleanup.

  By default mkdeb will use any of these files if they are present in deb-pkg:

  - preinst
  - postinst
  - prerm
  - postrm

  You can override this behavior by setting the relevant fields in your config.

BUILD OPTIONS

  The following options change how mkdeb runs when building packages.

  - tempPath: Controls where intermediate files are written during the build.
    This defaults to the system temp directory.

  - upgradeConfigs: Indicates whether apt should replace files under /etc when
    installing a new package version. By default these files are not upgraded.

  - preserveSymlinks: By default contents of symlink targets are copied. This
    option writes symlinks to the archive instead.

LICENSE

  Copyright 2016 Chris Bednarski <banzaimonkey@gmail.com>, and others

  Portions of mkdeb are licensed under the MIT, BSD and Go Licenses. Please
  refer to the project source for full license text and details.

  https://github.com/cbednarski/mkdeb
`
<commit_msg>Removed unused param<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/cbednarski/mkdeb/deb"
)

func main() {
	args := os.Args

	if len(args) < 2 {
		showUsage()
	}

	switch args[1] {
	case "archs":
		showArchs()
	case "build":
		buildCommand := flag.NewFlagSet("build", flag.ExitOnError)
		version := buildCommand.String("version", "1.0", "Package version")
		target := buildCommand.String("target", "", "Target folder with generated filename")
		buildCommand.Parse(args[2:])
		build(checkConfig(buildCommand.Args()), *version, *target)
	case "init":
		initialize()
	case "validate":
		commandArgs := flag.Args()

		validate(checkConfig(commandArgs))
	default:
		showUsage()
	}
	os.Exit(0)
}

func checkConfig(args []string) string {
	if len(args) < 1 {
		fmt.Printf("Missing config file\n")
		os.Exit(1)
	}
	if len(args) > 1 {
		fmt.Printf("Too many arguments\n")
		os.Exit(1)
	}
	return args[0]
}

// getAbsPaths takes a relative path to a file and returns both the containing
// directory and the absolute path to the file.
//
// Example: cat -> /bin /bin/cat
func getAbsPaths(filename string) (string, string) {
	path, err := filepath.Abs(filename)
	if err != nil {
		fmt.Printf("Can't find %q", filename)
		os.Exit(1)
	}
	dir, _ := filepath.Split(path)
	return dir, path
}

func showArchs() {
	fmt.Printf("mkdeb supported architectures: %s\n", strings.Join(deb.SupportedArchitectures(), ", "))
}

// initialize creates a new mkdeb config. This function is not called init()
// because that has a special meaning in Go.
func initialize() {
	// Get abs path to PWD
	workdir, err := os.Getwd()
	handleError(err)
	workdir, err = filepath.Abs(workdir)
	handleError(err)

	// Get config file name
	target := path.Join(workdir, "mkdeb.json")
	if deb.FileExists(target) {
		handleError(fmt.Errorf("mkdir.json already exists in this directory"))
	}

	// Create config file
	file, err := os.Create(target)
	handleError(err)
	defer file.Close()

	// Create config struct
	projectName := filepath.Base(workdir)
	p := deb.DefaultPackageSpec()
	p.Package = projectName
	p.Maintainer = "Your Name <you@example.com>"
	p.Architecture = "amd64"
	p.Description = projectName + " is an awsome project for..."
	p.Homepage = "https://www.example.com/project"
	p.Files = map[string]string{projectName: "/usr/local/bin/" + projectName}

	data, err := json.MarshalIndent(p, "", "  ")
	handleError(err)

	_, err = file.Write(data)
	handleError(err)
}

func validate(config string) {
	// Change to config path
	back, err := os.Getwd()
	handleError(err)
	workdir, filename := getAbsPaths(config)
	err = os.Chdir(workdir)
	handleError(err)
	defer os.Chdir(back)

	// Validate
	p, err := deb.NewPackageSpecFromFile(filename)
	handleError(err)
	handleError(p.Validate(false))
}

func build(config, version, target string) {
	// Change to config path
	back, err := os.Getwd()
	handleError(err)

	// Get the working directory to cd into and the absolute path to the file
	workdir, abspath := getAbsPaths(config)
	err = os.Chdir(workdir)
	handleError(err)
	defer os.Chdir(back)

	p, err := deb.NewPackageSpecFromFile(abspath)
	handleError(err)

	// Set version
	p.Version = version

	// Set target filename
	if target == "" {
		target = workdir
	} else {
		if !isDir(target) {
			handleError(fmt.Errorf("%q is not a directory", target))
		}
	}

	// Validate
	handleError(p.Validate(true))

	// Build
	handleError(p.Build(target))
	fmt.Printf("Built package %s\n", path.Join(target, p.Filename()))
}

func isDir(path string) bool {
	info, err := os.Stat(path)
	return err == nil && info.IsDir()
}

func handleError(err error) {
	if err != nil {
		fmt.Printf("Error: %s\n", err)
		os.Exit(1)
	}
}

func showUsage() {
	fmt.Print(usage)
	os.Exit(1)
}

const usage = `ABOUT

  mkdeb is a tool for building debian packages

  Report issues or get updates from https://github.com/cbednarski/mkdeb

COMMANDS

  build       Build a package based on the specified config file
  init        Create a new mkdeb config file in the current directory
  archs       List supported CPU architectures
  validate    Validate your config file

BUILD COMMAND

  mkdeb build -version=1.2.0 config.json

  Options:

    -version (required) Package version

    -target (optional) output artifact to this path

  By default the build artifact

  The build command will change to the directory where the config file is
  located, so paths should always be specified relative to the config file.

PACKAGING CONFIGURATION

  Required Fields

  - package: The name of your package
  - version: Must adhere to debian version syntax.
  - architecture: CPU arch for your binaries, or "all"
  - maintainer: Your Name <email@example.com>
  - description: Brief explanation of your package

  Optional Fields

  - depends: Other packages you depend on. E.g: "python" or "curl (>= 7.0.0)"
  - conflicts: Packages your package are not compatible with
  - breaks: Packages your package breaks
  - replaces: Packages your package replaces
  - homepage: URL to your project homepage or source repository, if you have one

  For more details on how to specify various config options, refer to the
  debian package specification:

  - https://www.debian.org/doc/debian-policy/ch-controlfields.html
  - https://www.debian.org/doc/manuals/debian-faq/ch-pkg_basics.en.html

PACKAGING LAYOUT

  autoPath

  mkdeb will automatically include any files deb-pkg, the default autoPath
  directory. For example, the following files will be automatically included and
  installed to their corresponding paths:

    deb-pkg/etc/mysqld/my.conf  -> /etc/mysqld/my.conf
    deb-pkg/usr/bin/mysqld      -> /usr/bin/mysqld

  You can override this behavior by setting autoPath to - (dash character) and /
  or by using the Files map to create a custom source -> dest mapping.

  Control Scripts

  Control scripts allow you to take action at various stages of your package's
  lifecycle. These are commonly used to create users, start or stop services, or
  perform cleanup.

  By default mkdeb will use any of these files if they are present in deb-pkg:

  - preinst
  - postinst
  - prerm
  - postrm

  You can override this behavior by setting the relevant fields in your config.

BUILD OPTIONS

  The following options change how mkdeb runs when building packages.

  - tempPath: Controls where intermediate files are written during the build.
    This defaults to the system temp directory.

  - upgradeConfigs: Indicates whether apt should replace files under /etc when
    installing a new package version. By default these files are not upgraded.

  - preserveSymlinks: By default contents of symlink targets are copied. This
    option writes symlinks to the archive instead.

LICENSE

  Copyright 2016 Chris Bednarski <banzaimonkey@gmail.com>, and others

  Portions of mkdeb are licensed under the MIT, BSD and Go Licenses. Please
  refer to the project source for full license text and details.

  https://github.com/cbednarski/mkdeb
`
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"encoding/json"

	"github.com/mailgun/kafka-pixy/Godeps/_workspace/src/github.com/mailgun/log"
	"github.com/mailgun/kafka-pixy/pixy"
)

const (
	defaultBrokers    = "localhost:9092"
	defaultUnixAddr   = "/var/run/kafka-pixy.sock"
	defaultPIDFile    = "/var/run/kafka-pixy.pid"
	defaultLoggingCfg = `[{"name": "console", "severity": "info"}]`
)

var (
	serviceCfg     pixy.ServiceCfg
	pidFile        string
	loggingJSONCfg string
)

func init() {
	flag.StringVar(&serviceCfg.UnixAddr, "unixAddr", defaultUnixAddr,
		"Unix domain socket address that the HTTP API should listen on")
	flag.StringVar(&serviceCfg.TCPAddr, "tcpAddr", "",
		"TCP address that the HTTP API should listen on")
	b := flag.String("brokers", defaultBrokers, "Comma separated list of brokers")
	flag.StringVar(&pidFile, "pidFile", defaultPIDFile, "Path to the PID file")
	flag.StringVar(&loggingJSONCfg, "logging", defaultLoggingCfg, "Logging configuration")
	flag.Parse()
	serviceCfg.BrokerAddrs = strings.Split(*b, ",")
}

func main() {
	if err := initLogging(); err != nil {
		fmt.Printf("Failed to initialize logger, cause=(%v)\n", err)
		os.Exit(1)
	}

	if err := writePID(pidFile); err != nil {
		log.Errorf("Failed to write PID file, cause=(%v)", err)
		os.Exit(1)
	}

	log.Infof("Starting with config: %+v", serviceCfg)
	svc, err := pixy.SpawnService(&serviceCfg)
	if err != nil {
		log.Errorf("Failed to start service, cause=(%v)", err)
		os.Exit(1)
	}

	// Spawn OS signal listener to ensure graceful stop.
	osSigCh := make(chan os.Signal)
	signal.Notify(osSigCh, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM)
	go func() {
		<-osSigCh
		svc.Stop()
	}()

	svc.Wait4Stop()
}

func initLogging() error {
	var loggingCfg []log.Config
	if err := json.Unmarshal([]byte(loggingJSONCfg), &loggingCfg); err != nil {
		return fmt.Errorf("failed to parse logger config, cause=(%v)", err)
	}
	if err := log.InitWithConfig(loggingCfg...); err != nil {
		return err
	}
	return nil
}

func writePID(path string) error {
	pid := os.Getpid()
	return ioutil.WriteFile(path, []byte(fmt.Sprint(pid)), 0644)
}
<commit_msg>Clean up abandoned unix socket file (fixes #4)<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"encoding/json"

	"github.com/mailgun/kafka-pixy/Godeps/_workspace/src/github.com/mailgun/log"
	"github.com/mailgun/kafka-pixy/pixy"
)

const (
	defaultBrokers    = "localhost:9092"
	defaultUnixAddr   = "/var/run/kafka-pixy.sock"
	defaultPIDFile    = "/var/run/kafka-pixy.pid"
	defaultLoggingCfg = `[{"name": "console", "severity": "info"}]`
)

var (
	serviceCfg     pixy.ServiceCfg
	pidFile        string
	loggingJSONCfg string
)

func init() {
	flag.StringVar(&serviceCfg.UnixAddr, "unixAddr", defaultUnixAddr,
		"Unix domain socket address that the HTTP API should listen on")
	flag.StringVar(&serviceCfg.TCPAddr, "tcpAddr", "",
		"TCP address that the HTTP API should listen on")
	b := flag.String("brokers", defaultBrokers, "Comma separated list of brokers")
	flag.StringVar(&pidFile, "pidFile", defaultPIDFile, "Path to the PID file")
	flag.StringVar(&loggingJSONCfg, "logging", defaultLoggingCfg, "Logging configuration")
	flag.Parse()
	serviceCfg.BrokerAddrs = strings.Split(*b, ",")
}

func main() {
	if err := initLogging(); err != nil {
		fmt.Printf("Failed to initialize logger, cause=(%v)\n", err)
		os.Exit(1)
	}

	if err := writePID(pidFile); err != nil {
		log.Errorf("Failed to write PID file, cause=(%v)", err)
		os.Exit(1)
	}

	// Clean up the unix domain socket file in case we failed to clean up on
	// shutdown the last time. Otherwise the service won't be able to listen
	// on this address and the service will terminated immediately.
	if err := os.Remove(serviceCfg.UnixAddr); err != nil && err.(*os.PathError).Err != os.ErrNotExist {
		log.Errorf("Cannot remove %s", serviceCfg.UnixAddr)
	}

	log.Infof("Starting with config: %+v", serviceCfg)
	svc, err := pixy.SpawnService(&serviceCfg)
	if err != nil {
		log.Errorf("Failed to start service, cause=(%v)", err)
		os.Exit(1)
	}

	// Spawn OS signal listener to ensure graceful stop.
	osSigCh := make(chan os.Signal)
	signal.Notify(osSigCh, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM)
	go func() {
		<-osSigCh
		svc.Stop()
	}()

	svc.Wait4Stop()
}

func initLogging() error {
	var loggingCfg []log.Config
	if err := json.Unmarshal([]byte(loggingJSONCfg), &loggingCfg); err != nil {
		return fmt.Errorf("failed to parse logger config, cause=(%v)", err)
	}
	if err := log.InitWithConfig(loggingCfg...); err != nil {
		return err
	}
	return nil
}

func writePID(path string) error {
	pid := os.Getpid()
	return ioutil.WriteFile(path, []byte(fmt.Sprint(pid)), 0644)
}
<|endoftext|>


<commit_before>package main

import "github.com/akwick/sqinco/sqlInjections"

func main() {
	// 10er examples
	sqlInjections.UnprepStmtArg()
	sqlInjections.UnprepStmtConst()
	sqlInjections.UnprepStmtConstDecl()
	sqlInjections.UnprepStmtConstDeclUntypedString()
	sqlInjections.UnprepStmtConstDeclTypedString()
	sqlInjections.UnprepStmtConstIsConstRetVal()
	sqlInjections.UnprepStmtConstIsConstRetValFlowSens()
	sqlInjections.UnprepStmtConstDeclChangeAfterQueryFunCall()

	// 20er examples
	sqlInjections.UnprepStmtConstDeclUntypedStringQueryRow()
	sqlInjections.UnprepStmtDeclStringQueryRow()
	sqlInjections.UnprepStmtFormatedString()
	sqlInjections.UnprepStmtConstDeclSmallKeywords()
	sqlInjections.WrappedFunc()
	sqlInjections.NotQuerry()
}
<commit_msg>Update called functions<commit_after>package main

import "github.com/akwick/sqinco/sqlInjections"

func main() {
	// 10er examples
	sqlInjections.UnprepStmtConst()
	sqlInjections.UnprepStmtConstDecl()
	sqlInjections.UnprepStmtConstDeclUntypedString()
	sqlInjections.UnprepStmtConstDeclTypedString()
	sqlInjections.UnprepStmtConstIsConstRetVal()
	sqlInjections.UnprepStmtConstIsConstRetValFlowSens()
	sqlInjections.UnprepStmtConstDeclChangeAfterQueryFunCall()

	// 20er examples
	sqlInjections.UnprepStmtConstDeclUntypedStringQueryRow()
	sqlInjections.UnprepStmtDeclStringQueryRow()
	sqlInjections.UnprepStmtFormatedString()
	sqlInjections.UnprepStmtConstDeclSmallKeywords()
	sqlInjections.WrappedFunc()
	sqlInjections.NotQuerry()
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
)

func writeStringToFile(filePath, content string) error {
	if filePath == "" {
		return errors.New("No path provided!")
	}

	file, err := os.Create(filePath)
	if err != nil {
		return err
	}
	defer file.Close()

	_, err = file.Write([]byte(content))
	if err != nil {
		return err
	}

	return nil
}

func transformIfSpecialEnv(envKeyValuePair EnvKeyValuePair) (EnvKeyValuePair, error) {
	if envKeyValuePair.Key == "__INPUT_FILE__" {
		log.Println(" (i) Special key: __INPUT_FILE__")
		usr, err := user.Current()
		if err != nil {
			return EnvKeyValuePair{}, err
		}
		tmpFolderPath := filepath.Join(usr.HomeDir, "bitrise/tmp")
		if err := os.MkdirAll(tmpFolderPath, 0777); err != nil {
			return EnvKeyValuePair{}, err
		}
		stepInputStoreFilePath := filepath.Join(tmpFolderPath, "step_input_store")
		if err := writeStringToFile(stepInputStoreFilePath, envKeyValuePair.Value); err != nil {
			return EnvKeyValuePair{}, err
		}
		envKeyValuePair.Value = stepInputStoreFilePath
	}
	return envKeyValuePair, nil
}

func filterEnvironmentKeyValuePairs(envKeyValuePair []EnvKeyValuePair) []EnvKeyValuePair {
	filteredPairs := []EnvKeyValuePair{}

	for _, aPair := range envKeyValuePair {
		if aPair.Key == "" {
			log.Println("[i] Key is missing - won't add it to the environment. Value: ", aPair.Value)
			continue
		}
		if aPair.Value == "" {
			log.Printf("[i] Value is missing - won't add it to the environment (default value will be used by the Step) (Key: %s)\n", aPair.Key)
			continue
		}

		aPair, err := transformIfSpecialEnv(aPair)
		if err != nil {
			log.Printf("[i] Failed to convert special Env - ignored (Key: %s | Value: %s)\n", aPair.Key, aPair.Value)
			continue
		}
		filteredPairs = append(filteredPairs, aPair)
	}

	return filteredPairs
}

func runStepWithAdditionalEnvironment(commandPath string, envsToAdd []EnvKeyValuePair) error {
	commandDir := filepath.Dir(commandPath)
	commandName := filepath.Base(commandPath)
	c := exec.Command("bash", commandName)

	envLength := len(envsToAdd)
	if envLength > 0 {
		envStringPairs := make([]string, len(envsToAdd), len(envsToAdd))
		for idx, aEnvPair := range envsToAdd {
			envStringPairs[idx] = aEnvPair.ToStringWithExpand()
		}
		c.Env = append(os.Environ(), envStringPairs...)
	}
	c.Dir = commandDir
	c.Stdout = os.Stdout
	c.Stderr = os.Stderr
	if err := c.Run(); err != nil {
		return err
	}
	return nil
}

func runCommandWithArgs(command string, cmdArgs ...string) error {
	c := exec.Command(command, cmdArgs...)
	c.Stdout = os.Stdout
	c.Stderr = os.Stderr
	if err := c.Run(); err != nil {
		return err
	}
	return nil
}

func perform(encodedStepPath, encodedCombinedStepEnvs string) error {
	if encodedStepPath == "" {
		return errors.New("No Step Path provided")
	}

	decodedStepCommand, err := decodeSingleValue(encodedStepPath)
	if err != nil {
		return err
	}
	decodedStepCommand = ExpandPath(decodedStepCommand)
	decodedStepEnvPairs, err := decodeCombinedEnvs(encodedCombinedStepEnvs)
	if err != nil {
		return err
	}

	filteredEnvPairs := filterEnvironmentKeyValuePairs(decodedStepEnvPairs)

	fmt.Println("Perform: ", decodedStepCommand, filteredEnvPairs)
	return runStepWithAdditionalEnvironment(decodedStepCommand, filteredEnvPairs)
}

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s [FLAGS]\n", os.Args[0])
	flag.PrintDefaults()
}

func main() {
	var (
		flagEncodedStepPath         = flag.String("steppath", "", "[REQUIRED] step's path (base64 encoded)")
		flagEncodedCombinedStepEnvs = flag.String("stepenvs", "", "[REQUIRED] step's encoded-combined environment key-value pairs")
	)

	flag.Usage = usage
	flag.Parse()

	if *flagEncodedStepPath == "" {
		flag.Usage()
		os.Exit(1)
	}

	if err := perform(*flagEncodedStepPath, *flagEncodedCombinedStepEnvs); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>expand the environment with the processed environments, so subsequent envs can use previous ones, even for the same step<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
)

func writeStringToFile(filePath, content string) error {
	if filePath == "" {
		return errors.New("No path provided!")
	}

	file, err := os.Create(filePath)
	if err != nil {
		return err
	}
	defer file.Close()

	_, err = file.Write([]byte(content))
	if err != nil {
		return err
	}

	return nil
}

func transformIfSpecialEnv(envKeyValuePair EnvKeyValuePair) (EnvKeyValuePair, error) {
	if envKeyValuePair.Key == "__INPUT_FILE__" {
		log.Println(" (i) Special key: __INPUT_FILE__")
		usr, err := user.Current()
		if err != nil {
			return EnvKeyValuePair{}, err
		}
		tmpFolderPath := filepath.Join(usr.HomeDir, "bitrise/tmp")
		if err := os.MkdirAll(tmpFolderPath, 0777); err != nil {
			return EnvKeyValuePair{}, err
		}
		stepInputStoreFilePath := filepath.Join(tmpFolderPath, "step_input_store")
		if err := writeStringToFile(stepInputStoreFilePath, envKeyValuePair.Value); err != nil {
			return EnvKeyValuePair{}, err
		}
		envKeyValuePair.Value = stepInputStoreFilePath
	}
	return envKeyValuePair, nil
}

func filterEnvironmentKeyValuePairs(envKeyValuePair []EnvKeyValuePair) []EnvKeyValuePair {
	filteredPairs := []EnvKeyValuePair{}

	for _, aPair := range envKeyValuePair {
		if aPair.Key == "" {
			log.Println("[i] Key is missing - won't add it to the environment. Value: ", aPair.Value)
			continue
		}
		if aPair.Value == "" {
			log.Printf("[i] Value is missing - won't add it to the environment (default value will be used by the Step) (Key: %s)\n", aPair.Key)
			continue
		}

		aPair, err := transformIfSpecialEnv(aPair)
		if err != nil {
			log.Printf("[i] Failed to convert special Env - ignored (Key: %s | Value: %s)\n", aPair.Key, aPair.Value)
			continue
		}
		filteredPairs = append(filteredPairs, aPair)
	}

	return filteredPairs
}

func runStepWithAdditionalEnvironment(commandPath string, envsToAdd []EnvKeyValuePair) error {
	commandDir := filepath.Dir(commandPath)
	commandName := filepath.Base(commandPath)
	c := exec.Command("bash", commandName)

	envLength := len(envsToAdd)
	if envLength > 0 {
		envStringPairs := make([]string, len(envsToAdd), len(envsToAdd))
		for idx, aEnvPair := range envsToAdd {
			envStringPairs[idx] = aEnvPair.ToStringWithExpand()
			// set as env, so subsequent expansions can use it
			if err := os.Setenv(aEnvPair.Key, os.ExpandEnv(aEnvPair.Value)); err != nil {
				fmt.Println(" [!] Failed to set Env: ", aEnvPair)
			}
		}
		c.Env = append(os.Environ(), envStringPairs...)
	}
	c.Dir = commandDir
	c.Stdout = os.Stdout
	c.Stderr = os.Stderr
	if err := c.Run(); err != nil {
		return err
	}
	return nil
}

func runCommandWithArgs(command string, cmdArgs ...string) error {
	c := exec.Command(command, cmdArgs...)
	c.Stdout = os.Stdout
	c.Stderr = os.Stderr
	if err := c.Run(); err != nil {
		return err
	}
	return nil
}

func perform(encodedStepPath, encodedCombinedStepEnvs string) error {
	if encodedStepPath == "" {
		return errors.New("No Step Path provided")
	}

	decodedStepCommand, err := decodeSingleValue(encodedStepPath)
	if err != nil {
		return err
	}
	decodedStepCommand = ExpandPath(decodedStepCommand)
	decodedStepEnvPairs, err := decodeCombinedEnvs(encodedCombinedStepEnvs)
	if err != nil {
		return err
	}

	filteredEnvPairs := filterEnvironmentKeyValuePairs(decodedStepEnvPairs)

	fmt.Println("Perform: ", decodedStepCommand, filteredEnvPairs)
	return runStepWithAdditionalEnvironment(decodedStepCommand, filteredEnvPairs)
}

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s [FLAGS]\n", os.Args[0])
	flag.PrintDefaults()
}

func main() {
	var (
		flagEncodedStepPath         = flag.String("steppath", "", "[REQUIRED] step's path (base64 encoded)")
		flagEncodedCombinedStepEnvs = flag.String("stepenvs", "", "[REQUIRED] step's encoded-combined environment key-value pairs")
	)

	flag.Usage = usage
	flag.Parse()

	if *flagEncodedStepPath == "" {
		flag.Usage()
		os.Exit(1)
	}

	if err := perform(*flagEncodedStepPath, *flagEncodedCombinedStepEnvs); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"net/http"
	"strings"

	"github.com/gogap/errors"
	"github.com/gogap/logs"
	"github.com/gogap/spirit"
	"github.com/spirit-contrib/inlet_http"
)

const (
	SPIRIT_NAME    = "inlet_http_api"
	METHOD_OPTIONS = "OPTIONS"
)

var (
	conf InletHTTPAPIConfig

	proxyAPI = make(map[string]bool)
)

func main() {
	conf = LoadConfig("./conf/inlet_http_api.conf")

	graphProvider := NewAPIGraphProvider(API_HEADER, conf.Address, conf.Graphs)

	httpConf := inlet_http.Config{Address: conf.HTTP.Address, Domain: conf.HTTP.CookiesDomain}

	inletHTTP := inlet_http.NewInletHTTP(
		inlet_http.SetHTTPConfig(httpConf),
		inlet_http.SetGraphProvider(graphProvider),
		inlet_http.SetResponseHandler(responseHandle),
		inlet_http.SetErrorResponseHandler(errorResponseHandler),
		inlet_http.SetRequestDecoder(requestDecoder),
		inlet_http.SetRequestPayloadHook(requestPayloadHook))

	httpAPISpirit := spirit.NewClassicSpirit(SPIRIT_NAME, "an http inlet with POST request", "1.0.0")
	httpAPIComponent := spirit.NewBaseComponent(SPIRIT_NAME)

	httpAPIComponent.RegisterHandler("callback", inletHTTP.CallBack)
	httpAPIComponent.RegisterHandler("error", inletHTTP.Error)

	httpAPISpirit.Hosting(httpAPIComponent).Build()

	inletHTTP.Requester().SetMessageSenderFactory(httpAPISpirit.GetMessageSenderFactory())

	go inletHTTP.Run(optionHandle)
	httpAPISpirit.Run()
}

type APIResponse struct {
	Code           uint64      `json:"code"`
	ErrorId        string      `json:"error_id,omitempty"`
	ErrorNamespace string      `json:"error_namespace,omitempty"`
	Message        string      `json:"message"`
	Result         interface{} `json:"result"`
}

func requestDecoder(data []byte) (ret map[string]interface{}, err error) {
	str := strings.TrimSpace(string(data))
	if str != "" {
		ret = make(map[string]interface{})
		err = json.Unmarshal(data, &ret)
	}
	return
}

func requestPayloadHook(r *http.Request, body []byte, payload *spirit.Payload) {
	apiName := r.Header.Get(conf.HTTP.APIHeader)

	if apiName == "" {
		return
	}

	if proxyAPI != nil {
		if isProxy, _ := proxyAPI[apiName]; isProxy {
			newPayload := spirit.Payload{}

			if e := newPayload.UnSerialize(body); e != nil {
				logs.Error(e)
			} else {
				payload.CopyFrom(&newPayload)
			}
		}
	}

	payload.SetContext(conf.HTTP.APIHeader, apiName)
}

func optionHandle(w http.ResponseWriter, r *http.Request) {
	if r.Method == METHOD_OPTIONS {
		writeAccessHeaders(w, r)
		writeBasicHeaders(w, r)
		w.Write([]byte(""))
	}
}

func errorResponseHandler(err error, w http.ResponseWriter, r *http.Request) {
	var resp APIResponse
	if errCode, ok := err.(errors.ErrCode); ok {
		resp = APIResponse{
			Code:           errCode.Code(),
			ErrorId:        errCode.Id(),
			ErrorNamespace: errCode.Namespace(),
			Message:        errCode.Error(),
			Result:         nil,
		}
	} else {
		resp = APIResponse{
			Code:           500,
			ErrorId:        "",
			ErrorNamespace: INLET_HTTP_API_ERR_NS,
			Message:        err.Error(),
			Result:         nil,
		}
	}

	statusCode := http.StatusInternalServerError

	if ERR_API_GRAPH_IS_NOT_EXIST.IsEqual(err) {
		statusCode = http.StatusNotFound
	} else if inlet_http.ERR_REQUEST_TIMEOUT.IsEqual(err) {
		statusCode = http.StatusRequestTimeout
	}

	writeErrorResponse(&resp, w, r, statusCode)
}

func responseHandle(payload spirit.Payload, w http.ResponseWriter, r *http.Request) {
	if payload.IsCorrect() {
		correctHandle(payload, w, r)
	} else {
		errorHandle(payload, w, r)
	}
}

func correctHandle(payload spirit.Payload, w http.ResponseWriter, r *http.Request) {
	resp := APIResponse{
		Code:   payload.Error().Code,
		Result: payload.GetContent(),
	}
	writeResponse(&resp, w, r)
}

func errorHandle(payload spirit.Payload, w http.ResponseWriter, r *http.Request) {
	resp := APIResponse{
		Code:           payload.Error().Code,
		ErrorId:        payload.Error().Id,
		ErrorNamespace: payload.Error().Namespace,
		Message:        payload.Error().Message,
		Result:         nil,
	}

	writeErrorResponse(&resp, w, r, http.StatusInternalServerError)
}

func writeResponse(v interface{}, w http.ResponseWriter, r *http.Request) {
	if data, e := json.Marshal(v); e != nil {
		err := ERR_MARSHAL_STRUCT_ERROR.New(errors.Params{"err": e})
		logs.Error(err)
		if _, ok := v.(error); !ok {
			writeResponse(&err, w, r)
		}
	} else {
		writeAccessHeaders(w, r)
		writeBasicHeaders(w, r)
		w.Header().Set("Content-Type", "application/json")
		w.Write(data)
	}
}

func writeErrorResponse(v interface{}, w http.ResponseWriter, r *http.Request, code int) {
	if data, e := json.Marshal(v); e != nil {
		err := ERR_MARSHAL_STRUCT_ERROR.New(errors.Params{"err": e})
		logs.Error(err)
		if _, ok := v.(error); !ok {
			writeErrorResponse(&err, w, r, code)
		}
	} else {
		writeAccessHeaders(w, r)
		writeBasicHeaders(w, r)
		w.Header().Set("Content-Type", "application/json")
		http.Error(w, string(data), code)
	}
}

func writeAccessHeaders(w http.ResponseWriter, r *http.Request) {
	refer := r.Referer()
	if refer == "" {
		refer = r.Header.Get("Origin")
	}

	if refProtocol, refDomain, isAllowd := conf.HTTP.ParseOrigin(refer); isAllowd {
		w.Header().Set("Access-Control-Allow-Credentials", "true")
		origin := refProtocol + "://" + refDomain
		if origin == "://" ||
			refProtocol == "chrome-extension" { //issue of post man, chrome limit.
			origin = "*"
		}
		w.Header().Set("Access-Control-Allow-Origin", origin)
	}

	w.Header().Set("Access-Control-Allow-Methods", "POST")
	w.Header().Set("Access-Control-Allow-Headers", conf.HTTP.allowHeaders())
}

func writeBasicHeaders(w http.ResponseWriter, r *http.Request) {
	for key, value := range conf.HTTP.responseHeaders {
		w.Header().Set(key, value)
	}
}
<commit_msg>fix responses error to status ok<commit_after>package main

import (
	"encoding/json"
	"net/http"
	"strings"

	"github.com/gogap/errors"
	"github.com/gogap/logs"
	"github.com/gogap/spirit"
	"github.com/spirit-contrib/inlet_http"
)

const (
	SPIRIT_NAME    = "inlet_http_api"
	METHOD_OPTIONS = "OPTIONS"
)

var (
	conf InletHTTPAPIConfig

	proxyAPI = make(map[string]bool)
)

func main() {
	conf = LoadConfig("./conf/inlet_http_api.conf")

	graphProvider := NewAPIGraphProvider(API_HEADER, conf.Address, conf.Graphs)

	httpConf := inlet_http.Config{Address: conf.HTTP.Address, Domain: conf.HTTP.CookiesDomain}

	inletHTTP := inlet_http.NewInletHTTP(
		inlet_http.SetHTTPConfig(httpConf),
		inlet_http.SetGraphProvider(graphProvider),
		inlet_http.SetResponseHandler(responseHandle),
		inlet_http.SetErrorResponseHandler(errorResponseHandler),
		inlet_http.SetRequestDecoder(requestDecoder),
		inlet_http.SetRequestPayloadHook(requestPayloadHook))

	httpAPISpirit := spirit.NewClassicSpirit(SPIRIT_NAME, "an http inlet with POST request", "1.0.0")
	httpAPIComponent := spirit.NewBaseComponent(SPIRIT_NAME)

	httpAPIComponent.RegisterHandler("callback", inletHTTP.CallBack)
	httpAPIComponent.RegisterHandler("error", inletHTTP.Error)

	httpAPISpirit.Hosting(httpAPIComponent).Build()

	inletHTTP.Requester().SetMessageSenderFactory(httpAPISpirit.GetMessageSenderFactory())

	go inletHTTP.Run(optionHandle)
	httpAPISpirit.Run()
}

type APIResponse struct {
	Code           uint64      `json:"code"`
	ErrorId        string      `json:"error_id,omitempty"`
	ErrorNamespace string      `json:"error_namespace,omitempty"`
	Message        string      `json:"message"`
	Result         interface{} `json:"result"`
}

func requestDecoder(data []byte) (ret map[string]interface{}, err error) {
	str := strings.TrimSpace(string(data))
	if str != "" {
		ret = make(map[string]interface{})
		err = json.Unmarshal(data, &ret)
	}
	return
}

func requestPayloadHook(r *http.Request, body []byte, payload *spirit.Payload) {
	apiName := r.Header.Get(conf.HTTP.APIHeader)

	if apiName == "" {
		return
	}

	if proxyAPI != nil {
		if isProxy, _ := proxyAPI[apiName]; isProxy {
			newPayload := spirit.Payload{}

			if e := newPayload.UnSerialize(body); e != nil {
				logs.Error(e)
			} else {
				payload.CopyFrom(&newPayload)
			}
		}
	}

	payload.SetContext(conf.HTTP.APIHeader, apiName)
}

func optionHandle(w http.ResponseWriter, r *http.Request) {
	if r.Method == METHOD_OPTIONS {
		writeAccessHeaders(w, r)
		writeBasicHeaders(w, r)
		w.Write([]byte(""))
	}
}

func errorResponseHandler(err error, w http.ResponseWriter, r *http.Request) {
	var resp APIResponse
	if errCode, ok := err.(errors.ErrCode); ok {
		resp = APIResponse{
			Code:           errCode.Code(),
			ErrorId:        errCode.Id(),
			ErrorNamespace: errCode.Namespace(),
			Message:        errCode.Error(),
			Result:         nil,
		}
	} else {
		resp = APIResponse{
			Code:           500,
			ErrorId:        "",
			ErrorNamespace: INLET_HTTP_API_ERR_NS,
			Message:        err.Error(),
			Result:         nil,
		}
	}

	statusCode := http.StatusInternalServerError

	if ERR_API_GRAPH_IS_NOT_EXIST.IsEqual(err) {
		statusCode = http.StatusNotFound
	} else if inlet_http.ERR_REQUEST_TIMEOUT.IsEqual(err) {
		statusCode = http.StatusRequestTimeout
	}

	writeErrorResponse(&resp, w, r, statusCode)
}

func responseHandle(payload spirit.Payload, w http.ResponseWriter, r *http.Request) {
	if payload.IsCorrect() {
		correctHandle(payload, w, r)
	} else {
		errorHandle(payload, w, r)
	}
}

func correctHandle(payload spirit.Payload, w http.ResponseWriter, r *http.Request) {
	resp := APIResponse{
		Code:   payload.Error().Code,
		Result: payload.GetContent(),
	}
	writeResponse(&resp, w, r)
}

func errorHandle(payload spirit.Payload, w http.ResponseWriter, r *http.Request) {
	resp := APIResponse{
		Code:           payload.Error().Code,
		ErrorId:        payload.Error().Id,
		ErrorNamespace: payload.Error().Namespace,
		Message:        payload.Error().Message,
		Result:         nil,
	}

	writeErrorResponse(&resp, w, r, http.StatusOK)
}

func writeResponse(v interface{}, w http.ResponseWriter, r *http.Request) {
	if data, e := json.Marshal(v); e != nil {
		err := ERR_MARSHAL_STRUCT_ERROR.New(errors.Params{"err": e})
		logs.Error(err)
		if _, ok := v.(error); !ok {
			writeResponse(&err, w, r)
		}
	} else {
		writeAccessHeaders(w, r)
		writeBasicHeaders(w, r)
		w.Header().Set("Content-Type", "application/json")
		w.Write(data)
	}
}

func writeErrorResponse(v interface{}, w http.ResponseWriter, r *http.Request, code int) {
	if data, e := json.Marshal(v); e != nil {
		err := ERR_MARSHAL_STRUCT_ERROR.New(errors.Params{"err": e})
		logs.Error(err)
		if _, ok := v.(error); !ok {
			writeErrorResponse(&err, w, r, code)
		}
	} else {
		writeAccessHeaders(w, r)
		writeBasicHeaders(w, r)
		w.Header().Set("Content-Type", "application/json")
		http.Error(w, string(data), code)
	}
}

func writeAccessHeaders(w http.ResponseWriter, r *http.Request) {
	refer := r.Referer()
	if refer == "" {
		refer = r.Header.Get("Origin")
	}

	if refProtocol, refDomain, isAllowd := conf.HTTP.ParseOrigin(refer); isAllowd {
		w.Header().Set("Access-Control-Allow-Credentials", "true")
		origin := refProtocol + "://" + refDomain
		if origin == "://" ||
			refProtocol == "chrome-extension" { //issue of post man, chrome limit.
			origin = "*"
		}
		w.Header().Set("Access-Control-Allow-Origin", origin)
	}

	w.Header().Set("Access-Control-Allow-Methods", "POST")
	w.Header().Set("Access-Control-Allow-Headers", conf.HTTP.allowHeaders())
}

func writeBasicHeaders(w http.ResponseWriter, r *http.Request) {
	for key, value := range conf.HTTP.responseHeaders {
		w.Header().Set(key, value)
	}
}
<|endoftext|>