<commit_before>package main

import (
	"database/sql"
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"path/filepath"

	"github.com/asaskevich/govalidator"
	"github.com/julienschmidt/httprouter"
)

// global variables
var conn *sql.DB
var handlerMap = make(map[string]interface{})
var SomaCfg SomaConfig

func main() {
	var (
		configFlag, configFile string
		err                    error
	)
	flag.StringVar(&configFlag, "config", "/srv/soma/conf/soma.conf", "Configuration file location")
	flag.Parse()

	version := "0.4.5"
	log.Printf("Starting runtime config initialization, SOMA v%s", version)
	/*
	 * Read configuration file
	 */
	if configFile, err = filepath.Abs(configFlag); err != nil {
		log.Fatal(err)
	}
	if configFile, err = filepath.EvalSymlinks(configFile); err != nil {
		log.Fatal(err)
	}
	err = SomaCfg.readConfigFile(configFile)
	if err != nil {
		log.Fatal(err)
	}

	/*
	 * Construct listen address
	 */
	SomaCfg.Daemon.url = &url.URL{}
	SomaCfg.Daemon.url.Host = fmt.Sprintf("%s:%s", SomaCfg.Daemon.Listen, SomaCfg.Daemon.Port)
	if SomaCfg.Daemon.Tls {
		SomaCfg.Daemon.url.Scheme = "https"
		if ok, pt := govalidator.IsFilePath(SomaCfg.Daemon.Cert); !ok {
			log.Fatal("Missing required certificate configuration config/daemon/cert-file")
		} else {
			if pt != govalidator.Unix {
				log.Fatal("config/daemon/cert-File: valid Windows paths are not helpful")
			}
		}
		if ok, pt := govalidator.IsFilePath(SomaCfg.Daemon.Key); !ok {
			log.Fatal("Missing required key configuration config/daemon/key-file")
		} else {
			if pt != govalidator.Unix {
				log.Fatal("config/daemon/key-file: valid Windows paths are not helpful")
			}
		}
	} else {
		SomaCfg.Daemon.url.Scheme = "http"
	}

	connectToDatabase()
	go pingDatabase()

	startHandlers()

	router := httprouter.New()

	router.GET("/views", ListView)
	router.GET("/views/:view", ShowView)

	router.GET("/environments", ListEnvironments)
	router.GET("/environments/:environment", ShowEnvironment)

	router.GET("/objstates", ListObjectStates)
	router.GET("/objstates/:state", ShowObjectState)

	router.GET("/objtypes", ListObjectTypes)
	router.GET("/objtypes/:type", ShowObjectType)

	router.GET("/datacenters", ListDatacenters)
	router.GET("/datacenters/:datacenter", ShowDatacenter)

	router.GET("/datacentergroups", ListDatacenterGroups)
	router.GET("/datacentergroups/:datacentergroup", ShowDatacenterGroup)

	router.GET("/levels/", ListLevel)
	router.GET("/levels/:level", ShowLevel)
	router.POST("/filter/levels/", ListLevel)

	router.GET("/predicates/", ListPredicate)
	router.GET("/predicates/:predicate", ShowPredicate)

	router.GET("/status/", ListStatus)
	router.GET("/status/:status", ShowStatus)

	router.GET("/oncall/", ListOncall)
	router.GET("/oncall/:oncall", ShowOncall)
	router.POST("/filter/oncall/", ListOncall)

	router.GET("/teams/", ListTeam)
	router.GET("/teams/:team", ShowTeam)
	router.POST("/filter/teams/", ListTeam)

	router.GET("/nodes/", ListNode)
	router.GET("/nodes/:node", ShowNode)
	router.GET("/nodes/:node/config", ShowNodeConfig)
	router.POST("/filter/nodes/", ListNode)

	router.GET("/servers/", ListServer)
	router.GET("/servers/:server", ShowServer)
	router.POST("/filter/servers/", ListServer)

	router.GET("/units/", ListUnit)
	router.GET("/units/:unit", ShowUnit)

	router.GET("/providers/", ListProvider)
	router.GET("/providers/:provider", ShowProvider)

	router.GET("/metrics/", ListMetric)
	router.GET("/metrics/:metric", ShowMetric)

	router.GET("/modes/", ListMode)
	router.GET("/modes/:mode", ShowMode)

	router.GET("/users/", ListUser)
	router.GET("/users/:user", ShowUser)
	router.POST("/filter/users/", ListUser)

	router.GET("/monitoring/", ListMonitoring)
	router.GET("/monitoring/:monitoring", ShowMonitoring)
	router.POST("/filter/monitoring/", ListMonitoring)

	router.GET("/capability/", ListCapability)
	router.GET("/capability/:capability", ShowCapability)
	router.POST("/filter/capability/", ListCapability)

	router.GET("/property/native/", ListProperty)
	router.GET("/property/native/:native", ShowProperty)

	router.GET("/property/system/", ListProperty)
	router.GET("/property/system/:system", ShowProperty)

	router.GET("/property/custom/:repository/", ListProperty)
	router.GET("/property/custom/:repository/:custom", ShowProperty)
	router.POST("/filter/property/custom/", ListProperty)

	router.GET("/property/service/global/", ListProperty)
	router.GET("/property/service/global/:service", ShowProperty)

	router.GET("/property/service/team/:team/", ListProperty)
	router.GET("/property/service/team/:team/:service", ShowProperty)

	router.GET("/validity/", ListValidity)
	router.GET("/validity/:property", ShowValidity)

	router.GET("/attributes/", ListAttribute)
	router.GET("/attributes/:attribute", ShowAttribute)

	router.GET("/repository/", ListRepository)
	router.GET("/repository/:repository", ShowRepository)
	router.GET("/filter/repository/", ListRepository)

	router.GET("/buckets/", ListBucket)
	router.GET("/buckets/:bucket", ShowBucket)
	router.POST("/filter/buckets/", ListBucket)

	router.GET("/groups/", ListGroup)
	router.GET("/groups/:group", ShowGroup)
	router.GET("/groups/:group/members/", ListGroupMembers)
	router.POST("/filter/groups/", ListGroup)

	router.GET("/clusters/", ListCluster)
	router.GET("/clusters/:cluster", ShowCluster)
	router.GET("/clusters/:cluster/members/", ListClusterMembers)
	router.POST("/filter/clusters/", ListCluster)

	router.GET("/checks/:repository/", ListCheckConfiguration)
	router.GET("/checks/:repository/:check", ShowCheckConfiguration)
	router.POST("/filter/checks/:repository/", ListCheckConfiguration)

	router.GET("/hostdeployment/:system/:assetid", GetHostDeployment)
	router.POST("/hostdeployment/:system/:assetid", AssembleHostUpdate)

	if !SomaCfg.ReadOnly {
		router.POST("/views/", AddView)
		router.DELETE("/views/:view", DeleteView)
		router.PATCH("/views/:view", RenameView)

		router.POST("/environments", AddEnvironment)
		router.DELETE("/environments/:environment", DeleteEnvironment)
		router.PUT("/environments/:environment", RenameEnvironment)

		router.POST("/objstates", AddObjectState)
		router.DELETE("/objstates/:state", DeleteObjectState)
		router.PUT("/objstates/:state", RenameObjectState)

		router.POST("/objtypes", AddObjectType)
		router.DELETE("/objtypes/:type", DeleteObjectType)
		router.PUT("/objtypes/:type", RenameObjectType)

		router.POST("/datacenters", AddDatacenter)
		router.DELETE("/datacenters/:datacenter", DeleteDatacenter)
		router.PUT("/datacenters/:datacenter", RenameDatacenter)

		router.PATCH("/datacentergroups/:datacentergroup", AddDatacenterToGroup)
		router.DELETE("/datacentergroups/:datacentergroup", DeleteDatacenterFromGroup)

		router.POST("/levels/", AddLevel)
		router.DELETE("/levels/:level", DeleteLevel)

		router.POST("/predicates/", AddPredicate)
		router.DELETE("/predicates/:predicate", DeletePredicate)

		router.POST("/status/", AddStatus)
		router.DELETE("/status/:status", DeleteStatus)

		router.POST("/oncall/", AddOncall)
		router.PATCH("/oncall/:oncall", UpdateOncall)
		router.DELETE("/oncall/:oncall", DeleteOncall)

		router.POST("/teams/", AddTeam)
		router.DELETE("/teams/:team", DeleteTeam)

		router.POST("/servers/", AddServer)
		router.DELETE("/servers/:server", DeleteServer)
		router.PUT("/servers/:server", InsertNullServer)

		router.POST("/units/", AddUnit)
		router.DELETE("/units/:unit", DeleteUnit)

		router.POST("/providers/", AddProvider)
		router.DELETE("/providers/:provider", DeleteProvider)

		router.POST("/metrics/", AddMetric)
		router.DELETE("/metrics/:metric", DeleteMetric)

		router.POST("/modes/", AddMode)
		router.DELETE("/modes/:mode", DeleteMode)

		router.POST("/users/", AddUser)
		router.DELETE("/users/:user", DeleteUser)

		router.POST("/monitoring/", AddMonitoring)
		router.DELETE("/monitoring/:monitoring", DeleteMonitoring)

		router.POST("/capability/", AddCapability)
		router.DELETE("/capability/:capability", DeleteCapability)

		router.POST("/property/native/", AddProperty)
		router.DELETE("/property/native/:native", DeleteProperty)

		router.POST("/property/system/", AddProperty)
		router.DELETE("/property/system/:system", DeleteProperty)

		router.POST("/property/custom/:repository/", AddProperty)
		router.DELETE("/property/custom/:repository/:custom", DeleteProperty)

		router.POST("/property/service/global/", AddProperty)
		router.DELETE("/property/service/global/:service", DeleteProperty)

		router.POST("/property/service/team/:team/", AddProperty)
		router.DELETE("/property/service/team/:team/:service", DeleteProperty)

		router.POST("/validity/", AddValidity)
		router.DELETE("/validity/:property", DeleteValidity)

		router.POST("/attributes/", AddAttribute)
		router.DELETE("/attributes/:attribute", DeleteAttribute)

		router.POST("/repository/", AddRepository)
		router.POST("/repository/:repository/property/:type/", AddPropertyToRepository)

		router.POST("/buckets/", AddBucket)
		router.POST("/buckets/:bucket/property/:type/", AddPropertyToBucket)

		router.POST("/groups/", AddGroup)
		router.POST("/groups/:group/members/", AddMemberToGroup)
		router.POST("/groups/:group/property/:type/", AddPropertyToGroup)

		router.POST("/clusters/", AddCluster)
		router.POST("/clusters/:cluster/members/", AddMemberToCluster)
		router.POST("/clusters/:cluster/property/:type/", AddPropertyToCluster)

		router.POST("/nodes/", AddNode)
		router.DELETE("/nodes/:node", DeleteNode)
		router.PUT("/nodes/:node/config", AssignNode)
		router.POST("/nodes/:node/property/:type/", AddPropertyToNode)

		router.POST("/checks/:repository/", AddCheckConfiguration)

		router.GET("/deployments/id/:uuid", DeliverDeploymentDetails)
		router.GET("/deployments/monitoring/:uuid", DeliverMonitoringDeployments)
		router.GET("/deployments/monitoring/:uuid/:all", DeliverMonitoringDeployments)
		router.PATCH("/deployments/id/:uuid/:result", UpdateDeploymentDetails)
	}

	if SomaCfg.Daemon.Tls {
		log.Fatal(http.ListenAndServeTLS(
			SomaCfg.Daemon.url.Host,
			SomaCfg.Daemon.Cert,
			SomaCfg.Daemon.Key,
			router))
	} else {
		log.Fatal(http.ListenAndServe(SomaCfg.Daemon.url.Host, router))
	}
}

// vim: ts=4 sw=4 sts=4 noet fenc=utf-8 ffs=unix
<commit_msg>0.0.5: merge property-redux branch<commit_after>package main

import (
	"database/sql"
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"path/filepath"

	"github.com/asaskevich/govalidator"
	"github.com/julienschmidt/httprouter"
)

// global variables
var conn *sql.DB
var handlerMap = make(map[string]interface{})
var SomaCfg SomaConfig

func main() {
	var (
		configFlag, configFile string
		err                    error
	)
	flag.StringVar(&configFlag, "config", "/srv/soma/conf/soma.conf", "Configuration file location")
	flag.Parse()

	version := "0.5.0"
	log.Printf("Starting runtime config initialization, SOMA v%s", version)
	/*
	 * Read configuration file
	 */
	if configFile, err = filepath.Abs(configFlag); err != nil {
		log.Fatal(err)
	}
	if configFile, err = filepath.EvalSymlinks(configFile); err != nil {
		log.Fatal(err)
	}
	err = SomaCfg.readConfigFile(configFile)
	if err != nil {
		log.Fatal(err)
	}

	/*
	 * Construct listen address
	 */
	SomaCfg.Daemon.url = &url.URL{}
	SomaCfg.Daemon.url.Host = fmt.Sprintf("%s:%s", SomaCfg.Daemon.Listen, SomaCfg.Daemon.Port)
	if SomaCfg.Daemon.Tls {
		SomaCfg.Daemon.url.Scheme = "https"
		if ok, pt := govalidator.IsFilePath(SomaCfg.Daemon.Cert); !ok {
			log.Fatal("Missing required certificate configuration config/daemon/cert-file")
		} else {
			if pt != govalidator.Unix {
				log.Fatal("config/daemon/cert-File: valid Windows paths are not helpful")
			}
		}
		if ok, pt := govalidator.IsFilePath(SomaCfg.Daemon.Key); !ok {
			log.Fatal("Missing required key configuration config/daemon/key-file")
		} else {
			if pt != govalidator.Unix {
				log.Fatal("config/daemon/key-file: valid Windows paths are not helpful")
			}
		}
	} else {
		SomaCfg.Daemon.url.Scheme = "http"
	}

	connectToDatabase()
	go pingDatabase()

	startHandlers()

	router := httprouter.New()

	router.GET("/views", ListView)
	router.GET("/views/:view", ShowView)

	router.GET("/environments", ListEnvironments)
	router.GET("/environments/:environment", ShowEnvironment)

	router.GET("/objstates", ListObjectStates)
	router.GET("/objstates/:state", ShowObjectState)

	router.GET("/objtypes", ListObjectTypes)
	router.GET("/objtypes/:type", ShowObjectType)

	router.GET("/datacenters", ListDatacenters)
	router.GET("/datacenters/:datacenter", ShowDatacenter)

	router.GET("/datacentergroups", ListDatacenterGroups)
	router.GET("/datacentergroups/:datacentergroup", ShowDatacenterGroup)

	router.GET("/levels/", ListLevel)
	router.GET("/levels/:level", ShowLevel)
	router.POST("/filter/levels/", ListLevel)

	router.GET("/predicates/", ListPredicate)
	router.GET("/predicates/:predicate", ShowPredicate)

	router.GET("/status/", ListStatus)
	router.GET("/status/:status", ShowStatus)

	router.GET("/oncall/", ListOncall)
	router.GET("/oncall/:oncall", ShowOncall)
	router.POST("/filter/oncall/", ListOncall)

	router.GET("/teams/", ListTeam)
	router.GET("/teams/:team", ShowTeam)
	router.POST("/filter/teams/", ListTeam)

	router.GET("/nodes/", ListNode)
	router.GET("/nodes/:node", ShowNode)
	router.GET("/nodes/:node/config", ShowNodeConfig)
	router.POST("/filter/nodes/", ListNode)

	router.GET("/servers/", ListServer)
	router.GET("/servers/:server", ShowServer)
	router.POST("/filter/servers/", ListServer)

	router.GET("/units/", ListUnit)
	router.GET("/units/:unit", ShowUnit)

	router.GET("/providers/", ListProvider)
	router.GET("/providers/:provider", ShowProvider)

	router.GET("/metrics/", ListMetric)
	router.GET("/metrics/:metric", ShowMetric)

	router.GET("/modes/", ListMode)
	router.GET("/modes/:mode", ShowMode)

	router.GET("/users/", ListUser)
	router.GET("/users/:user", ShowUser)
	router.POST("/filter/users/", ListUser)

	router.GET("/monitoring/", ListMonitoring)
	router.GET("/monitoring/:monitoring", ShowMonitoring)
	router.POST("/filter/monitoring/", ListMonitoring)

	router.GET("/capability/", ListCapability)
	router.GET("/capability/:capability", ShowCapability)
	router.POST("/filter/capability/", ListCapability)

	router.GET("/property/native/", ListProperty)
	router.GET("/property/native/:native", ShowProperty)

	router.GET("/property/system/", ListProperty)
	router.GET("/property/system/:system", ShowProperty)

	router.GET("/property/custom/:repository/", ListProperty)
	router.GET("/property/custom/:repository/:custom", ShowProperty)
	router.POST("/filter/property/custom/", ListProperty)

	router.GET("/property/service/global/", ListProperty)
	router.GET("/property/service/global/:service", ShowProperty)

	router.GET("/property/service/team/:team/", ListProperty)
	router.GET("/property/service/team/:team/:service", ShowProperty)

	router.GET("/validity/", ListValidity)
	router.GET("/validity/:property", ShowValidity)

	router.GET("/attributes/", ListAttribute)
	router.GET("/attributes/:attribute", ShowAttribute)

	router.GET("/repository/", ListRepository)
	router.GET("/repository/:repository", ShowRepository)
	router.GET("/filter/repository/", ListRepository)

	router.GET("/buckets/", ListBucket)
	router.GET("/buckets/:bucket", ShowBucket)
	router.POST("/filter/buckets/", ListBucket)

	router.GET("/groups/", ListGroup)
	router.GET("/groups/:group", ShowGroup)
	router.GET("/groups/:group/members/", ListGroupMembers)
	router.POST("/filter/groups/", ListGroup)

	router.GET("/clusters/", ListCluster)
	router.GET("/clusters/:cluster", ShowCluster)
	router.GET("/clusters/:cluster/members/", ListClusterMembers)
	router.POST("/filter/clusters/", ListCluster)

	router.GET("/checks/:repository/", ListCheckConfiguration)
	router.GET("/checks/:repository/:check", ShowCheckConfiguration)
	router.POST("/filter/checks/:repository/", ListCheckConfiguration)

	router.GET("/hostdeployment/:system/:assetid", GetHostDeployment)
	router.POST("/hostdeployment/:system/:assetid", AssembleHostUpdate)

	if !SomaCfg.ReadOnly {
		router.POST("/views/", AddView)
		router.DELETE("/views/:view", DeleteView)
		router.PATCH("/views/:view", RenameView)

		router.POST("/environments", AddEnvironment)
		router.DELETE("/environments/:environment", DeleteEnvironment)
		router.PUT("/environments/:environment", RenameEnvironment)

		router.POST("/objstates", AddObjectState)
		router.DELETE("/objstates/:state", DeleteObjectState)
		router.PUT("/objstates/:state", RenameObjectState)

		router.POST("/objtypes", AddObjectType)
		router.DELETE("/objtypes/:type", DeleteObjectType)
		router.PUT("/objtypes/:type", RenameObjectType)

		router.POST("/datacenters", AddDatacenter)
		router.DELETE("/datacenters/:datacenter", DeleteDatacenter)
		router.PUT("/datacenters/:datacenter", RenameDatacenter)

		router.PATCH("/datacentergroups/:datacentergroup", AddDatacenterToGroup)
		router.DELETE("/datacentergroups/:datacentergroup", DeleteDatacenterFromGroup)

		router.POST("/levels/", AddLevel)
		router.DELETE("/levels/:level", DeleteLevel)

		router.POST("/predicates/", AddPredicate)
		router.DELETE("/predicates/:predicate", DeletePredicate)

		router.POST("/status/", AddStatus)
		router.DELETE("/status/:status", DeleteStatus)

		router.POST("/oncall/", AddOncall)
		router.PATCH("/oncall/:oncall", UpdateOncall)
		router.DELETE("/oncall/:oncall", DeleteOncall)

		router.POST("/teams/", AddTeam)
		router.DELETE("/teams/:team", DeleteTeam)

		router.POST("/servers/", AddServer)
		router.DELETE("/servers/:server", DeleteServer)
		router.PUT("/servers/:server", InsertNullServer)

		router.POST("/units/", AddUnit)
		router.DELETE("/units/:unit", DeleteUnit)

		router.POST("/providers/", AddProvider)
		router.DELETE("/providers/:provider", DeleteProvider)

		router.POST("/metrics/", AddMetric)
		router.DELETE("/metrics/:metric", DeleteMetric)

		router.POST("/modes/", AddMode)
		router.DELETE("/modes/:mode", DeleteMode)

		router.POST("/users/", AddUser)
		router.DELETE("/users/:user", DeleteUser)

		router.POST("/monitoring/", AddMonitoring)
		router.DELETE("/monitoring/:monitoring", DeleteMonitoring)

		router.POST("/capability/", AddCapability)
		router.DELETE("/capability/:capability", DeleteCapability)

		router.POST("/property/native/", AddProperty)
		router.DELETE("/property/native/:native", DeleteProperty)

		router.POST("/property/system/", AddProperty)
		router.DELETE("/property/system/:system", DeleteProperty)

		router.POST("/property/custom/:repository/", AddProperty)
		router.DELETE("/property/custom/:repository/:custom", DeleteProperty)

		router.POST("/property/service/global/", AddProperty)
		router.DELETE("/property/service/global/:service", DeleteProperty)

		router.POST("/property/service/team/:team/", AddProperty)
		router.DELETE("/property/service/team/:team/:service", DeleteProperty)

		router.POST("/validity/", AddValidity)
		router.DELETE("/validity/:property", DeleteValidity)

		router.POST("/attributes/", AddAttribute)
		router.DELETE("/attributes/:attribute", DeleteAttribute)

		router.POST("/repository/", AddRepository)
		router.POST("/repository/:repository/property/:type/", AddPropertyToRepository)

		router.POST("/buckets/", AddBucket)
		router.POST("/buckets/:bucket/property/:type/", AddPropertyToBucket)

		router.POST("/groups/", AddGroup)
		router.POST("/groups/:group/members/", AddMemberToGroup)
		router.POST("/groups/:group/property/:type/", AddPropertyToGroup)

		router.POST("/clusters/", AddCluster)
		router.POST("/clusters/:cluster/members/", AddMemberToCluster)
		router.POST("/clusters/:cluster/property/:type/", AddPropertyToCluster)

		router.POST("/nodes/", AddNode)
		router.DELETE("/nodes/:node", DeleteNode)
		router.PUT("/nodes/:node/config", AssignNode)
		router.POST("/nodes/:node/property/:type/", AddPropertyToNode)

		router.POST("/checks/:repository/", AddCheckConfiguration)

		router.GET("/deployments/id/:uuid", DeliverDeploymentDetails)
		router.GET("/deployments/monitoring/:uuid", DeliverMonitoringDeployments)
		router.GET("/deployments/monitoring/:uuid/:all", DeliverMonitoringDeployments)
		router.PATCH("/deployments/id/:uuid/:result", UpdateDeploymentDetails)
	}

	if SomaCfg.Daemon.Tls {
		log.Fatal(http.ListenAndServeTLS(
			SomaCfg.Daemon.url.Host,
			SomaCfg.Daemon.Cert,
			SomaCfg.Daemon.Key,
			router))
	} else {
		log.Fatal(http.ListenAndServe(SomaCfg.Daemon.url.Host, router))
	}
}

// vim: ts=4 sw=4 sts=4 noet fenc=utf-8 ffs=unix
<|endoftext|>


<commit_before>package main

// cors-proxy is a reverse proxy to allow cross origin
// requests (eg. from a javascript XHTTPRequest) to
// another service that doesn't respond to OPTIONS requests.
//
// Author: Patrice FERLET <metal3d@gmail.com>
// Licence: BSD

import (
	"crypto/tls"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"strings"
)

var (
	// host:port to connect to.
	portTo = "127.0.0.1:8000"
	// host:port to listen.
	listen = "0.0.0.0:3000"
	// verbose message.
	verbose = false
)

// handleReverseRequest writes back the server response to client.
// If an "OPTIONS" request is called, we
// only return Access-Control-Allow-* to let XHttpRequest working.
func handleReverseRequest(w http.ResponseWriter, r *http.Request) {

	// check scheme
	scheme := "http"
	if r.TLS != nil {
		scheme = "https"
	}

	// build url
	toCall := fmt.Sprintf("%s://%s%s", scheme, portTo, r.URL.String())
	debug("Create request for ", toCall)

	// always allow access origin
	w.Header().Add("Access-Control-Allow-Origin", "*")
	w.Header().Add("Access-Control-Allow-Methods", "GET, PUT, POST, HEAD, TRACE, DELETE, PATCH, COPY, HEAD, LINK, OPTIONS")

	if r.Method == "OPTIONS" {
		debug("CORS asked for ", toCall)
		for n, h := range r.Header {
			if strings.Contains(n, "Access-Control-Request") {
				for _, h := range h {
					k := strings.Replace(n, "Request", "Allow", 1)
					w.Header().Add(k, h)
				}
			}
		}
		// end
		return
	}

	// create request to server
	req, err := http.NewRequest(r.Method, toCall, r.Body)

	// add ALL header to the connection
	for n, h := range r.Header {
		for _, h := range h {
			req.Header.Add(n, h)
		}
	}

	// create a basic client to send request
	client := http.Client{}
	if r.TLS != nil {
		client.Transport = &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
	}
	resp, err := client.Do(req)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte(err.Error()))
		return
	}

	// copy the reponse from server to the connected client request
	w.WriteHeader(resp.StatusCode)
	for h, v := range resp.Header {
		for _, v := range v {
			w.Header().Add(h, v)
		}
	}

	wr, err := io.Copy(w, resp.Body)
	if err != nil {
		log.Println(wr, err)
	} else {
		debug("Writen", wr, "bytes")
	}

}

// validateFlags checks if host:port format is ok.
func validateFlags() {
	for _, f := range []string{portTo, listen} {
		if !strings.Contains(f, ":") {
			log.Fatalf("%s is not right, you must use a coma mark to separate host and port", f)
		}

	}

	parts := strings.Split(portTo, ":")
	if parts[0] == "" {
		log.Println("You didn't set host to connect, using 127.0.0.1:" + parts[1])
		portTo = "127.0.0.1:" + parts[1]
	}

}

// debug writes message when verbose flag is true.
func debug(v ...interface{}) {
	if verbose {
		log.Println(v...)
	}
}

func main() {
	flag.StringVar(&portTo, "p", portTo, "service port")
	flag.StringVar(&listen, "l", listen, "listen interface")
	flag.BoolVar(&verbose, "v", verbose, "verbose")
	flag.Parse()

	validateFlags()
	http.HandleFunc("/", handleReverseRequest)
	log.Println(listen, "-->", portTo)
	http.ListenAndServe(listen, nil)
}
<commit_msg>Call WriteHeader only after proxying through response headers<commit_after>package main

// cors-proxy is a reverse proxy to allow cross origin
// requests (eg. from a javascript XHTTPRequest) to
// another service that doesn't respond to OPTIONS requests.
//
// Author: Patrice FERLET <metal3d@gmail.com>
// Licence: BSD

import (
	"crypto/tls"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"strings"
)

var (
	// host:port to connect to.
	portTo = "127.0.0.1:8000"
	// host:port to listen.
	listen = "0.0.0.0:3000"
	// verbose message.
	verbose = false
)

// handleReverseRequest writes back the server response to client.
// If an "OPTIONS" request is called, we
// only return Access-Control-Allow-* to let XHttpRequest working.
func handleReverseRequest(w http.ResponseWriter, r *http.Request) {

	// check scheme
	scheme := "http"
	if r.TLS != nil {
		scheme = "https"
	}

	// build url
	toCall := fmt.Sprintf("%s://%s%s", scheme, portTo, r.URL.String())
	debug("Create request for ", toCall)

	// always allow access origin
	w.Header().Add("Access-Control-Allow-Origin", "*")
	w.Header().Add("Access-Control-Allow-Methods", "GET, PUT, POST, HEAD, TRACE, DELETE, PATCH, COPY, HEAD, LINK, OPTIONS")

	if r.Method == "OPTIONS" {
		debug("CORS asked for ", toCall)
		for n, h := range r.Header {
			if strings.Contains(n, "Access-Control-Request") {
				for _, h := range h {
					k := strings.Replace(n, "Request", "Allow", 1)
					w.Header().Add(k, h)
				}
			}
		}
		// end
		return
	}

	// create request to server
	req, err := http.NewRequest(r.Method, toCall, r.Body)

	// add ALL header to the connection
	for n, h := range r.Header {
		for _, h := range h {
			req.Header.Add(n, h)
		}
	}

	// create a basic client to send request
	client := http.Client{}
	if r.TLS != nil {
		client.Transport = &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
	}
	resp, err := client.Do(req)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte(err.Error()))
		return
	}

	for h, v := range resp.Header {
		for _, v := range v {
			w.Header().Add(h, v)
		}
	}
	// copy the reponse from server to the connected client request
	w.WriteHeader(resp.StatusCode)

	wr, err := io.Copy(w, resp.Body)
	if err != nil {
		log.Println(wr, err)
	} else {
		debug("Writen", wr, "bytes")
	}

}

// validateFlags checks if host:port format is ok.
func validateFlags() {
	for _, f := range []string{portTo, listen} {
		if !strings.Contains(f, ":") {
			log.Fatalf("%s is not right, you must use a coma mark to separate host and port", f)
		}

	}

	parts := strings.Split(portTo, ":")
	if parts[0] == "" {
		log.Println("You didn't set host to connect, using 127.0.0.1:" + parts[1])
		portTo = "127.0.0.1:" + parts[1]
	}

}

// debug writes message when verbose flag is true.
func debug(v ...interface{}) {
	if verbose {
		log.Println(v...)
	}
}

func main() {
	flag.StringVar(&portTo, "p", portTo, "service port")
	flag.StringVar(&listen, "l", listen, "listen interface")
	flag.BoolVar(&verbose, "v", verbose, "verbose")
	flag.Parse()

	validateFlags()
	http.HandleFunc("/", handleReverseRequest)
	log.Println(listen, "-->", portTo)
	http.ListenAndServe(listen, nil)
}
<|endoftext|>


<commit_before><commit_msg>Fixed wrong method call<commit_after><|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"github.com/jcoene/gologger"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"regexp"
	"sort"
	"strconv"
	"time"
)

type Packet struct {
	Bucket   string
	Value    string
	Modifier string
	Sampling float32
}

var log *logger.Logger

var (
	serviceAddress = flag.String("address", "0.0.0.0:8125", "UDP service address")
	libratoUser    = flag.String("user", "", "Librato Username")
	libratoToken   = flag.String("token", "", "Librato API Token")
	flushInterval  = flag.Int64("flush", 60, "Flush Interval (seconds)")
	debug          = flag.Bool("debug", false, "Enable Debugging")
)

var (
	In       = make(chan Packet, 10000)
	counters = make(map[string]int64)
	timers   = make(map[string][]float64)
	gauges   = make(map[string]float64)
)

type Measurement struct {
	Counters []Counter     `json:"counters"`
	Gauges   []interface{} `json:"gauges"`
}

func (m *Measurement) Count() int {
	return (len(m.Counters) + len(m.Gauges))
}

type Counter struct {
	Name  string `json:"name"`
	Value int64  `json:"value"`
}

type SimpleGauge struct {
	Name  string  `json:"name"`
	Value float64 `json:"value"`
}

type ComplexGauge struct {
	Name       string  `json:"name"`
	Count      int     `json:"count"`
	Sum        float64 `json:"sum"`
	Min        float64 `json:"min"`
	Max        float64 `json:"max"`
	SumSquares float64 `json:"sum_squares"`
}

func monitor() {
	t := time.NewTicker(time.Duration(*flushInterval) * time.Second)

	for {
		select {
		case <-t.C:
			if err := submit(); err != nil {
				log.Error("submit: %s", err)
			}
		case s := <-In:
			if s.Modifier == "ms" {
				_, ok := timers[s.Bucket]
				if !ok {
					var t []float64
					timers[s.Bucket] = t
				}
				floatValue, _ := strconv.ParseFloat(s.Value, 64)
				timers[s.Bucket] = append(timers[s.Bucket], floatValue)
			} else if s.Modifier == "g" {
				_, ok := gauges[s.Bucket]
				if !ok {
					gauges[s.Bucket] = float64(0)
				}
				floatValue, _ := strconv.ParseFloat(s.Value, 64)
				gauges[s.Bucket] = floatValue
			} else {
				_, ok := counters[s.Bucket]
				if !ok {
					counters[s.Bucket] = 0
				}
				floatValue, _ := strconv.ParseFloat(s.Value, 32)
				counters[s.Bucket] += int64(float32(floatValue) * (1 / s.Sampling))
			}
		}
	}
}

func submit() (err error) {
	m := new(Measurement)
	m.Counters = make([]Counter, 0)
	m.Gauges = make([]interface{}, 0)

	for k, v := range counters {
		c := new(Counter)
		c.Name = k
		c.Value = v
		m.Counters = append(m.Counters, *c)
	}

	for k, v := range gauges {
		g := new(SimpleGauge)
		g.Name = k
		g.Value = v
		m.Gauges = append(m.Gauges, *g)
	}

	for k, t := range timers {
		g := new(ComplexGauge)
		g.Name = k
		g.Count = len(t)

		if g.Count > 0 {
			sort.Float64s(t)
			g.Min = t[0]
			g.Max = t[len(t)-1]
			for _, v := range t {
				g.Sum += v
				g.SumSquares += (v * v)
			}
		}

		m.Gauges = append(m.Gauges, *g)
	}

	if m.Count() == 0 {
		log.Info("no new measurements in the last %d seconds", *flushInterval)
		return
	}

	payload, err := json.MarshalIndent(m, "", "  ")
	if err != nil {
		return
	}

	log.Debug("sending payload:\n%s", payload)

	req, err := http.NewRequest("POST", "https://metrics-api.librato.com/v1/metrics", bytes.NewBuffer(payload))
	if err != nil {
		return
	}

	req.Header.Add("Content-Type", "application/json")
	req.Header.Set("User-Agent", "statsd/1.0")
	req.SetBasicAuth(*libratoUser, *libratoToken)
	resp, err := http.DefaultClient.Do(req)
	if err == nil && resp.StatusCode != 200 {
		if err == nil {
			raw, _ := ioutil.ReadAll(resp.Body)
			err = errors.New(fmt.Sprintf("%s: %s", resp.Status, string(raw)))
		}
		log.Warn("error sending %d measurements: %s", m.Count(), err)
		return
	}

	log.Info("%d measurements sent", m.Count())

	for k, _ := range timers {
		delete(timers, k)
	}

	return
}

func handle(conn *net.UDPConn, remaddr net.Addr, buf *bytes.Buffer) {
	var packet Packet
	var value string
	var sanitizeRegexp = regexp.MustCompile("[^a-zA-Z0-9\\-_\\.:\\|@]")
	var packetRegexp = regexp.MustCompile("([a-zA-Z0-9_\\.]+):(\\-?[0-9\\.]+)\\|(c|ms|g)(\\|@([0-9\\.]+))?")
	s := sanitizeRegexp.ReplaceAllString(buf.String(), "")

	for _, item := range packetRegexp.FindAllStringSubmatch(s, -1) {
		value = item[2]
		if item[3] == "ms" {
			_, err := strconv.ParseFloat(item[2], 32)
			if err != nil {
				value = "0"
			}
		}

		sampleRate, err := strconv.ParseFloat(item[5], 32)
		if err != nil {
			sampleRate = 1
		}

		packet.Bucket = item[1]
		packet.Value = value
		packet.Modifier = item[3]
		packet.Sampling = float32(sampleRate)

		In <- packet
	}
}

func listen() {
	address, _ := net.ResolveUDPAddr("udp", *serviceAddress)
	listener, err := net.ListenUDP("udp", address)
	defer listener.Close()
	if err != nil {
		log.Fatal("unable to listen: %s", err)
		os.Exit(1)
	}

	log.Info("listening for events...")

	for {
		message := make([]byte, 512)
		n, remaddr, error := listener.ReadFrom(message)
		if error != nil {
			continue
		}
		buf := bytes.NewBuffer(message[0:n])
		go handle(listener, remaddr, buf)
	}
}

func main() {
	flag.Parse()

	if *debug {
		log = logger.NewLogger(logger.LOG_LEVEL_DEBUG, "statsd")
	} else {
		log = logger.NewLogger(logger.LOG_LEVEL_INFO, "statsd")
	}

	go listen()
	monitor()
}
<commit_msg>Move regex out of critical path<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"github.com/jcoene/gologger"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"regexp"
	"sort"
	"strconv"
	"time"
)

type Packet struct {
	Bucket   string
	Value    string
	Modifier string
	Sampling float32
}

var log *logger.Logger
var sanitizeRegexp = regexp.MustCompile("[^a-zA-Z0-9\\-_\\.:\\|@]")
var packetRegexp = regexp.MustCompile("([a-zA-Z0-9_\\.]+):(\\-?[0-9\\.]+)\\|(c|ms|g)(\\|@([0-9\\.]+))?")

var (
	serviceAddress = flag.String("address", "0.0.0.0:8125", "UDP service address")
	libratoUser    = flag.String("user", "", "Librato Username")
	libratoToken   = flag.String("token", "", "Librato API Token")
	flushInterval  = flag.Int64("flush", 60, "Flush Interval (seconds)")
	debug          = flag.Bool("debug", false, "Enable Debugging")
)

var (
	In       = make(chan Packet, 10000)
	counters = make(map[string]int64)
	timers   = make(map[string][]float64)
	gauges   = make(map[string]float64)
)

type Measurement struct {
	Counters []Counter     `json:"counters"`
	Gauges   []interface{} `json:"gauges"`
}

func (m *Measurement) Count() int {
	return (len(m.Counters) + len(m.Gauges))
}

type Counter struct {
	Name  string `json:"name"`
	Value int64  `json:"value"`
}

type SimpleGauge struct {
	Name  string  `json:"name"`
	Value float64 `json:"value"`
}

type ComplexGauge struct {
	Name       string  `json:"name"`
	Count      int     `json:"count"`
	Sum        float64 `json:"sum"`
	Min        float64 `json:"min"`
	Max        float64 `json:"max"`
	SumSquares float64 `json:"sum_squares"`
}

func monitor() {
	t := time.NewTicker(time.Duration(*flushInterval) * time.Second)

	for {
		select {
		case <-t.C:
			if err := submit(); err != nil {
				log.Error("submit: %s", err)
			}
		case s := <-In:
			if s.Modifier == "ms" {
				_, ok := timers[s.Bucket]
				if !ok {
					var t []float64
					timers[s.Bucket] = t
				}
				floatValue, _ := strconv.ParseFloat(s.Value, 64)
				timers[s.Bucket] = append(timers[s.Bucket], floatValue)
			} else if s.Modifier == "g" {
				_, ok := gauges[s.Bucket]
				if !ok {
					gauges[s.Bucket] = float64(0)
				}
				floatValue, _ := strconv.ParseFloat(s.Value, 64)
				gauges[s.Bucket] = floatValue
			} else {
				_, ok := counters[s.Bucket]
				if !ok {
					counters[s.Bucket] = 0
				}
				floatValue, _ := strconv.ParseFloat(s.Value, 32)
				counters[s.Bucket] += int64(float32(floatValue) * (1 / s.Sampling))
			}
		}
	}
}

func submit() (err error) {
	m := new(Measurement)
	m.Counters = make([]Counter, 0)
	m.Gauges = make([]interface{}, 0)

	for k, v := range counters {
		c := new(Counter)
		c.Name = k
		c.Value = v
		m.Counters = append(m.Counters, *c)
	}

	for k, v := range gauges {
		g := new(SimpleGauge)
		g.Name = k
		g.Value = v
		m.Gauges = append(m.Gauges, *g)
	}

	for k, t := range timers {
		g := new(ComplexGauge)
		g.Name = k
		g.Count = len(t)

		if g.Count > 0 {
			sort.Float64s(t)
			g.Min = t[0]
			g.Max = t[len(t)-1]
			for _, v := range t {
				g.Sum += v
				g.SumSquares += (v * v)
			}
		}

		m.Gauges = append(m.Gauges, *g)
	}

	if m.Count() == 0 {
		log.Info("no new measurements in the last %d seconds", *flushInterval)
		return
	}

	payload, err := json.MarshalIndent(m, "", "  ")
	if err != nil {
		return
	}

	log.Debug("sending payload:\n%s", payload)

	req, err := http.NewRequest("POST", "https://metrics-api.librato.com/v1/metrics", bytes.NewBuffer(payload))
	if err != nil {
		return
	}

	req.Header.Add("Content-Type", "application/json")
	req.Header.Set("User-Agent", "statsd/1.0")
	req.SetBasicAuth(*libratoUser, *libratoToken)
	resp, err := http.DefaultClient.Do(req)
	if err == nil && resp.StatusCode != 200 {
		if err == nil {
			raw, _ := ioutil.ReadAll(resp.Body)
			err = errors.New(fmt.Sprintf("%s: %s", resp.Status, string(raw)))
		}
		log.Warn("error sending %d measurements: %s", m.Count(), err)
		return
	}

	log.Info("%d measurements sent", m.Count())

	for k, _ := range timers {
		delete(timers, k)
	}

	return
}

func handle(conn *net.UDPConn, remaddr net.Addr, buf *bytes.Buffer) {
	var packet Packet
	var value string
	s := sanitizeRegexp.ReplaceAllString(buf.String(), "")

	for _, item := range packetRegexp.FindAllStringSubmatch(s, -1) {
		value = item[2]
		if item[3] == "ms" {
			_, err := strconv.ParseFloat(item[2], 32)
			if err != nil {
				value = "0"
			}
		}

		sampleRate, err := strconv.ParseFloat(item[5], 32)
		if err != nil {
			sampleRate = 1
		}

		packet.Bucket = item[1]
		packet.Value = value
		packet.Modifier = item[3]
		packet.Sampling = float32(sampleRate)

		In <- packet
	}
}

func listen() {
	address, _ := net.ResolveUDPAddr("udp", *serviceAddress)
	listener, err := net.ListenUDP("udp", address)
	defer listener.Close()
	if err != nil {
		log.Fatal("unable to listen: %s", err)
		os.Exit(1)
	}

	log.Info("listening for events...")

	for {
		message := make([]byte, 512)
		n, remaddr, error := listener.ReadFrom(message)
		if error != nil {
			continue
		}
		buf := bytes.NewBuffer(message[0:n])
		go handle(listener, remaddr, buf)
	}
}

func main() {
	flag.Parse()

	if *debug {
		log = logger.NewLogger(logger.LOG_LEVEL_DEBUG, "statsd")
	} else {
		log = logger.NewLogger(logger.LOG_LEVEL_INFO, "statsd")
	}

	go listen()
	monitor()
}
<|endoftext|>


<commit_before>package logger

import (
	"fmt"
	"log"

	"github.com/clever/kayvee-go"
)

// M is an alias for map[string]interface{} to make log lines less painful to write.
type M map[string]interface{}

// Logging levels
const (
	CRITICAL = "critical"
	ERROR    = "error"
	WARNING  = "warning"
	INFO     = "info"
	TRACE    = "trace"
)

// Info logs at the info level
func Info(title string, data M) {
	logWithLevel(title, INFO, data)
}

// Trace logs at the trace level
func Trace(title string, data M) {
	logWithLevel(title, TRACE, data)
}

// Warning logs at the warning level
func Warning(title string, data M) {
	logWithLevel(title, WARNING, data)
}

// Critical logs at teh critical level
func Critical(title string, data M) {
	logWithLevel(title, CRITICAL, data)
}

// Error logs an error at the error level
func Error(title string, err error) {
	logWithLevel(title, ERROR, M{"error": fmt.Sprint(err)})
}

// ErrorDetailed logs an error at the error level, along with an extra info you can provide
func ErrorDetailed(title string, err error, extras M) {
	extras["error"] = fmt.Sprint(err)
	logWithLevel(title, ERROR, M{"error": fmt.Sprint(err)})
}

func logWithLevel(title string, level string, data M) {
	formatted := kayvee.FormatLog("moredis", level, title, data)
	log.Println(formatted)
}
<commit_msg>fix logger.ErrorDetailed<commit_after>package logger

import (
	"fmt"
	"log"

	"github.com/clever/kayvee-go"
)

// M is an alias for map[string]interface{} to make log lines less painful to write.
type M map[string]interface{}

// Logging levels
const (
	CRITICAL = "critical"
	ERROR    = "error"
	WARNING  = "warning"
	INFO     = "info"
	TRACE    = "trace"
)

// Info logs at the info level
func Info(title string, data M) {
	logWithLevel(title, INFO, data)
}

// Trace logs at the trace level
func Trace(title string, data M) {
	logWithLevel(title, TRACE, data)
}

// Warning logs at the warning level
func Warning(title string, data M) {
	logWithLevel(title, WARNING, data)
}

// Critical logs at teh critical level
func Critical(title string, data M) {
	logWithLevel(title, CRITICAL, data)
}

// Error logs an error at the error level
func Error(title string, err error) {
	logWithLevel(title, ERROR, M{"error": fmt.Sprint(err)})
}

// ErrorDetailed logs an error at the error level, along with an extra info you can provide
func ErrorDetailed(title string, err error, extras M) {
	extras["error"] = fmt.Sprint(err)
	logWithLevel(title, ERROR, extras)
}

func logWithLevel(title string, level string, data M) {
	formatted := kayvee.FormatLog("moredis", level, title, data)
	log.Println(formatted)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"database/sql"
	"encoding/base64"
	"fmt"
	"io"
	"log"
	"mime"
	"net/mail"
	"net/smtp"
	"os"
	"strconv"

	message "github.com/matsuev/go-message"
	charset "github.com/matsuev/go-message/charset"

	_ "github.com/go-sql-driver/mysql"
)

func main() {
	// Перенаправление логов в файл
	// создать файл лога, установить права доступа
	// l, err := os.OpenFile("./klshmail.log", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
	// l, err := os.OpenFile("/var/log/klshmail.log", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
	// logFatal(err)
	// defer l.Close()

	// log.SetOutput(l)

	// Заголовки исходного сообщения,
	// которые нужно оставить
	hh := []string{
		"MIME-Version",
		"Message-Id",
		"Content-Type",
		"Content-Transfer-Encoding",
		"In-Reply-To",
		"References",
		// "Subject",
	}

	r, err := os.Open("./testmessage.eml")
	if err != nil {
		log.Fatal(err)
	}
	defer r.Close()

	// Читаем сообщение из стандартного ввода
	// msg, err := message.Read(os.Stdin)
	msg, err := message.Read(r)
	logFatal(err)
	log.Println("New message accepted...")

	// Разбор заголовков сообщения
	to, err := getMailHeader("To", msg.Header)
	logFatal(err)
	log.Printf("To: %s <%s>\n", to.Name, to.Address)

	from, err := getMailHeader("From", msg.Header)
	logFatal(err)
	log.Printf("From: %s <%s>\n", from.Name, from.Address)

	hSubject, err := charset.DecodeHeader(msg.Header.Get("Subject"))
	logFatal(err)

	subj := mime.BEncoding.Encode("utf-8", hSubject)

	if to.Address == "" || from.Address == "" {
		log.Fatalln("Empty address. Reject message.")
	}

	// Сокдинение с сервером БД
	db, err := sql.Open("mysql", "klshmail:euXoe8uSha1xu4sh@/klshmail?charset=utf8")
	logFatal(err)

	// Проверка соединения с сервером БД
	err = db.Ping()
	logFatal(err)

	// Запрос данных о списке рассылки
	var lid uint64
	var lprefix string

	err = db.QueryRow(`
		SELECT list.id, list.prefix
		FROM list
		WHERE LCASE(list.email)=TRIM(LCASE(?))
		AND list.active
		`, to.Address).Scan(&lid, &lprefix)
	if err != nil {
		log.Println("No list with address:", to.Address)
		os.Exit(0)
	}

	// Запрос на проверку прав пользователя на отправку сообщений в список
	var uid uint64
	err = db.QueryRow(`
		SELECT user.id
		FROM user
		INNER JOIN user_list
		ON (user_list.lid=?
			AND user.id=user_list.uid
			AND user_list.canwrite
		)
		WHERE LCASE(user.email)=TRIM(LCASE(?))
		AND user.active
		`, lid, from.Address).Scan(&uid)
	if err != nil {
		log.Println("User", from.Address, "can't send messages to", to.Address)
		os.Exit(0)
	}

	// Формирование заголовков нового сообщения
	newHeader := make(message.Header)
	for _, hk := range hh {
		if hv := msg.Header.Get(hk); hv != "" {
			newHeader.Set(hk, hv)
		}
	}

	sender := new(mail.Address)
	sender.Name = from.Name
	sender.Address = from.Address

	from.Name = fmt.Sprintf("%s", lprefix)
	from.Address = to.Address

	newHeader.Set("From", from.String())
	newHeader.Set("Reply-To", to.Address)
	newHeader.Set("Subject", subj)
	newHeader.Set("X-KLSH-Sender", strconv.FormatUint(uid, 10))

	var b bytes.Buffer
	w, err := message.CreateWriter(&b, newHeader)
	if err != nil {
		log.Fatal(err)
	}
	defer w.Close()

	if err := transform(w, msg, sender); err != nil {
		log.Fatal(err)
	}

	// Подключение к SMTP серверу
	c, err := smtp.Dial("127.0.0.1:25")
	if err != nil {
		log.Println("SMTP connection error")
		log.Fatal(err)
	}
	defer c.Close()

	// // Заголовки для отправки
	// c.Mail(to.Address)
	// c.Rcpt(fmt.Sprintf("%v@klshmail", lid))
	//
	// wc, err := c.Data()
	// logFatal(err)
	// defer wc.Close()
	//
	// // Отправка сообщения
	// br := bytes.NewReader(b.Bytes())
	// if _, err = io.Copy(wc, br); err != nil {
	// 	log.Println("SMTP send body error")
	// 	log.Fatalln(err)
	// }

	fmt.Println(b.String())

	// Завершение работы
	log.Println("Message processing done.")

}

func logFatal(e error) {
	if e != nil {
		log.Fatalln(e)
	}
}

const senderHtml string = `<p><b>Сообщение от:</b> %s <<a href="mailto:%s">%s</a>><p>`
const senderPlain string = "| Сообщение от:  %s <%s>\n——\n\n"

func transform(w *message.Writer, e *message.Entity, sender *mail.Address) error {
	if mr := e.MultipartReader(); mr != nil {
		// This is a multipart entity, transform each of its parts
		for {
			p, err := mr.NextPart()
			if err == io.EOF {
				break
			} else if err != nil {
				return err
			}

			fmt.Println(p.Header)

			pw, err := w.CreatePart(p.Header)
			if err != nil {
				return err
			}

			if err := transform(pw, p, sender); err != nil {
				return err
			}

			pw.Close()
		}
		return nil
	} else {
		e.Header.Add("Content-Transfer-Encoding", "base64")
		body := e.Body
		// var newLine string
		// if strings.HasPrefix(e.Header.Get("Content-Type"), "text/plain") {
		// 	newLine = fmt.Sprintf(senderPlain, sender.Name, sender.Address)
		// }
		// if strings.HasPrefix(e.Header.Get("Content-Type"), "text/html") {
		// 	newLine = fmt.Sprintf(senderHtml, sender.Name, sender.Address, sender.Address)
		// }
		// body = io.MultiReader(strings.NewReader(newLine), body)

		rb := make([]byte, 0)
		io.ReadFull(body, rb)

		encoder := base64.NewEncoder(base64.StdEncoding, w)
		_, err := encoder.Write(rb)
		// _, err := io.Copy(w, body)
		return err
	}
}

func getMailHeader(k string, h message.Header) (*mail.Address, error) {
	dh, err := charset.DecodeHeader(h.Get(k))
	if err != nil {
		return nil, err
	}

	rh, err := mail.ParseAddress(dh)
	return rh, err
}
<commit_msg>Update main.go<commit_after>package main

import (
	"bytes"
	"database/sql"
	"encoding/base64"
	"fmt"
	"io"
	"log"
	"mime"
	"net/mail"
	"net/smtp"
	"os"
	"strconv"

	message "github.com/matsuev/go-message"
	charset "github.com/matsuev/go-message/charset"

	_ "github.com/go-sql-driver/mysql"
)

func main() {
	// Перенаправление логов в файл
	// создать файл лога, установить права доступа
	// l, err := os.OpenFile("./klshmail.log", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
	// l, err := os.OpenFile("/var/log/klshmail.log", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
	// logFatal(err)
	// defer l.Close()

	// log.SetOutput(l)

	// Заголовки исходного сообщения,
	// которые нужно оставить
	hh := []string{
		"MIME-Version",
		"Message-Id",
		"Content-Type",
		"Content-Transfer-Encoding",
		"In-Reply-To",
		"References",
		// "Subject",
	}

	r, err := os.Open("./testmessage.eml")
	if err != nil {
		log.Fatal(err)
	}
	defer r.Close()

	// Читаем сообщение из стандартного ввода
	// msg, err := message.Read(os.Stdin)
	msg, err := message.Read(r)
	logFatal(err)
	log.Println("New message accepted...")

	// Разбор заголовков сообщения
	to, err := getMailHeader("To", msg.Header)
	logFatal(err)
	log.Printf("To: %s <%s>\n", to.Name, to.Address)

	from, err := getMailHeader("From", msg.Header)
	logFatal(err)
	log.Printf("From: %s <%s>\n", from.Name, from.Address)

	hSubject, err := charset.DecodeHeader(msg.Header.Get("Subject"))
	logFatal(err)

	subj := mime.BEncoding.Encode("utf-8", hSubject)

	if to.Address == "" || from.Address == "" {
		log.Fatalln("Empty address. Reject message.")
	}

	// Сокдинение с сервером БД
	db, err := sql.Open("mysql", "klshmail:euXoe8uSha1xu4sh@/klshmail?charset=utf8")
	logFatal(err)

	// Проверка соединения с сервером БД
	err = db.Ping()
	logFatal(err)

	// Запрос данных о списке рассылки
	var lid uint64
	var lprefix string

	err = db.QueryRow(`
		SELECT list.id, list.prefix
		FROM list
		WHERE LCASE(list.email)=TRIM(LCASE(?))
		AND list.active
		`, to.Address).Scan(&lid, &lprefix)
	if err != nil {
		log.Println("No list with address:", to.Address)
		os.Exit(0)
	}

	// Запрос на проверку прав пользователя на отправку сообщений в список
	var uid uint64
	err = db.QueryRow(`
		SELECT user.id
		FROM user
		INNER JOIN user_list
		ON (user_list.lid=?
			AND user.id=user_list.uid
			AND user_list.canwrite
		)
		WHERE LCASE(user.email)=TRIM(LCASE(?))
		AND user.active
		`, lid, from.Address).Scan(&uid)
	if err != nil {
		log.Println("User", from.Address, "can't send messages to", to.Address)
		os.Exit(0)
	}

	// Формирование заголовков нового сообщения
	newHeader := make(message.Header)
	for _, hk := range hh {
		if hv := msg.Header.Get(hk); hv != "" {
			newHeader.Set(hk, hv)
		}
	}

	sender := new(mail.Address)
	sender.Name = from.Name
	sender.Address = from.Address

	from.Name = fmt.Sprintf("%s", lprefix)
	from.Address = to.Address

	newHeader.Set("From", from.String())
	newHeader.Set("Reply-To", to.Address)
	newHeader.Set("Subject", subj)
	newHeader.Set("X-KLSH-Sender", strconv.FormatUint(uid, 10))

	var b bytes.Buffer
	w, err := message.CreateWriter(&b, newHeader)
	if err != nil {
		log.Fatal(err)
	}
	defer w.Close()

	if err := transform(w, msg, sender); err != nil {
		log.Fatal(err)
	}

	// Подключение к SMTP серверу
	c, err := smtp.Dial("127.0.0.1:25")
	if err != nil {
		log.Println("SMTP connection error")
		log.Fatal(err)
	}
	defer c.Close()

	// // Заголовки для отправки
	// c.Mail(to.Address)
	// c.Rcpt(fmt.Sprintf("%v@klshmail", lid))
	//
	// wc, err := c.Data()
	// logFatal(err)
	// defer wc.Close()
	//
	// // Отправка сообщения
	// br := bytes.NewReader(b.Bytes())
	// if _, err = io.Copy(wc, br); err != nil {
	// 	log.Println("SMTP send body error")
	// 	log.Fatalln(err)
	// }

	fmt.Println(b.String())

	// Завершение работы
	log.Println("Message processing done.")

}

func logFatal(e error) {
	if e != nil {
		log.Fatalln(e)
	}
}

const senderHtml string = `<p><b>Сообщение от:</b> %s <<a href="mailto:%s">%s</a>><p>`
const senderPlain string = "| Сообщение от:  %s <%s>\n——\n\n"

func transform(w *message.Writer, e *message.Entity, sender *mail.Address) error {
	if mr := e.MultipartReader(); mr != nil {
		// This is a multipart entity, transform each of its parts
		for {
			p, err := mr.NextPart()
			if err == io.EOF {
				break
			} else if err != nil {
				return err
			}

			fmt.Println(p.Header)

			p.Header.Add("Content-Transfer-Encoding", "base64")
			pw, err := w.CreatePart(p.Header)
			if err != nil {
				return err
			}

			if err := transform(pw, p, sender); err != nil {
				return err
			}

			pw.Close()
		}
		return nil
	} else {
		body := e.Body
		// var newLine string
		// if strings.HasPrefix(e.Header.Get("Content-Type"), "text/plain") {
		// 	newLine = fmt.Sprintf(senderPlain, sender.Name, sender.Address)
		// }
		// if strings.HasPrefix(e.Header.Get("Content-Type"), "text/html") {
		// 	newLine = fmt.Sprintf(senderHtml, sender.Name, sender.Address, sender.Address)
		// }
		// body = io.MultiReader(strings.NewReader(newLine), body)

		rb := make([]byte, 0)
		io.ReadFull(body, rb)

		encoder := base64.NewEncoder(base64.StdEncoding, w)
		_, err := encoder.Write(rb)
		// _, err := io.Copy(w, body)
		return err
	}
}

func getMailHeader(k string, h message.Header) (*mail.Address, error) {
	dh, err := charset.DecodeHeader(h.Get(k))
	if err != nil {
		return nil, err
	}

	rh, err := mail.ParseAddress(dh)
	return rh, err
}
<|endoftext|>


<commit_before>package logger

import (
	"fmt"
	"log"
	"sort"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
	"github.com/honeybadger-io/honeybadger-go"
	"github.com/satori/go.uuid"
)

const (
	maxBatchByteSize = 1048576
	maxBatchLength   = 10000
	logEventOverhead = 26
)

type Logger interface {
	Log(t time.Time, s string)
}

type logBatch []*cloudwatchlogs.InputLogEvent

func (b logBatch) Len() int {
	return len(b)
}

func (b logBatch) Less(i, j int) bool {
	return *b[i].Timestamp < *b[j].Timestamp
}

func (b logBatch) Swap(i, j int) {
	b[i], b[j] = b[j], b[i]
}

type CloudWatchLogger struct {
	logGroupName  string
	logStreamName string
	sequenceToken *string
	retention     int
	logs          chan *cloudwatchlogs.InputLogEvent
	batch         logBatch
	batchByteSize int
	timeout       <-chan time.Time
	client        *cloudwatchlogs.CloudWatchLogs
}

func NewCloudWatchLogger(logGroupName string, retention int) (*CloudWatchLogger, error) {
	sess, err := session.NewSession()
	if err != nil {
		return nil, fmt.Errorf("failed to create AWS session: %s", err)
	}

	client := cloudwatchlogs.New(sess)

	cwl := &CloudWatchLogger{
		logGroupName:  logGroupName,
		logStreamName: uuid.NewV4().String(),
		retention:     retention,
		logs:          make(chan *cloudwatchlogs.InputLogEvent, 100),
		client:        client,
	}
	go cwl.worker()
	return cwl, nil
}

func (cwl *CloudWatchLogger) Log(t time.Time, s string) {
	cwl.logs <- &cloudwatchlogs.InputLogEvent{
		Message:   aws.String(s),
		Timestamp: aws.Int64(t.UnixNano() / int64(time.Millisecond)),
	}
}

func (cwl *CloudWatchLogger) worker() {
	cwl.resetBatch()
	for {
		select {
		case logEvent := <-cwl.logs:
			cwl.addToBatch(logEvent)
		case <-cwl.timeout:
			cwl.flush()
		}
	}
}

func (cwl *CloudWatchLogger) addToBatch(logEvent *cloudwatchlogs.InputLogEvent) {
	logEventSize := len(*logEvent.Message) + logEventOverhead

	if logEventSize+cwl.batchByteSize > maxBatchByteSize || len(cwl.batch) == maxBatchLength {
		cwl.flush()
	}

	if cwl.timeout == nil {
		cwl.timeout = time.After(time.Second)
	}

	cwl.batch = append(cwl.batch, logEvent)
	cwl.batchByteSize += logEventSize
}

func (cwl *CloudWatchLogger) flush() {
	batch := cwl.batch
	cwl.resetBatch()
	sort.Sort(batch)
	if err := cwl.sendToCloudWatchLogs(batch); err != nil {
		if honeybadger.Config.APIKey == "" {
			honeybadger.Notify(err)
		}
		log.Println(err)
	}
}

func (cwl *CloudWatchLogger) resetBatch() {
	cwl.batch = logBatch{}
	cwl.batchByteSize = 0
	cwl.timeout = nil
}

func (cwl *CloudWatchLogger) sendToCloudWatchLogs(batch logBatch) error {
	s := time.Now()
	params := &cloudwatchlogs.PutLogEventsInput{
		LogEvents:     batch,
		LogGroupName:  aws.String(cwl.logGroupName),
		LogStreamName: aws.String(cwl.logStreamName),
		SequenceToken: cwl.sequenceToken,
	}
	resp, err := cwl.client.PutLogEvents(params)

	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "ResourceNotFoundException" {
				if err = cwl.createLogStream(); err != nil {
					return err
				}
				return cwl.sendToCloudWatchLogs(batch)
			}
		}
		return fmt.Errorf("PutLogEvents failed: %s", err)
	}
	log.Printf("wrote %d log events (%d bytes) in %s\n", len(batch), cwl.batchByteSize, time.Since(s))

	cwl.sequenceToken = resp.NextSequenceToken
	return nil
}

func (cwl *CloudWatchLogger) createLogStream() error {
	params := &cloudwatchlogs.CreateLogStreamInput{
		LogGroupName:  aws.String(cwl.logGroupName),
		LogStreamName: aws.String(cwl.logStreamName),
	}
	if _, err := cwl.client.CreateLogStream(params); err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "ResourceNotFoundException" {
				if err = cwl.createLogGroup(); err != nil {
					return err
				}
				return cwl.createLogStream()
			}
		}
		return fmt.Errorf("CreateLogStream failed: %s", err)
	}
	return nil
}

func (cwl *CloudWatchLogger) createLogGroup() error {
	params := &cloudwatchlogs.CreateLogGroupInput{
		LogGroupName: aws.String(cwl.logGroupName),
	}
	if _, err := cwl.client.CreateLogGroup(params); err != nil {
		return fmt.Errorf("CreateLogGroup failed: %s", err)
	}
	return cwl.putRetentionPolicy()
}

func (cwl *CloudWatchLogger) putRetentionPolicy() error {
	if cwl.retention == 0 {
		return nil
	}
	params := &cloudwatchlogs.PutRetentionPolicyInput{
		LogGroupName:    aws.String(cwl.logGroupName),
		RetentionInDays: aws.Int64(int64(cwl.retention)),
	}
	_, err := cwl.client.PutRetentionPolicy(params)
	if err != nil {
		return fmt.Errorf("PutRetentionPolicy failed: %s", err)
	}
	return nil
}
<commit_msg>Pass the batch size to sendToCloudWatchLogs<commit_after>package logger

import (
	"fmt"
	"log"
	"sort"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
	"github.com/honeybadger-io/honeybadger-go"
	"github.com/satori/go.uuid"
)

const (
	maxBatchByteSize = 1048576
	maxBatchLength   = 10000
	logEventOverhead = 26
)

type Logger interface {
	Log(t time.Time, s string)
}

type logBatch []*cloudwatchlogs.InputLogEvent

func (b logBatch) Len() int {
	return len(b)
}

func (b logBatch) Less(i, j int) bool {
	return *b[i].Timestamp < *b[j].Timestamp
}

func (b logBatch) Swap(i, j int) {
	b[i], b[j] = b[j], b[i]
}

type CloudWatchLogger struct {
	logGroupName  string
	logStreamName string
	sequenceToken *string
	retention     int
	logs          chan *cloudwatchlogs.InputLogEvent
	batch         logBatch
	batchByteSize int
	timeout       <-chan time.Time
	client        *cloudwatchlogs.CloudWatchLogs
}

func NewCloudWatchLogger(logGroupName string, retention int) (*CloudWatchLogger, error) {
	sess, err := session.NewSession()
	if err != nil {
		return nil, fmt.Errorf("failed to create AWS session: %s", err)
	}

	client := cloudwatchlogs.New(sess)

	cwl := &CloudWatchLogger{
		logGroupName:  logGroupName,
		logStreamName: uuid.NewV4().String(),
		retention:     retention,
		logs:          make(chan *cloudwatchlogs.InputLogEvent, 100),
		client:        client,
	}
	go cwl.worker()
	return cwl, nil
}

func (cwl *CloudWatchLogger) Log(t time.Time, s string) {
	cwl.logs <- &cloudwatchlogs.InputLogEvent{
		Message:   aws.String(s),
		Timestamp: aws.Int64(t.UnixNano() / int64(time.Millisecond)),
	}
}

func (cwl *CloudWatchLogger) worker() {
	cwl.resetBatch()
	for {
		select {
		case logEvent := <-cwl.logs:
			cwl.addToBatch(logEvent)
		case <-cwl.timeout:
			cwl.flush()
		}
	}
}

func (cwl *CloudWatchLogger) addToBatch(logEvent *cloudwatchlogs.InputLogEvent) {
	logEventSize := len(*logEvent.Message) + logEventOverhead

	if logEventSize+cwl.batchByteSize > maxBatchByteSize || len(cwl.batch) == maxBatchLength {
		cwl.flush()
	}

	if cwl.timeout == nil {
		cwl.timeout = time.After(time.Second)
	}

	cwl.batch = append(cwl.batch, logEvent)
	cwl.batchByteSize += logEventSize
}

func (cwl *CloudWatchLogger) flush() {
	batch := cwl.batch
	batchByteSize := cwl.batchByteSize
	cwl.resetBatch()
	sort.Sort(batch)
	if err := cwl.sendToCloudWatchLogs(batch, batchByteSize); err != nil {
		if honeybadger.Config.APIKey == "" {
			honeybadger.Notify(err)
		}
		log.Println(err)
	}
}

func (cwl *CloudWatchLogger) resetBatch() {
	cwl.batch = logBatch{}
	cwl.batchByteSize = 0
	cwl.timeout = nil
}

func (cwl *CloudWatchLogger) sendToCloudWatchLogs(batch logBatch, batchByteSize int) error {
	s := time.Now()
	params := &cloudwatchlogs.PutLogEventsInput{
		LogEvents:     batch,
		LogGroupName:  aws.String(cwl.logGroupName),
		LogStreamName: aws.String(cwl.logStreamName),
		SequenceToken: cwl.sequenceToken,
	}
	resp, err := cwl.client.PutLogEvents(params)

	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "ResourceNotFoundException" {
				if err = cwl.createLogStream(); err != nil {
					return err
				}
				return cwl.sendToCloudWatchLogs(batch, batchByteSize)
			}
		}
		return fmt.Errorf("PutLogEvents failed: %s", err)
	}
	log.Printf("wrote %d log events (%d bytes) in %s\n", len(batch), batchByteSize, time.Since(s))

	cwl.sequenceToken = resp.NextSequenceToken
	return nil
}

func (cwl *CloudWatchLogger) createLogStream() error {
	params := &cloudwatchlogs.CreateLogStreamInput{
		LogGroupName:  aws.String(cwl.logGroupName),
		LogStreamName: aws.String(cwl.logStreamName),
	}
	if _, err := cwl.client.CreateLogStream(params); err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "ResourceNotFoundException" {
				if err = cwl.createLogGroup(); err != nil {
					return err
				}
				return cwl.createLogStream()
			}
		}
		return fmt.Errorf("CreateLogStream failed: %s", err)
	}
	return nil
}

func (cwl *CloudWatchLogger) createLogGroup() error {
	params := &cloudwatchlogs.CreateLogGroupInput{
		LogGroupName: aws.String(cwl.logGroupName),
	}
	if _, err := cwl.client.CreateLogGroup(params); err != nil {
		return fmt.Errorf("CreateLogGroup failed: %s", err)
	}
	return cwl.putRetentionPolicy()
}

func (cwl *CloudWatchLogger) putRetentionPolicy() error {
	if cwl.retention == 0 {
		return nil
	}
	params := &cloudwatchlogs.PutRetentionPolicyInput{
		LogGroupName:    aws.String(cwl.logGroupName),
		RetentionInDays: aws.Int64(int64(cwl.retention)),
	}
	_, err := cwl.client.PutRetentionPolicy(params)
	if err != nil {
		return fmt.Errorf("PutRetentionPolicy failed: %s", err)
	}
	return nil
}
<|endoftext|>


<commit_before>// netbackup - Consistent multi-method backup tool
//
// See instructions in the README.md file that accompanies this program.
//
// (C) 2015 by Marco Paganini <paganini AT paganini DOT net>

package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	"github.com/marcopaganini/logger"
	"github.com/marcopaganini/netbackup/config"
	"github.com/marcopaganini/netbackup/execute"
	"github.com/marcopaganini/netbackup/transports"
)

const (
	defaultLogDir = "/tmp/log/netbackup"
	devMapperDir  = "/dev/mapper"

	// Default permissions for log directories and files.
	// The current umask will apply to these.
	defaultLogDirMode  = 0777
	defaultLogFileMode = 0666

	// Return codes
	osSuccess = 0
	osError   = 1

	// External commands.
	mountCmd      = "mount"
	umountCmd     = "umount"
	cryptSetupCmd = "cryptsetup"
	fsckCmd       = "fsck"
	tunefsCmd     = "tune2fs"
	dfCmd         = "df"
)

var (
	// Generic logging object
	log *logger.Logger
)

// usage prints an error message and program usage to stderr, exiting after
// that.
func usage(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %v\n\n", err)
	}
	fmt.Fprintf(os.Stderr, "Usage%s:\n", os.Args[0])
	flag.PrintDefaults()
	os.Exit(2)
}

// createOutputLog creates a new output log file. If logPath is set, it is used
// unchanged. If not, a new log is created based under logDir using the
// configuration name, and the system date. Intermediate directories are
// created as needed. Returns a *os.File, the file created, error.
func createOutputLog(logPath string, logDir string, configName string) (*os.File, string, error) {
	path := logPath
	if path == "" {
		dir := filepath.Join(logDir, configName)
		if err := os.MkdirAll(dir, defaultLogDirMode); err != nil {
			return nil, "", fmt.Errorf("Error trying to create dir tree %q: %v", dir, err)
		}
		ymd := time.Now().Format("2006-01-02")
		path = filepath.Join(dir, configName+"-"+ymd+".log")
	}

	w, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY|os.O_CREATE, defaultLogFileMode)
	if err != nil {
		return nil, path, fmt.Errorf("Error opening %q: %v", path, err)
	}
	return w, path, err
}

// shellRun run a command string using the shell using the specified execute object.
func shellRun(e *execute.Execute, cmd string) error {
	shell := os.Getenv("SHELL")
	if shell == "" {
		shell = "/bin/sh"
	}
	a := []string{shell, "-c", "--", cmd}
	return e.Exec(a)
}

// runCommand executes the pre or post commands using the shell. A prefix will
// be used to log the commands to the output log (usually, "PRE" for
// pre-commands or "POST" for post-commands Returns error.
func runCommand(prefix string, cmd string, ex *execute.Execute, outLog io.Writer) error {
	m := fmt.Sprintf("%s Command: %q", prefix, cmd)
	log.Verboseln(int(opt.verbose), m)
	if opt.dryrun {
		return nil
	}

	// Create a new execute object, if current is nil
	e := ex
	if e == nil {
		e = execute.New()
	}

	// All streams copied to output log with "PRE:" as a prefix.
	e.SetStdout(func(buf string) error { _, err := fmt.Fprintf(outLog, "%s(stdout): %s\n", prefix, buf); return err })
	e.SetStderr(func(buf string) error { _, err := fmt.Fprintf(outLog, "%s(stderr): %s\n", prefix, buf); return err })

	fmt.Fprintf(outLog, "*** %s\n", m)
	err := shellRun(e, cmd)
	fmt.Fprintf(outLog, "*** %s returned: %v\n", prefix, err)

	return err
}

// mountDestDev mounts the destination device specified in config.DestDev into
// a temporary mount point and set config.DestDir to point to this directory.
func mountDestDev(config *config.Config, outLog io.Writer) error {
	tmpdir, err := ioutil.TempDir("", "netbackup_mount")
	if err != nil {
		return fmt.Errorf("unable to create temp directory: %v", err)
	}

	// We use the mount command instead of the mount syscal as it makes
	// simpler to specify defaults in /etc/fstab.
	cmd := mountCmd + " " + config.DestDev + " " + tmpdir
	if err := runCommand("MOUNT", cmd, nil, outLog); err != nil {
		return fmt.Errorf("error running %q: %v", cmd, err)
	}

	config.DestDir = tmpdir
	return nil
}

// umountDestDev dismounts the destination device specified in config.DestDev.
func umountDestDev(config *config.Config, outLog io.Writer) error {
	cmd := umountCmd + " " + config.DestDev
	if err := runCommand("UMOUNT", cmd, nil, outLog); err != nil {
		return fmt.Errorf("error running %q: %v", cmd, err)
	}
	return nil
}

// openLuksDestDev opens the luks device specified by config.LuksDestDev and sets
// config.DestDev to the /dev/mapper device.
func openLuksDestDev(config *config.Config, outLog io.Writer) error {
	// Our temporary dev/mapper device is based on the config name
	devname := "netbackup_" + config.Name
	devfile := filepath.Join(devMapperDir, devname)

	// Make sure it doesn't already exist
	if _, err := os.Stat(devfile); err == nil {
		return fmt.Errorf("device mapper file %q already exists.", devfile)
	}

	// cryptsetup LuksOpen
	cmd := cryptSetupCmd
	if config.LuksKeyFile != "" {
		cmd += " --key-file " + config.LuksKeyFile
	}
	cmd += " luksOpen " + config.LuksDestDev + " " + devname
	if err := runCommand("LUKS_OPEN", cmd, nil, outLog); err != nil {
		return fmt.Errorf("error running %q: %v", cmd, err)
	}

	// Set the destination device to devfile so the normal processing
	// will be sufficient to mount and dismount this device.
	config.DestDev = devfile
	return nil
}

// closeLuksDestDev closes the luks device specified by config.LuksDestDev.
func closeLuksDestDev(config *config.Config, outLog io.Writer) error {
	// Note that even though this function is called closeLuksDestDev we use
	// the mount point under /dev/mapper to close the device.  The mount point
	// was previously set by openLuksDestDev.
	cmd := cryptSetupCmd + " luksClose " + config.DestDev
	if err := runCommand("LUKS_CLOSE", cmd, nil, outLog); err != nil {
		return fmt.Errorf("error running %q: %v", cmd, err)
	}
	return nil
}

// displayDiskSpace runs "df" on the system and writes the output to the logfile.
func displayDiskSpace(config *config.Config, outLog io.Writer) error {
	cmd := dfCmd + " -m"
	if err := runCommand("DF", cmd, nil, outLog); err != nil {
		return fmt.Errorf("error running %q: %v", cmd, err)
	}
	return nil
}

// fsCleanup runs fsck to make sure the filesystem under config.dest_dev is
// intact, and sets the number of times to check to 0 and the last time
// checked to now. This option should only be used in EXTn filesystems or
// filesystems that support tunefs.
func fsCleanup(config *config.Config, outLog io.Writer) error {
	// fsck (read-only check)
	cmd := fsckCmd + " -n " + config.DestDev
	if err := runCommand("FS_CLEANUP", cmd, nil, outLog); err != nil {
		return fmt.Errorf("error running %q: %v", cmd, err)
	}
	// Tunefs
	cmd = tunefsCmd + " -C 0 -T now " + config.DestDev
	if err := runCommand("FS_CLEANUP", cmd, nil, outLog); err != nil {
		return fmt.Errorf("error running %q: %v", cmd, err)
	}
	return nil
}

func backup() int {
	var transp interface {
		Run() error
	}
	log = logger.New("")

	// Parse command line flags and read config file.
	if err := parseFlags(); err != nil {
		log.Printf("Error: %v", err)
		return osError
	}
	// Set verbose level
	if opt.verbose > 0 {
		log.SetVerboseLevel(int(opt.verbose))
	}
	if opt.dryrun {
		log.Verbosef(2, "Warning: Dry-Run mode. Won't execute any commands.")
	}

	// Open and parse config file
	cfg, err := os.Open(opt.config)
	if err != nil {
		log.Printf("Unable to open config file: %v", err)
		return osError
	}
	config, err := config.ParseConfig(cfg)
	if err != nil {
		log.Printf("Configuration error in %q: %v", opt.config, err)
		return osError
	}

	// Open or create the output log file. This log will contain a transcript
	// of stdout and stderr from all commands executed by this program.
	outLog, outPath, err := createOutputLog(config.Logfile, defaultLogDir, config.Name)
	if err != nil {
		log.Printf("Error opening output logfile: %q: %v", outPath, err)
		return osError
	}
	defer outLog.Close()

	if !opt.dryrun {
		// Open LUKS device, if needed
		if config.LuksDestDev != "" {
			if err := openLuksDestDev(config, outLog); err != nil {
				log.Printf("Error opening LUKS device %q: %v", config.LuksDestDev, err)
				return osError
			}
			// close luks device at the end
			defer closeLuksDestDev(config, outLog)
			defer time.Sleep(2 * time.Second)
		}

		// Run cleanup on fs prior to backup, if requested.
		if config.FSCleanup {
			if err := fsCleanup(config, outLog); err != nil {
				log.Printf("Error performing pre-backup cleanup on %q: %v", config.DestDev, err)
				return osError
			}
		}

		// Mount destination device, if needed.
		if config.DestDev != "" {
			if err := mountDestDev(config, outLog); err != nil {
				log.Printf("Error opening destination device %q: %v", config.DestDev, err)
				return osError
			}
			// umount destination filesystem and remove temp mount point.
			defer os.Remove(config.DestDir)
			defer umountDestDev(config, outLog)
			// For some reason, not having a pause before attempting to unmount
			// can generate a race condition where umount complains that the fs
			// is busy (even though the transport is already down.)
			defer time.Sleep(2 * time.Second)
		}
	}

	// Create new transport based on config.Transport
	switch config.Transport {
	case "rclone":
		transp, err = transports.NewRcloneTransport(config, nil, outLog, int(opt.verbose), opt.dryrun)
	case "rdiff-backup":
		transp, err = transports.NewRdiffBackupTransport(config, nil, outLog, int(opt.verbose), opt.dryrun)
	case "rsync":
		transp, err = transports.NewRsyncTransport(config, nil, outLog, int(opt.verbose), opt.dryrun)
	default:
		log.Printf("Unknown transport %q", config.Transport)
		return osError
	}
	if err != nil {
		log.Printf("Error creating %s transport: %v", config.Transport, err)
		return osError
	}

	// Execute pre-commands, if any.
	if config.PreCommand != "" {
		if err := runCommand("PRE", config.PreCommand, nil, outLog); err != nil {
			log.Printf("Error running pre-command: %v", err)
			return osError
		}
	}

	// Make it so...
	if err := transp.Run(); err != nil {
		log.Printf("Error running backup: %v", err)
		return osError
	}
	fmt.Fprintf(outLog, "*** Backup Result: Success\n")

	// Execute post-commands, if any.
	if config.PostCommand != "" {
		if err := runCommand("POST", config.PostCommand, nil, outLog); err != nil {
			fmt.Fprintf(outLog, "*** Backup Result: Failure (%v)\n", err)
			log.Printf("Error running post-command: %v", err)
			return osError
		}
	}

	// Log df output (best effort)
	displayDiskSpace(config, outLog)

	return osSuccess
}

func main() {
	os.Exit(backup())
}
<commit_msg>Improve runCommand and make output more consistent.<commit_after>// netbackup - Consistent multi-method backup tool
//
// See instructions in the README.md file that accompanies this program.
//
// (C) 2015 by Marco Paganini <paganini AT paganini DOT net>

package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	"github.com/marcopaganini/logger"
	"github.com/marcopaganini/netbackup/config"
	"github.com/marcopaganini/netbackup/execute"
	"github.com/marcopaganini/netbackup/transports"
)

const (
	defaultLogDir = "/tmp/log/netbackup"
	devMapperDir  = "/dev/mapper"

	// Default permissions for log directories and files.
	// The current umask will apply to these.
	defaultLogDirMode  = 0777
	defaultLogFileMode = 0666

	// Return codes
	osSuccess = 0
	osError   = 1

	// External commands.
	mountCmd      = "mount"
	umountCmd     = "umount"
	cryptSetupCmd = "cryptsetup"
	fsckCmd       = "fsck"
	tunefsCmd     = "tune2fs"
	dfCmd         = "df"
)

var (
	// Generic logging object
	log *logger.Logger
)

// usage prints an error message and program usage to stderr, exiting after
// that.
func usage(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %v\n\n", err)
	}
	fmt.Fprintf(os.Stderr, "Usage%s:\n", os.Args[0])
	flag.PrintDefaults()
	os.Exit(2)
}

// createOutputLog creates a new output log file. If logPath is set, it is used
// unchanged. If not, a new log is created based under logDir using the
// configuration name, and the system date. Intermediate directories are
// created as needed. Returns a *os.File, the file created, error.
func createOutputLog(logPath string, logDir string, configName string) (*os.File, string, error) {
	path := logPath
	if path == "" {
		dir := filepath.Join(logDir, configName)
		if err := os.MkdirAll(dir, defaultLogDirMode); err != nil {
			return nil, "", fmt.Errorf("Error trying to create dir tree %q: %v", dir, err)
		}
		ymd := time.Now().Format("2006-01-02")
		path = filepath.Join(dir, configName+"-"+ymd+".log")
	}

	w, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY|os.O_CREATE, defaultLogFileMode)
	if err != nil {
		return nil, path, fmt.Errorf("Error opening %q: %v", path, err)
	}
	return w, path, err
}

// runCommand executes the pre or post commands using the shell. A prefix will
// be used to log the commands to the output log (usually, "PRE" for
// pre-commands or "POST" for post-commands Returns error.
func runCommand(prefix string, cmd string, ex *execute.Execute, outLog io.Writer) error {
	m := fmt.Sprintf("%s Command: %q", prefix, cmd)
	fmt.Fprintf(outLog, "%s\n", m)
	log.Verboseln(int(opt.verbose), m)

	if opt.dryrun {
		return nil
	}

	// Create a new execute object, if current is nil
	e := ex
	if e == nil {
		e = execute.New()
	}

	// All streams copied to output log with "PRE:" as a prefix.
	e.SetStdout(func(buf string) error { _, err := fmt.Fprintf(outLog, "%s(stdout): %s\n", prefix, buf); return err })
	e.SetStderr(func(buf string) error { _, err := fmt.Fprintf(outLog, "%s(stderr): %s\n", prefix, buf); return err })

	// Run using shell
	shell := os.Getenv("SHELL")
	if shell == "" {
		shell = "/bin/sh"
	}
	err := e.Exec([]string{shell, "-c", "--", cmd})
	if err != nil {
		errmsg := fmt.Sprintf("%s returned: %v", prefix, err)
		fmt.Fprintf(outLog, "*** %s\n", errmsg)
		return fmt.Errorf(errmsg)
	}
	fmt.Fprintf(outLog, "%s returned: OK\n", prefix)
	return nil
}

// mountDestDev mounts the destination device specified in config.DestDev into
// a temporary mount point and set config.DestDir to point to this directory.
func mountDestDev(config *config.Config, outLog io.Writer) error {
	tmpdir, err := ioutil.TempDir("", "netbackup_mount")
	if err != nil {
		return fmt.Errorf("unable to create temp directory: %v", err)
	}

	// We use the mount command instead of the mount syscal as it makes
	// simpler to specify defaults in /etc/fstab.
	cmd := mountCmd + " " + config.DestDev + " " + tmpdir
	if err := runCommand("MOUNT", cmd, nil, outLog); err != nil {
		return err
	}

	config.DestDir = tmpdir
	return nil
}

// umountDestDev dismounts the destination device specified in config.DestDev.
func umountDestDev(config *config.Config, outLog io.Writer) error {
	cmd := umountCmd + " " + config.DestDev
	return runCommand("UMOUNT", cmd, nil, outLog)
}

// openLuksDestDev opens the luks device specified by config.LuksDestDev and sets
// config.DestDev to the /dev/mapper device.
func openLuksDestDev(config *config.Config, outLog io.Writer) error {
	// Our temporary dev/mapper device is based on the config name
	devname := "netbackup_" + config.Name
	devfile := filepath.Join(devMapperDir, devname)

	// Make sure it doesn't already exist
	if _, err := os.Stat(devfile); err == nil {
		return fmt.Errorf("device mapper file %q already exists", devfile)
	}

	// cryptsetup LuksOpen
	cmd := cryptSetupCmd
	if config.LuksKeyFile != "" {
		cmd += " --key-file " + config.LuksKeyFile
	}
	cmd += " luksOpen " + config.LuksDestDev + " " + devname
	if err := runCommand("LUKS_OPEN", cmd, nil, outLog); err != nil {
		return err
	}

	// Set the destination device to devfile so the normal processing
	// will be sufficient to mount and dismount this device.
	config.DestDev = devfile
	return nil
}

// closeLuksDestDev closes the luks device specified by config.LuksDestDev.
func closeLuksDestDev(config *config.Config, outLog io.Writer) error {
	// Note that even though this function is called closeLuksDestDev we use
	// the mount point under /dev/mapper to close the device.  The mount point
	// was previously set by openLuksDestDev.
	cmd := cryptSetupCmd + " luksClose " + config.DestDev
	return runCommand("LUKS_CLOSE", cmd, nil, outLog)
}

// displayDiskSpace runs "df" on the system and writes the output to the logfile.
func displayDiskSpace(config *config.Config, outLog io.Writer) error {
	cmd := dfCmd + " -m"
	return runCommand("DF", cmd, nil, outLog)
}

// fsCleanup runs fsck to make sure the filesystem under config.dest_dev is
// intact, and sets the number of times to check to 0 and the last time
// checked to now. This option should only be used in EXTn filesystems or
// filesystems that support tunefs.
func fsCleanup(config *config.Config, outLog io.Writer) error {
	// fsck (read-only check)
	cmd := fsckCmd + " -n " + config.DestDev
	if err := runCommand("FS_CLEANUP", cmd, nil, outLog); err != nil {
		return fmt.Errorf("error running %q: %v", cmd, err)
	}
	// Tunefs
	cmd = tunefsCmd + " -C 0 -T now " + config.DestDev
	return runCommand("FS_CLEANUP", cmd, nil, outLog)
}

func backup() int {
	var transp interface {
		Run() error
	}
	log = logger.New("")

	// Parse command line flags and read config file.
	if err := parseFlags(); err != nil {
		log.Printf("Error: %v", err)
		return osError
	}
	// Set verbose level
	if opt.verbose > 0 {
		log.SetVerboseLevel(int(opt.verbose))
	}
	if opt.dryrun {
		log.Verbosef(2, "Warning: Dry-Run mode. Won't execute any commands.")
	}

	// Open and parse config file
	cfg, err := os.Open(opt.config)
	if err != nil {
		log.Printf("Unable to open config file: %v", err)
		return osError
	}
	config, err := config.ParseConfig(cfg)
	if err != nil {
		log.Printf("Configuration error in %q: %v", opt.config, err)
		return osError
	}

	// Open or create the output log file. This log will contain a transcript
	// of stdout and stderr from all commands executed by this program.
	outLog, outPath, err := createOutputLog(config.Logfile, defaultLogDir, config.Name)
	if err != nil {
		log.Printf("Error opening output logfile: %q: %v", outPath, err)
		return osError
	}
	defer outLog.Close()

	if !opt.dryrun {
		// Open LUKS device, if needed
		if config.LuksDestDev != "" {
			if err := openLuksDestDev(config, outLog); err != nil {
				log.Printf("Error opening LUKS device %q: %v", config.LuksDestDev, err)
				return osError
			}
			// close luks device at the end
			defer closeLuksDestDev(config, outLog)
			defer time.Sleep(2 * time.Second)
		}

		// Run cleanup on fs prior to backup, if requested.
		if config.FSCleanup {
			if err := fsCleanup(config, outLog); err != nil {
				log.Printf("Error performing pre-backup cleanup on %q: %v", config.DestDev, err)
				return osError
			}
		}

		// Mount destination device, if needed.
		if config.DestDev != "" {
			if err := mountDestDev(config, outLog); err != nil {
				log.Printf("Error opening destination device %q: %v", config.DestDev, err)
				return osError
			}
			// umount destination filesystem and remove temp mount point.
			defer os.Remove(config.DestDir)
			defer umountDestDev(config, outLog)
			// For some reason, not having a pause before attempting to unmount
			// can generate a race condition where umount complains that the fs
			// is busy (even though the transport is already down.)
			defer time.Sleep(2 * time.Second)
		}
	}

	// Create new transport based on config.Transport
	switch config.Transport {
	case "rclone":
		transp, err = transports.NewRcloneTransport(config, nil, outLog, int(opt.verbose), opt.dryrun)
	case "rdiff-backup":
		transp, err = transports.NewRdiffBackupTransport(config, nil, outLog, int(opt.verbose), opt.dryrun)
	case "rsync":
		transp, err = transports.NewRsyncTransport(config, nil, outLog, int(opt.verbose), opt.dryrun)
	default:
		log.Printf("Unknown transport %q", config.Transport)
		return osError
	}
	if err != nil {
		log.Printf("Error creating %s transport: %v", config.Transport, err)
		return osError
	}

	// Execute pre-commands, if any.
	if config.PreCommand != "" {
		if err := runCommand("PRE", config.PreCommand, nil, outLog); err != nil {
			log.Printf("Error running pre-command: %v", err)
			return osError
		}
	}

	// Make it so...
	if err := transp.Run(); err != nil {
		log.Printf("Error running backup: %v", err)
		return osError
	}
	fmt.Fprintf(outLog, "*** Backup Result: Success\n")

	// Execute post-commands, if any.
	if config.PostCommand != "" {
		if err := runCommand("POST", config.PostCommand, nil, outLog); err != nil {
			fmt.Fprintf(outLog, "*** Backup Result: Failure (%v)\n", err)
			log.Printf("Error running post-command: %v", err)
			return osError
		}
	}

	// Log df output (best effort)
	displayDiskSpace(config, outLog)

	return osSuccess
}

func main() {
	os.Exit(backup())
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/hex"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"os/user"
	"path/filepath"
	"runtime/pprof"
)

var (
	cpuprofile = flag.String("cpuprofile", "", "If not empty, collects CPU profile samples and writes the profile to the given file before the program exits")
	memprofile = flag.String("memprofile", "", "If not empty, writes memory heap allocations to the given file before the program exits")
	id         = flag.String("id", "", "The id of the share")
)

var torrent string

func main() {
	flag.Usage = usage
	flag.Parse()

	if *id == "" {
		fmt.Println("Missing a share id")
		usage()
		os.Exit(2)
	}
	shareID := NewShareID(*id)

	if flag.NArg() != 0 {
		log.Println("Don't want arguments")
		usage()
	}

	if *cpuprofile != "" {
		cpuf, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(cpuf)
		defer pprof.StopCPUProfile()
	}

	if *memprofile != "" {
		defer func(file string) {
			memf, err := os.Create(file)
			if err != nil {
				log.Fatal(err)
			}
			pprof.WriteHeapProfile(memf)
		}(*memprofile)
	}

	// Bitshare dir
	u, err := user.Current()
	if err != nil {
		log.Fatal("Couldn't watch dir: ", err)
	}
	pathArgs := []string{u.HomeDir, ".local", "share", "bitshare"}
	bitshareDir := filepath.Join(pathArgs...)

	// Watcher
	if fileDir == "." {
		fmt.Println("fileDir option is missing")
		flag.Usage()
	}
	_, err = os.Stat(fileDir)
	if err != nil {
		fmt.Printf("%s is an invalid dir: %s\n", fileDir, err)
		os.Exit(1)
	}
	watcher := NewWatcher(bitshareDir, filepath.Clean(fileDir))

	log.Println("Starting.")

	// torrentDir and torrentFile
	user, err := user.Current()
	if err != nil {
		log.Fatal("Couldn't get current user: ", err)
	}
	torrentDir := filepath.Join(user.HomeDir, ".local", "share", "bitshare")
	if _, err := os.Stat(torrentDir); os.IsNotExist(err) {
		err := os.MkdirAll(torrentDir, os.ModeDir|0755)
		if err != nil {
			log.Fatal("Couldn't make torrent dir:", err)
		}
	}

	// External listener
	conChan, listenPort, err := listenForPeerConnections()
	if err != nil {
		log.Fatal("Couldn't listen for peers connection: ", err)
	}

	currentSession := NewCurrentSession(torrentDir, listenPort)

	// quitChan
	quitChan := listenSigInt()

	// LPD
	lpd, err := NewAnnouncer(listenPort)
	if err != nil {
		log.Fatal("Couldn't listen for Local Peer Discoveries: ", err)
	}

	// Control session
	controlSession, err := NewControlSession(shareID, listenPort)
	if err != nil {
		log.Fatal(err)
	}
	id, err := hex.DecodeString(shareID.PublicID())
	if err != nil {
		log.Fatal(err)
	}
	lpd.Announce(string(id))

mainLoop:
	for {
		select {
		case <-quitChan:
			err := currentSession.Quit()
			if err != nil {
				log.Println("Failed: ", err)
			} else {
				log.Println("Done")
			}
			break mainLoop
		case c := <-conChan:
			if currentSession.Matches(c.infohash) {
				currentSession.AcceptNewPeer(c)
			} else if controlSession.Matches(c.infohash) {
				controlSession.AcceptNewPeer(c)
			}
		case announce := <-lpd.announces:
			hexhash, err := hex.DecodeString(announce.infohash)
			if err != nil {
				log.Println("Err with hex-decoding:", err)
				break
			}
			if currentSession.Matches(string(hexhash)) {
				currentSession.hintNewPeer(announce.peer)
			} else if controlSession.Matches(string(hexhash)) {
				controlSession.hintNewPeer(announce.peer)
			}
		case ih := <-watcher.PingNewTorrent:
			controlSession.Broadcast(ih)

			torrentFile := filepath.Join(bitshareDir, fmt.Sprintf("%x", ih))
			currentSession, err = NewTorrentSession(torrentFile, listenPort)
			if err != nil {
				log.Fatal("Couldn't start new session: ", err)
			}
			go currentSession.DoTorrent()
			lpd.Announce(ih)
		case ih := <-controlSession.Torrents:
			log.Printf("Got new metadata from control session: %x\n", ih)
		}
	}

}

func usage() {
	log.Printf("usage: Taipei-Torrent [options] (torrent-file | torrent-url)")

	flag.PrintDefaults()
	os.Exit(2)
}

func listenSigInt() chan os.Signal {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill)
	return c
}

func NewCurrentSession(bitshareDir string, listenPort int) *TorrentSession {
	currentFile := filepath.Join(bitshareDir, "current")
	if _, err := os.Stat(currentFile); err != nil && os.IsNotExist(err) {
		log.Fatal("No current file!")
	}

	currentIH, err := ioutil.ReadFile(currentFile)
	if err != nil {
		log.Fatal(err)
	}

	currentTorrent := filepath.Join(bitshareDir, string(currentIH))

	ts, err := NewTorrentSession(currentTorrent, listenPort)
	if err != nil {
		log.Fatal(err)
	}

	return ts
}
<commit_msg>Don't use lpd if useLPD flag is set to false<commit_after>package main

import (
	"encoding/hex"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"os/user"
	"path/filepath"
	"runtime/pprof"
)

var (
	cpuprofile = flag.String("cpuprofile", "", "If not empty, collects CPU profile samples and writes the profile to the given file before the program exits")
	memprofile = flag.String("memprofile", "", "If not empty, writes memory heap allocations to the given file before the program exits")
	id         = flag.String("id", "", "The id of the share")
)

var torrent string

func main() {
	flag.Usage = usage
	flag.Parse()

	if *id == "" {
		fmt.Println("Missing a share id")
		usage()
		os.Exit(2)
	}
	shareID := NewShareID(*id)

	if flag.NArg() != 0 {
		log.Println("Don't want arguments")
		usage()
	}

	if *cpuprofile != "" {
		cpuf, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(cpuf)
		defer pprof.StopCPUProfile()
	}

	if *memprofile != "" {
		defer func(file string) {
			memf, err := os.Create(file)
			if err != nil {
				log.Fatal(err)
			}
			pprof.WriteHeapProfile(memf)
		}(*memprofile)
	}

	// Bitshare dir
	u, err := user.Current()
	if err != nil {
		log.Fatal("Couldn't watch dir: ", err)
	}
	pathArgs := []string{u.HomeDir, ".local", "share", "bitshare"}
	bitshareDir := filepath.Join(pathArgs...)

	// Watcher
	if fileDir == "." {
		fmt.Println("fileDir option is missing")
		flag.Usage()
	}
	_, err = os.Stat(fileDir)
	if err != nil {
		fmt.Printf("%s is an invalid dir: %s\n", fileDir, err)
		os.Exit(1)
	}
	watcher := NewWatcher(bitshareDir, filepath.Clean(fileDir))

	log.Println("Starting.")

	// torrentDir and torrentFile
	user, err := user.Current()
	if err != nil {
		log.Fatal("Couldn't get current user: ", err)
	}
	torrentDir := filepath.Join(user.HomeDir, ".local", "share", "bitshare")
	if _, err := os.Stat(torrentDir); os.IsNotExist(err) {
		err := os.MkdirAll(torrentDir, os.ModeDir|0755)
		if err != nil {
			log.Fatal("Couldn't make torrent dir:", err)
		}
	}

	// External listener
	conChan, listenPort, err := listenForPeerConnections()
	if err != nil {
		log.Fatal("Couldn't listen for peers connection: ", err)
	}

	currentSession := NewCurrentSession(torrentDir, listenPort)

	// quitChan
	quitChan := listenSigInt()

	// LPD
	lpd := &Announcer{announces: make(chan *Announce)}
	if *useLPD {
		lpd, err = NewAnnouncer(listenPort)
		if err != nil {
			log.Fatal("Couldn't listen for Local Peer Discoveries: ", err)
		}
	}

	// Control session
	controlSession, err := NewControlSession(shareID, listenPort)
	if err != nil {
		log.Fatal(err)
	}
	id, err := hex.DecodeString(shareID.PublicID())
	if err != nil {
		log.Fatal(err)
	}
	if *useLPD {
		lpd.Announce(string(id))
	}

mainLoop:
	for {
		select {
		case <-quitChan:
			err := currentSession.Quit()
			if err != nil {
				log.Println("Failed: ", err)
			} else {
				log.Println("Done")
			}
			break mainLoop
		case c := <-conChan:
			if currentSession.Matches(c.infohash) {
				currentSession.AcceptNewPeer(c)
			} else if controlSession.Matches(c.infohash) {
				controlSession.AcceptNewPeer(c)
			}
		case announce := <-lpd.announces:
			hexhash, err := hex.DecodeString(announce.infohash)
			if err != nil {
				log.Println("Err with hex-decoding:", err)
				break
			}
			if currentSession.Matches(string(hexhash)) {
				currentSession.hintNewPeer(announce.peer)
			} else if controlSession.Matches(string(hexhash)) {
				controlSession.hintNewPeer(announce.peer)
			}
		case ih := <-watcher.PingNewTorrent:
			controlSession.Broadcast(ih)

			torrentFile := filepath.Join(bitshareDir, fmt.Sprintf("%x", ih))
			currentSession, err = NewTorrentSession(torrentFile, listenPort)
			if err != nil {
				log.Fatal("Couldn't start new session: ", err)
			}
			go currentSession.DoTorrent()
			if *useLPD {
				lpd.Announce(ih)
			}
		case ih := <-controlSession.Torrents:
			log.Printf("Got new metadata from control session: %x\n", ih)
		}
	}

}

func usage() {
	log.Printf("usage: Taipei-Torrent [options] (torrent-file | torrent-url)")

	flag.PrintDefaults()
	os.Exit(2)
}

func listenSigInt() chan os.Signal {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill)
	return c
}

func NewCurrentSession(bitshareDir string, listenPort int) *TorrentSession {
	currentFile := filepath.Join(bitshareDir, "current")
	if _, err := os.Stat(currentFile); err != nil && os.IsNotExist(err) {
		log.Fatal("No current file!")
	}

	currentIH, err := ioutil.ReadFile(currentFile)
	if err != nil {
		log.Fatal(err)
	}

	currentTorrent := filepath.Join(bitshareDir, string(currentIH))

	ts, err := NewTorrentSession(currentTorrent, listenPort)
	if err != nil {
		log.Fatal(err)
	}

	return ts
}
<|endoftext|>


<commit_before>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
)

var testData uint32

func checkSymbols(t *testing.T, nmoutput []byte) {
	var checkSymbolsFound, testDataFound bool
	scanner := bufio.NewScanner(bytes.NewBuffer(nmoutput))
	for scanner.Scan() {
		f := strings.Fields(scanner.Text())
		if len(f) < 3 {
			continue
		}
		switch f[2] {
		case "cmd/nm.checkSymbols":
			checkSymbolsFound = true
			addr := "0x" + f[0]
			if addr != fmt.Sprintf("%p", checkSymbols) {
				t.Errorf("nm shows wrong address %v for checkSymbols (%p)", addr, checkSymbols)
			}
		case "cmd/nm.testData":
			testDataFound = true
			addr := "0x" + f[0]
			if addr != fmt.Sprintf("%p", &testData) {
				t.Errorf("nm shows wrong address %v for testData (%p)", addr, &testData)
			}
		}
	}
	if err := scanner.Err(); err != nil {
		t.Errorf("error while reading symbols: %v", err)
		return
	}
	if !checkSymbolsFound {
		t.Error("nm shows no checkSymbols symbol")
	}
	if !testDataFound {
		t.Error("nm shows no testData symbol")
	}
}

func TestNM(t *testing.T) {
	if runtime.GOOS == "nacl" {
		t.Skip("skipping on nacl")
	}

	tmpDir, err := ioutil.TempDir("", "TestNM")
	if err != nil {
		t.Fatal("TempDir failed: ", err)
	}
	defer os.RemoveAll(tmpDir)

	testnmpath := filepath.Join(tmpDir, "testnm.exe")
	out, err := exec.Command("go", "build", "-o", testnmpath, "cmd/nm").CombinedOutput()
	if err != nil {
		t.Fatalf("go build -o %v cmd/nm: %v\n%s", testnmpath, err, string(out))
	}

	testfiles := []string{
		"elf/testdata/gcc-386-freebsd-exec",
		"elf/testdata/gcc-amd64-linux-exec",
		"macho/testdata/gcc-386-darwin-exec",
		"macho/testdata/gcc-amd64-darwin-exec",
		"pe/testdata/gcc-amd64-mingw-exec",
		"pe/testdata/gcc-386-mingw-exec",
		"plan9obj/testdata/amd64-plan9-exec",
		"plan9obj/testdata/386-plan9-exec",
	}
	for _, f := range testfiles {
		exepath := filepath.Join(runtime.GOROOT(), "src", "pkg", "debug", f)
		cmd := exec.Command(testnmpath, exepath)
		out, err := cmd.CombinedOutput()
		if err != nil {
			t.Fatalf("go tool nm %v: %v\n%s", exepath, err, string(out))
		}
	}

	cmd := exec.Command(testnmpath, os.Args[0])
	out, err = cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("go tool nm %v: %v\n%s", os.Args[0], err, string(out))
	}
	checkSymbols(t, out)
}
<commit_msg>cmd/nm: skip test on android (no Go tool)<commit_after>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
)

var testData uint32

func checkSymbols(t *testing.T, nmoutput []byte) {
	var checkSymbolsFound, testDataFound bool
	scanner := bufio.NewScanner(bytes.NewBuffer(nmoutput))
	for scanner.Scan() {
		f := strings.Fields(scanner.Text())
		if len(f) < 3 {
			continue
		}
		switch f[2] {
		case "cmd/nm.checkSymbols":
			checkSymbolsFound = true
			addr := "0x" + f[0]
			if addr != fmt.Sprintf("%p", checkSymbols) {
				t.Errorf("nm shows wrong address %v for checkSymbols (%p)", addr, checkSymbols)
			}
		case "cmd/nm.testData":
			testDataFound = true
			addr := "0x" + f[0]
			if addr != fmt.Sprintf("%p", &testData) {
				t.Errorf("nm shows wrong address %v for testData (%p)", addr, &testData)
			}
		}
	}
	if err := scanner.Err(); err != nil {
		t.Errorf("error while reading symbols: %v", err)
		return
	}
	if !checkSymbolsFound {
		t.Error("nm shows no checkSymbols symbol")
	}
	if !testDataFound {
		t.Error("nm shows no testData symbol")
	}
}

func TestNM(t *testing.T) {
	switch runtime.GOOS {
	case "android", "nacl":
		t.Skipf("skipping on %s", runtime.GOOS)
	}

	tmpDir, err := ioutil.TempDir("", "TestNM")
	if err != nil {
		t.Fatal("TempDir failed: ", err)
	}
	defer os.RemoveAll(tmpDir)

	testnmpath := filepath.Join(tmpDir, "testnm.exe")
	out, err := exec.Command("go", "build", "-o", testnmpath, "cmd/nm").CombinedOutput()
	if err != nil {
		t.Fatalf("go build -o %v cmd/nm: %v\n%s", testnmpath, err, string(out))
	}

	testfiles := []string{
		"elf/testdata/gcc-386-freebsd-exec",
		"elf/testdata/gcc-amd64-linux-exec",
		"macho/testdata/gcc-386-darwin-exec",
		"macho/testdata/gcc-amd64-darwin-exec",
		"pe/testdata/gcc-amd64-mingw-exec",
		"pe/testdata/gcc-386-mingw-exec",
		"plan9obj/testdata/amd64-plan9-exec",
		"plan9obj/testdata/386-plan9-exec",
	}
	for _, f := range testfiles {
		exepath := filepath.Join(runtime.GOROOT(), "src", "pkg", "debug", f)
		cmd := exec.Command(testnmpath, exepath)
		out, err := cmd.CombinedOutput()
		if err != nil {
			t.Fatalf("go tool nm %v: %v\n%s", exepath, err, string(out))
		}
	}

	cmd := exec.Command(testnmpath, os.Args[0])
	out, err = cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("go tool nm %v: %v\n%s", os.Args[0], err, string(out))
	}
	checkSymbols(t, out)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"go/ast"
	"go/parser"
	"go/printer"
	"go/token"
	"os"
	"strings"
)

func main() {
	fset := token.NewFileSet()
	pkgs, err := parser.ParseDir(fset, ".", isValidFile, 0)
	if err != nil {
		fatal(err)
	}

	for _, ast := range pkgs {
		for _, f := range ast.Files {
			processFile(fset, f)
		}
	}
}

func isValidFile(info os.FileInfo) bool {
	return !strings.HasSuffix(info.Name(), "_test.go")
}

func processFile(fset *token.FileSet, f *ast.File) {
	var decls []ast.Decl
	for _, d := range f.Decls {
		genDecl, ok := d.(*ast.GenDecl)
		if !ok {
			continue
		}

		for _, spec := range genDecl.Specs {
			typeSpec, ok := spec.(*ast.TypeSpec)

			if !ok {
				continue
			}

			if _, ok := typeSpec.Type.(*ast.InterfaceType); !ok {
				continue
			}

			decls = append(decls, d)
		}
	}
	f.Decls = decls

	printer.Fprint(os.Stdout, fset, f)
}

func fatal(err error) {
	fmt.Printf("Error %v\n", err)
	os.Exit(1)
}
<commit_msg>Rename filter<commit_after>package main

import (
	"fmt"
	"go/ast"
	"go/parser"
	"go/printer"
	"go/token"
	"os"
	"strings"
)

func main() {
	fset := token.NewFileSet()
	pkgs, err := parser.ParseDir(fset, ".", isSrcFile, 0)
	if err != nil {
		fatal(err)
	}

	for _, ast := range pkgs {
		for _, f := range ast.Files {
			processFile(fset, f)
		}
	}
}

func isSrcFile(info os.FileInfo) bool {
	return !strings.HasSuffix(info.Name(), "_test.go")
}

func processFile(fset *token.FileSet, f *ast.File) {
	var decls []ast.Decl
	for _, d := range f.Decls {
		genDecl, ok := d.(*ast.GenDecl)
		if !ok {
			continue
		}

		for _, spec := range genDecl.Specs {
			typeSpec, ok := spec.(*ast.TypeSpec)

			if !ok {
				continue
			}

			if _, ok := typeSpec.Type.(*ast.InterfaceType); !ok {
				continue
			}

			decls = append(decls, d)
		}
	}
	f.Decls = decls

	printer.Fprint(os.Stdout, fset, f)
}

func fatal(err error) {
	fmt.Printf("Error %v\n", err)
	os.Exit(1)
}
<|endoftext|>