<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/http/pprof"
	"os"
	"sort"
	"strings"

	"github.com/golang/glog"
	"github.com/openshift/origin/pkg/util/proc"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/spf13/pflag"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"

	"k8s.io/kube-state-metrics/collectors"
	"k8s.io/kube-state-metrics/version"
)

const (
	metricsPath = "/metrics"
	healthzPath = "/healthz"
)

var (
	defaultCollectors = collectorSet{
		"componentstatuses":        struct{}{},
		"daemonsets":               struct{}{},
		"deployments":              struct{}{},
		"limitranges":              struct{}{},
		"nodes":                    struct{}{},
		"pods":                     struct{}{},
		"replicasets":              struct{}{},
		"replicationcontrollers":   struct{}{},
		"resourcequotas":           struct{}{},
		"services":                 struct{}{},
		"jobs":                     struct{}{},
		"cronjobs":                 struct{}{},
		"statefulsets":             struct{}{},
		"persistentvolumes":        struct{}{},
		"persistentvolumeclaims":   struct{}{},
		"namespaces":               struct{}{},
		"horizontalpodautoscalers": struct{}{},
	}
	availableCollectors = map[string]func(registry prometheus.Registerer, kubeClient clientset.Interface, namespace string){
		"componentstatuses":        collectors.RegisterComponentStatusCollector,
		"cronjobs":                 collectors.RegisterCronJobCollector,
		"daemonsets":               collectors.RegisterDaemonSetCollector,
		"deployments":              collectors.RegisterDeploymentCollector,
		"jobs":                     collectors.RegisterJobCollector,
		"limitranges":              collectors.RegisterLimitRangeCollector,
		"nodes":                    collectors.RegisterNodeCollector,
		"pods":                     collectors.RegisterPodCollector,
		"replicasets":              collectors.RegisterReplicaSetCollector,
		"replicationcontrollers":   collectors.RegisterReplicationControllerCollector,
		"resourcequotas":           collectors.RegisterResourceQuotaCollector,
		"services":                 collectors.RegisterServiceCollector,
		"statefulsets":             collectors.RegisterStatefulSetCollector,
		"persistentvolumes":        collectors.RegisterPersistentVolumeCollector,
		"persistentvolumeclaims":   collectors.RegisterPersistentVolumeClaimCollector,
		"namespaces":               collectors.RegisterNamespaceCollector,
		"horizontalpodautoscalers": collectors.RegisterHorizontalPodAutoScalerCollector,
	}
)

type collectorSet map[string]struct{}

func (c *collectorSet) String() string {
	s := *c
	ss := s.asSlice()
	sort.Strings(ss)
	return strings.Join(ss, ",")
}

func (c *collectorSet) Set(value string) error {
	s := *c
	cols := strings.Split(value, ",")
	for _, col := range cols {
		_, ok := availableCollectors[col]
		if !ok {
			glog.Fatalf("Collector \"%s\" does not exist", col)
		}
		s[col] = struct{}{}
	}
	return nil
}

func (c collectorSet) asSlice() []string {
	cols := []string{}
	for col := range c {
		cols = append(cols, col)
	}
	return cols
}

func (c collectorSet) isEmpty() bool {
	return len(c.asSlice()) == 0
}

func (c *collectorSet) Type() string {
	return "string"
}

type options struct {
	inCluster  bool
	apiserver  string
	kubeconfig string
	help       bool
	port       int
	collectors collectorSet
	namespace  string
	version    bool
}

func main() {
	options := &options{collectors: make(collectorSet)}
	flags := pflag.NewFlagSet("", pflag.ExitOnError)
	// add glog flags
	flags.AddGoFlagSet(flag.CommandLine)
	flags.Lookup("logtostderr").Value.Set("true")
	flags.Lookup("logtostderr").DefValue = "true"
	flags.Lookup("logtostderr").NoOptDefVal = "true"
	flags.BoolVar(&options.inCluster, "in-cluster", true, `If true, use the built in kubernetes cluster for creating the client`)
	flags.StringVar(&options.apiserver, "apiserver", "", `The URL of the apiserver to use as a master`)
	flags.StringVar(&options.kubeconfig, "kubeconfig", "", "Absolute path to the kubeconfig file")
	flags.BoolVarP(&options.help, "help", "h", false, "Print help text")
	flags.IntVar(&options.port, "port", 80, `Port to expose metrics on.`)
	flags.Var(&options.collectors, "collectors", fmt.Sprintf("Comma-separated list of collectors to be enabled. Defaults to %q", &defaultCollectors))
	flags.StringVar(&options.namespace, "namespace", metav1.NamespaceAll, "namespace to be enabled for collecting resources")
	flags.BoolVarP(&options.version, "version", "", false, "kube-state-metrics build version information")

	flags.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		flags.PrintDefaults()
	}

	err := flags.Parse(os.Args)
	if err != nil {
		glog.Fatalf("Error: %s", err)
	}

	if options.version {
		fmt.Printf("%#v\n", version.GetVersion())
		os.Exit(0)
	}

	if options.help {
		flags.Usage()
		os.Exit(0)
	}

	var collectors collectorSet
	if len(options.collectors) == 0 {
		glog.Info("Using default collectors")
		collectors = defaultCollectors
	} else {
		collectors = options.collectors
	}

	if options.namespace == metav1.NamespaceAll {
		glog.Info("Using all namespace")
	} else {
		glog.Infof("Using %s namespace", options.namespace)
	}

	if isNotExists(options.kubeconfig) && !(options.inCluster) {
		glog.Fatalf("kubeconfig invalid and --in-cluster is false; kubeconfig must be set to a valid file(kubeconfig default file name: $HOME/.kube/config)")
	}
	if options.apiserver != "" {
		glog.Infof("apiserver set to: %v", options.apiserver)
	}

	proc.StartReaper()

	kubeClient, err := createKubeClient(options.inCluster, options.apiserver, options.kubeconfig)
	if err != nil {
		glog.Fatalf("Failed to create client: %v", err)
	}

	registry := prometheus.NewRegistry()
	registerCollectors(registry, kubeClient, collectors, options.namespace)
	metricsServer(registry, options.port)
}

func isNotExists(file string) bool {
	if file == "" {
		file = clientcmd.NewDefaultClientConfigLoadingRules().GetDefaultFilename()
	}
	_, err := os.Stat(file)
	return os.IsNotExist(err)
}

func createKubeClient(inCluster bool, apiserver string, kubeconfig string) (kubeClient clientset.Interface, err error) {
	if inCluster {
		config, err := rest.InClusterConfig()
		if err != nil {
			return nil, err
		}
		// Allow overriding of apiserver even if using inClusterConfig
		// (necessary if kube-proxy isn't properly set up).
		if apiserver != "" {
			config.Host = apiserver
		}
		tokenPresent := false
		if len(config.BearerToken) > 0 {
			tokenPresent = true
		}
		glog.Infof("service account token present: %v", tokenPresent)
		glog.Infof("service host: %s", config.Host)
		if kubeClient, err = clientset.NewForConfig(config); err != nil {
			return nil, err
		}
	} else {
		loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
		// if you want to change the loading rules (which files in which order), you can do so here
		loadingRules.ExplicitPath = kubeconfig
		configOverrides := &clientcmd.ConfigOverrides{}
		// if you want to change override values or bind them to flags, there are methods to help you
		kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides)
		config, err := kubeConfig.ClientConfig()
		//config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
		//config, err := clientcmd.DefaultClientConfig.ClientConfig()
		if err != nil {
			return nil, err
		}
		kubeClient, err = clientset.NewForConfig(config)
		if err != nil {
			return nil, err
		}
	}

	// Informers don't seem to do a good job logging error messages when it
	// can't reach the server, making debugging hard. This makes it easier to
	// figure out if apiserver is configured incorrectly.
	glog.Infof("Testing communication with server")
	v, err := kubeClient.Discovery().ServerVersion()
	if err != nil {
		return nil, fmt.Errorf("ERROR communicating with apiserver: %v", err)
	}
	glog.Infof("Running with Kubernetes cluster version: v%s.%s. git version: %s. git tree state: %s. commit: %s. platform: %s",
		v.Major, v.Minor, v.GitVersion, v.GitTreeState, v.GitCommit, v.Platform)
	glog.Infof("Communication with server successful")

	return kubeClient, nil
}

func metricsServer(registry prometheus.Gatherer, port int) {
	// Address to listen on for web interface and telemetry
	listenAddress := fmt.Sprintf(":%d", port)

	glog.Infof("Starting metrics server: %s", listenAddress)

	mux := http.NewServeMux()

	mux.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
	mux.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
	mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
	mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
	mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))

	// Add metricsPath
	mux.Handle(metricsPath, promhttp.HandlerFor(registry, promhttp.HandlerOpts{}))
	// Add healthzPath
	mux.HandleFunc(healthzPath, func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(200)
		w.Write([]byte("ok"))
	})
	// Add index
	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
             <title>Kube Metrics Server</title>

Kube Metrics

<a href='` + metricsPath + `'>metrics

<a href='` + healthzPath + `'>healthz

             `))
	})
	log.Fatal(http.ListenAndServe(listenAddress, mux))
}

// registerCollectors creates and starts informers and initializes and
// registers metrics for collection.
func registerCollectors(registry prometheus.Registerer, kubeClient clientset.Interface, enabledCollectors collectorSet, namespace string) {
	activeCollectors := []string{}
	for c := range enabledCollectors {
		f, ok := availableCollectors[c]
		if ok {
			f(registry, kubeClient, namespace)
			activeCollectors = append(activeCollectors, c)
		}
	}

	glog.Infof("Active collectors: %s", strings.Join(activeCollectors, ","))
}
<commit_msg>Allow setting host/ip to bind HTTP server to<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"net/http/pprof"
	"os"
	"sort"
	"strconv"
	"strings"

	"github.com/golang/glog"
	"github.com/openshift/origin/pkg/util/proc"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/spf13/pflag"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"

	"k8s.io/kube-state-metrics/collectors"
	"k8s.io/kube-state-metrics/version"
)

const (
	metricsPath = "/metrics"
	healthzPath = "/healthz"
)

var (
	defaultCollectors = collectorSet{
		"componentstatuses":        struct{}{},
		"daemonsets":               struct{}{},
		"deployments":              struct{}{},
		"limitranges":              struct{}{},
		"nodes":                    struct{}{},
		"pods":                     struct{}{},
		"replicasets":              struct{}{},
		"replicationcontrollers":   struct{}{},
		"resourcequotas":           struct{}{},
		"services":                 struct{}{},
		"jobs":                     struct{}{},
		"cronjobs":                 struct{}{},
		"statefulsets":             struct{}{},
		"persistentvolumes":        struct{}{},
		"persistentvolumeclaims":   struct{}{},
		"namespaces":               struct{}{},
		"horizontalpodautoscalers": struct{}{},
	}
	availableCollectors = map[string]func(registry prometheus.Registerer, kubeClient clientset.Interface, namespace string){
		"componentstatuses":        collectors.RegisterComponentStatusCollector,
		"cronjobs":                 collectors.RegisterCronJobCollector,
		"daemonsets":               collectors.RegisterDaemonSetCollector,
		"deployments":              collectors.RegisterDeploymentCollector,
		"jobs":                     collectors.RegisterJobCollector,
		"limitranges":              collectors.RegisterLimitRangeCollector,
		"nodes":                    collectors.RegisterNodeCollector,
		"pods":                     collectors.RegisterPodCollector,
		"replicasets":              collectors.RegisterReplicaSetCollector,
		"replicationcontrollers":   collectors.RegisterReplicationControllerCollector,
		"resourcequotas":           collectors.RegisterResourceQuotaCollector,
		"services":                 collectors.RegisterServiceCollector,
		"statefulsets":             collectors.RegisterStatefulSetCollector,
		"persistentvolumes":        collectors.RegisterPersistentVolumeCollector,
		"persistentvolumeclaims":   collectors.RegisterPersistentVolumeClaimCollector,
		"namespaces":               collectors.RegisterNamespaceCollector,
		"horizontalpodautoscalers": collectors.RegisterHorizontalPodAutoScalerCollector,
	}
)

type collectorSet map[string]struct{}

func (c *collectorSet) String() string {
	s := *c
	ss := s.asSlice()
	sort.Strings(ss)
	return strings.Join(ss, ",")
}

func (c *collectorSet) Set(value string) error {
	s := *c
	cols := strings.Split(value, ",")
	for _, col := range cols {
		_, ok := availableCollectors[col]
		if !ok {
			glog.Fatalf("Collector \"%s\" does not exist", col)
		}
		s[col] = struct{}{}
	}
	return nil
}

func (c collectorSet) asSlice() []string {
	cols := []string{}
	for col := range c {
		cols = append(cols, col)
	}
	return cols
}

func (c collectorSet) isEmpty() bool {
	return len(c.asSlice()) == 0
}

func (c *collectorSet) Type() string {
	return "string"
}

type options struct {
	inCluster  bool
	apiserver  string
	kubeconfig string
	help       bool
	port       int
	host       string
	collectors collectorSet
	namespace  string
	version    bool
}

func main() {
	options := &options{collectors: make(collectorSet)}
	flags := pflag.NewFlagSet("", pflag.ExitOnError)
	// add glog flags
	flags.AddGoFlagSet(flag.CommandLine)
	flags.Lookup("logtostderr").Value.Set("true")
	flags.Lookup("logtostderr").DefValue = "true"
	flags.Lookup("logtostderr").NoOptDefVal = "true"
	flags.BoolVar(&options.inCluster, "in-cluster", true, `If true, use the built in kubernetes cluster for creating the client`)
	flags.StringVar(&options.apiserver, "apiserver", "", `The URL of the apiserver to use as a master`)
	flags.StringVar(&options.kubeconfig, "kubeconfig", "", "Absolute path to the kubeconfig file")
	flags.BoolVarP(&options.help, "help", "h", false, "Print help text")
	flags.IntVar(&options.port, "port", 80, `Port to expose metrics on.`)
	flags.StringVar(&options.host, "host", "0.0.0.0", `Host to expose metrics on.`)
	flags.Var(&options.collectors, "collectors", fmt.Sprintf("Comma-separated list of collectors to be enabled. Defaults to %q", &defaultCollectors))
	flags.StringVar(&options.namespace, "namespace", metav1.NamespaceAll, "namespace to be enabled for collecting resources")
	flags.BoolVarP(&options.version, "version", "", false, "kube-state-metrics build version information")

	flags.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		flags.PrintDefaults()
	}

	err := flags.Parse(os.Args)
	if err != nil {
		glog.Fatalf("Error: %s", err)
	}

	if options.version {
		fmt.Printf("%#v\n", version.GetVersion())
		os.Exit(0)
	}

	if options.help {
		flags.Usage()
		os.Exit(0)
	}

	var collectors collectorSet
	if len(options.collectors) == 0 {
		glog.Info("Using default collectors")
		collectors = defaultCollectors
	} else {
		collectors = options.collectors
	}

	if options.namespace == metav1.NamespaceAll {
		glog.Info("Using all namespace")
	} else {
		glog.Infof("Using %s namespace", options.namespace)
	}

	if isNotExists(options.kubeconfig) && !(options.inCluster) {
		glog.Fatalf("kubeconfig invalid and --in-cluster is false; kubeconfig must be set to a valid file(kubeconfig default file name: $HOME/.kube/config)")
	}
	if options.apiserver != "" {
		glog.Infof("apiserver set to: %v", options.apiserver)
	}

	proc.StartReaper()

	kubeClient, err := createKubeClient(options.inCluster, options.apiserver, options.kubeconfig)
	if err != nil {
		glog.Fatalf("Failed to create client: %v", err)
	}

	registry := prometheus.NewRegistry()
	registerCollectors(registry, kubeClient, collectors, options.namespace)
	metricsServer(registry, options.host, options.port)
}

func isNotExists(file string) bool {
	if file == "" {
		file = clientcmd.NewDefaultClientConfigLoadingRules().GetDefaultFilename()
	}
	_, err := os.Stat(file)
	return os.IsNotExist(err)
}

func createKubeClient(inCluster bool, apiserver string, kubeconfig string) (kubeClient clientset.Interface, err error) {
	if inCluster {
		config, err := rest.InClusterConfig()
		if err != nil {
			return nil, err
		}
		// Allow overriding of apiserver even if using inClusterConfig
		// (necessary if kube-proxy isn't properly set up).
		if apiserver != "" {
			config.Host = apiserver
		}
		tokenPresent := false
		if len(config.BearerToken) > 0 {
			tokenPresent = true
		}
		glog.Infof("service account token present: %v", tokenPresent)
		glog.Infof("service host: %s", config.Host)
		if kubeClient, err = clientset.NewForConfig(config); err != nil {
			return nil, err
		}
	} else {
		loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
		// if you want to change the loading rules (which files in which order), you can do so here
		loadingRules.ExplicitPath = kubeconfig
		configOverrides := &clientcmd.ConfigOverrides{}
		// if you want to change override values or bind them to flags, there are methods to help you
		kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides)
		config, err := kubeConfig.ClientConfig()
		//config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
		//config, err := clientcmd.DefaultClientConfig.ClientConfig()
		if err != nil {
			return nil, err
		}
		kubeClient, err = clientset.NewForConfig(config)
		if err != nil {
			return nil, err
		}
	}

	// Informers don't seem to do a good job logging error messages when it
	// can't reach the server, making debugging hard. This makes it easier to
	// figure out if apiserver is configured incorrectly.
	glog.Infof("Testing communication with server")
	v, err := kubeClient.Discovery().ServerVersion()
	if err != nil {
		return nil, fmt.Errorf("ERROR communicating with apiserver: %v", err)
	}
	glog.Infof("Running with Kubernetes cluster version: v%s.%s. git version: %s. git tree state: %s. commit: %s. platform: %s",
		v.Major, v.Minor, v.GitVersion, v.GitTreeState, v.GitCommit, v.Platform)
	glog.Infof("Communication with server successful")

	return kubeClient, nil
}

func metricsServer(registry prometheus.Gatherer, host string, port int) {
	// Address to listen on for web interface and telemetry
	listenAddress := net.JoinHostPort(host, strconv.Itoa(port))

	glog.Infof("Starting metrics server: %s", listenAddress)

	mux := http.NewServeMux()

	mux.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
	mux.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
	mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
	mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
	mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))

	// Add metricsPath
	mux.Handle(metricsPath, promhttp.HandlerFor(registry, promhttp.HandlerOpts{}))
	// Add healthzPath
	mux.HandleFunc(healthzPath, func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(200)
		w.Write([]byte("ok"))
	})
	// Add index
	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
             <title>Kube Metrics Server</title>

Kube Metrics

<a href='` + metricsPath + `'>metrics

<a href='` + healthzPath + `'>healthz

             `))
	})
	log.Fatal(http.ListenAndServe(listenAddress, mux))
}

// registerCollectors creates and starts informers and initializes and
// registers metrics for collection.
func registerCollectors(registry prometheus.Registerer, kubeClient clientset.Interface, enabledCollectors collectorSet, namespace string) {
	activeCollectors := []string{}
	for c := range enabledCollectors {
		f, ok := availableCollectors[c]
		if ok {
			f(registry, kubeClient, namespace)
			activeCollectors = append(activeCollectors, c)
		}
	}

	glog.Infof("Active collectors: %s", strings.Join(activeCollectors, ","))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
)

const listeningAddress = ":7070"

func main() {
	fmt.Printf("Listening on %s ...\n", listeningAddress)
	http.HandleFunc("/get", handleGet)
	http.ListenAndServe(listeningAddress, nil)
}

func handleGet(w http.ResponseWriter, r *http.Request) {
	log.Println(r.URL)
	encodedUrl := r.URL.Query().Get("url")

	remoteUrl, err := url.QueryUnescape(encodedUrl)
	if err != nil {
		log.Printf("Error parsing URL: %s\n", remoteUrl)
		return
	}

	data, err := fetchBody(remoteUrl)
	if err != nil {
		log.Printf("Error fetching <%s>: %s\n", remoteUrl, err)
		return
	}

	w.Write(data)
}

func fetchBody(url string) ([]byte, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	contents, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	return contents, nil
}
<commit_msg>wrap data in callback if present<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
)

const listeningAddress = ":7070"

func main() {
	fmt.Printf("Listening on %s ...\n", listeningAddress)
	http.HandleFunc("/get", handleGet)
	http.ListenAndServe(listeningAddress, nil)
}

func handleGet(w http.ResponseWriter, r *http.Request) {
	log.Println(r.URL)
	encodedUrl := r.URL.Query().Get("url")

	remoteUrl, err := url.QueryUnescape(encodedUrl)
	if err != nil {
		log.Printf("Error parsing URL: %s\n", remoteUrl)
		return
	}

	data, err := fetchBody(remoteUrl)
	if err != nil {
		log.Printf("Error fetching <%s>: %s\n", remoteUrl, err)
		return
	}

	callback := r.URL.Query().Get("callback")
	if len(callback) > 0 {
		w.Write([]byte(callback))
		w.Write([]byte("("))
		w.Write(data)
		w.Write([]byte(");"))
	} else {
		w.Write(data)
	}
}

func fetchBody(url string) ([]byte, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	contents, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	return contents, nil
}
<|endoftext|>


<commit_before>/*
log2kafka

A simple daemon that reads a file (tail -f style)
and sends every line to a Kafka topic.

Intended for nginx access logs -- so it does some special
character encoding/escaping for that format.

2015, DECK36 GmbH & Co. KG, <martin.schuette@deck36.de>
*/
package main

import (
	"flag"
	"fmt"
	"github.com/ActiveState/tail"
	"gopkg.in/Shopify/sarama.v1"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"
)

const thisVersion = "0.3"
const thisProgram = "log2kafka"

// indicate what variables are our payload data,
// to improve readability (hopefully)
type Logline string

// all command line options
type CommandLineOptions struct {
	filename *string
	server   *string
	topic    *string
	verbose  *bool
	nofollow *bool
}

var options CommandLineOptions

func init() {
	// this does not look right...
	// I am looking for a pattern how to group command line arguments in a struct
	options = CommandLineOptions{
		flag.String("file", "/var/log/syslog", "filename to watch"),
		flag.String("server", "localhost:9092", "Kafka endpoints, comma separated (default: localhost:9092)"),
		flag.String("topic", "logs", "Kafka topic name (default: logs)"),
		flag.Bool("v", false, "Verbose output"),
		flag.Bool("n", false, "Quit after file is read, do not wait for more data, do not read/write state"),
	}
	flag.Parse()
}

func readFileInode(fname string) uint64 {
	var stat syscall.Stat_t

	err := syscall.Stat(fname, &stat)
	if err != nil {
		return 0
	} else {
		return stat.Ino
	}
}

// readStateFile gets previously saved file stat, i.e. inode and offset
func readStateFile(fname string, statefile string, current_inode uint64) (offset int64) {
	var time int64
	var inode uint64
	offset = 0

	stateline, err := ioutil.ReadFile(statefile)
	if err != nil {
		return // no state
	}

	n, err := fmt.Sscanf(string(stateline), "Offset %d Time %d Inode %d\n",
		&offset, &time, &inode)
	if n != 3 || err != nil {
		log.Printf("ignoring statefile, cannot parse data in %s: %v", statefile, err)
		return
	}

	if current_inode != inode {
		log.Printf("not resuming file %s, changed inode from %d to %d\n",
			fname, inode, current_inode)
		return
	}

	log.Printf("resume logfile tail of file %s (inode %d) at offset %d\n",
		fname, inode, offset)
	return offset
}

// write inode and offset to continue later
func writeStateFile(statefile string, inode uint64, offset int64) {
	data := []byte(fmt.Sprintf("Offset %d Time %d Inode %d\n",
		offset, time.Now().UTC().Unix(), inode))
	ioutil.WriteFile(statefile, data, 0664)
}

// readLogsFromFile reads log lines from file and send them to `queue`
// notify `shutdown` when file is completely read
func readLogsFromFile(fname string, queue chan<- Logline, shutdown chan<- string, savestate <-chan bool) {
	var statefile string
	var offset int64
	var inode uint64
	var doFollowFile bool = !*options.nofollow

	if *options.verbose {
		log.Printf("readLogsFromFile: dofollow=%v", doFollowFile)
	}

	if doFollowFile {
		statefile = fname + ".state"
		inode = readFileInode(fname)
		offset = readStateFile(fname, statefile, inode)
	}

	// setup
	config := tail.Config{
		Follow:    doFollowFile,
		ReOpen:    doFollowFile,
		MustExist: true,
		Logger:    tail.DiscardingLogger,
		Location: &tail.SeekInfo{
			Offset: offset,
			Whence: 0,
		},
	}
	t, err := tail.TailFile(fname, config)
	if err != nil {
		shutdown <- fmt.Sprintf("cannot tail file %s: %v", fname, err)
	} else if *options.verbose {
		log.Printf("opened log file %s", fname)
	}

	// now just sleep and wait for input and control channel
	for {
		select {
		case line := <-t.Lines:
			if line != nil {
				queue <- Logline(line.Text)
			} else {
				shutdown <- "Logfile closed"
				return
			}
		case <-savestate:
			offset, _ := t.Tell()
			if doFollowFile {
				writeStateFile(statefile, inode, offset)
			}
			if *options.verbose {
				log.Printf("reading %s, now at offset %d", fname, offset)
			}
		}
	}
}

// general error "handler"
func failOnError(err error, msg string) {
	if err != nil {
		log.Fatalf("%s: %s", msg, err)
	}
}

// read logs from `queue` and send by AMQP
// TODO: there is too little error handling. in case of problems we simply panic and quit
func writeLogsToKafka(queue <-chan Logline, shutdown chan<- string) {
	if *options.verbose {
		log.Println("connecting to ", *options.server, "...")
	}
	serverlist := strings.Split(*options.server, ",")
	client, err := sarama.NewClient(serverlist, sarama.NewConfig())
	// TODO: set client id via Config
	// clientID := thisProgram + " " + thisVersion
	failOnError(err, "cannot open Kafka connection")
	if *options.verbose {
		log.Println("opened Kafka connection")
	}
	defer client.Close()

	producer, err := sarama.NewSyncProducerFromClient(client)
	failOnError(err, "cannot create Kafka SyncProducer")
	if *options.verbose {
		log.Println("created Kafka SyncProducer")
	}
	defer producer.Close()

	for message := range queue {
		if *options.verbose {
			log.Println("sending Kafka Message")
		}

		msg := &sarama.ProducerMessage{
			Topic: *options.topic,
			Value: sarama.ByteEncoder(Unescape([]byte(message))),
		}
		_, _, err := producer.SendMessage(msg)
		if err != nil {
			failOnError(err, "Kafka error")
		} else if *options.verbose {
			fmt.Printf(".")
		}
	}
}

// let the OS tell us to shutdown
func osSignalHandler(shutdown chan<- string) {
	var sigs = make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)

	sig := <-sigs // this is the blocking part

	go func() {
		time.Sleep(2 * time.Second)
		log.Fatalf("shutdown was ignored, bailing out now.\n")
	}()

	shutdown <- fmt.Sprintf("received signal %v", sig)
}

func main() {
	if *options.verbose {
		log.Printf("Start %s %s", thisProgram, thisVersion)
	}
	// let goroutines tell us to shutdown (on error)
	var sig_shutdown = make(chan string)
	var file_shutdown = make(chan string)
	var conn_shutdown = make(chan string)
	// the main data queue, between reader and writer goroutines
	var queue = make(chan Logline)

	// let the OS tell us to shutdown
	go osSignalHandler(sig_shutdown)

	// tell goroutine to save state before shutdown
	var savestate = make(chan bool)
	go readLogsFromFile(*options.filename, queue, file_shutdown, savestate)

	go writeLogsToKafka(queue, conn_shutdown)

	// keep track of last offset
	ticker := time.NewTicker(time.Second * 2)
	go func() {
		for _ = range ticker.C {
			savestate <- true
		}
	}()

	select {
	case message := <-sig_shutdown:
		if *options.verbose {
			log.Println("sig_shutdown:", message)
		}
	case message := <-file_shutdown:
		if *options.verbose {
			log.Println("file_shutdown:", message)
		}
	case message := <-conn_shutdown:
		if *options.verbose {
			log.Println("connection_shutdown:", message)
		}
		savestate <- true // file reader still alive
	}
	if *options.verbose {
		log.Println("The End.")
	}
}
<commit_msg>add flag for compression codec<commit_after>/*
log2kafka

A simple daemon that reads a file (tail -f style)
and sends every line to a Kafka topic.

Intended for nginx access logs -- so it does some special
character encoding/escaping for that format.

2015, DECK36 GmbH & Co. KG, <martin.schuette@deck36.de>
*/
package main

import (
	"flag"
	"fmt"
	"github.com/ActiveState/tail"
	"gopkg.in/Shopify/sarama.v1"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"
)

const thisVersion = "0.3"
const thisProgram = "log2kafka"

// indicate what variables are our payload data,
// to improve readability (hopefully)
type Logline string

// all command line options
type CommandLineOptions struct {
	filename    *string
	server      *string
	topic       *string
	verbose     *bool
	nofollow    *bool
	compression *int
}

var options CommandLineOptions

func init() {
	// this does not look right...
	// I am looking for a pattern how to group command line arguments in a struct
	options = CommandLineOptions{
		flag.String("file", "/var/log/syslog", "filename to watch"),
		flag.String("server", "localhost:9092", "Kafka endpoints, comma separated (default: localhost:9092)"),
		flag.String("topic", "logs", "Kafka topic name (default: logs)"),
		flag.Bool("v", false, "Verbose output"),
		flag.Bool("n", false, "Quit after file is read, do not wait for more data, do not read/write state"),
		flag.Int("comp", 0, "Kafka compression codec (default: 0 for no compression)"),
	}
	flag.Parse()
}

func readFileInode(fname string) uint64 {
	var stat syscall.Stat_t

	err := syscall.Stat(fname, &stat)
	if err != nil {
		return 0
	} else {
		return stat.Ino
	}
}

// readStateFile gets previously saved file stat, i.e. inode and offset
func readStateFile(fname string, statefile string, current_inode uint64) (offset int64) {
	var time int64
	var inode uint64
	offset = 0

	stateline, err := ioutil.ReadFile(statefile)
	if err != nil {
		return // no state
	}

	n, err := fmt.Sscanf(string(stateline), "Offset %d Time %d Inode %d\n",
		&offset, &time, &inode)
	if n != 3 || err != nil {
		log.Printf("ignoring statefile, cannot parse data in %s: %v", statefile, err)
		return
	}

	if current_inode != inode {
		log.Printf("not resuming file %s, changed inode from %d to %d\n",
			fname, inode, current_inode)
		return
	}

	log.Printf("resume logfile tail of file %s (inode %d) at offset %d\n",
		fname, inode, offset)
	return offset
}

// write inode and offset to continue later
func writeStateFile(statefile string, inode uint64, offset int64) {
	data := []byte(fmt.Sprintf("Offset %d Time %d Inode %d\n",
		offset, time.Now().UTC().Unix(), inode))
	ioutil.WriteFile(statefile, data, 0664)
}

// readLogsFromFile reads log lines from file and send them to `queue`
// notify `shutdown` when file is completely read
func readLogsFromFile(fname string, queue chan<- Logline, shutdown chan<- string, savestate <-chan bool) {
	var statefile string
	var offset int64
	var inode uint64
	var doFollowFile bool = !*options.nofollow

	if *options.verbose {
		log.Printf("readLogsFromFile: dofollow=%v", doFollowFile)
	}

	if doFollowFile {
		statefile = fname + ".state"
		inode = readFileInode(fname)
		offset = readStateFile(fname, statefile, inode)
	}

	// setup
	config := tail.Config{
		Follow:    doFollowFile,
		ReOpen:    doFollowFile,
		MustExist: true,
		Logger:    tail.DiscardingLogger,
		Location: &tail.SeekInfo{
			Offset: offset,
			Whence: 0,
		},
	}
	t, err := tail.TailFile(fname, config)
	if err != nil {
		shutdown <- fmt.Sprintf("cannot tail file %s: %v", fname, err)
	} else if *options.verbose {
		log.Printf("opened log file %s", fname)
	}

	// now just sleep and wait for input and control channel
	for {
		select {
		case line := <-t.Lines:
			if line != nil {
				queue <- Logline(line.Text)
			} else {
				shutdown <- "Logfile closed"
				return
			}
		case <-savestate:
			offset, _ := t.Tell()
			if doFollowFile {
				writeStateFile(statefile, inode, offset)
			}
			if *options.verbose {
				log.Printf("reading %s, now at offset %d", fname, offset)
			}
		}
	}
}

// general error "handler"
func failOnError(err error, msg string) {
	if err != nil {
		log.Fatalf("%s: %s", msg, err)
	}
}

// read logs from `queue` and send by AMQP
// TODO: there is too little error handling. in case of problems we simply panic and quit
func writeLogsToKafka(queue <-chan Logline, shutdown chan<- string) {
	if *options.verbose {
		log.Println("connecting to ", *options.server, "...")
	}
	serverlist := strings.Split(*options.server, ",")

	config := sarama.NewConfig()
	config.Producer.Compression = sarama.CompressionCodec(*options.compression)
	config.ClientID = thisProgram + " " + thisVersion
	err := config.Validate()
	if err != nil {
		failOnError(err, "Kafka config validation error")
	} else if *options.verbose {
		fmt.Printf("validated Kafka client config")
	}

	client, err := sarama.NewClient(serverlist, config)
	failOnError(err, "cannot open Kafka connection")
	if *options.verbose {
		log.Println("opened Kafka connection")
	}
	defer client.Close()

	producer, err := sarama.NewSyncProducerFromClient(client)
	failOnError(err, "cannot create Kafka SyncProducer")
	if *options.verbose {
		log.Println("created Kafka SyncProducer")
	}
	defer producer.Close()

	for message := range queue {
		if *options.verbose {
			log.Println("sending Kafka Message")
		}

		msg := &sarama.ProducerMessage{
			Topic: *options.topic,
			Value: sarama.ByteEncoder(Unescape([]byte(message))),
		}
		_, _, err := producer.SendMessage(msg)
		if err != nil {
			failOnError(err, "Kafka error")
		} else if *options.verbose {
			fmt.Printf(".")
		}
	}
}

// let the OS tell us to shutdown
func osSignalHandler(shutdown chan<- string) {
	var sigs = make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)

	sig := <-sigs // this is the blocking part

	go func() {
		time.Sleep(2 * time.Second)
		log.Fatalf("shutdown was ignored, bailing out now.\n")
	}()

	shutdown <- fmt.Sprintf("received signal %v", sig)
}

func main() {
	if *options.verbose {
		log.Printf("Start %s %s", thisProgram, thisVersion)
	}
	// let goroutines tell us to shutdown (on error)
	var sig_shutdown = make(chan string)
	var file_shutdown = make(chan string)
	var conn_shutdown = make(chan string)
	// the main data queue, between reader and writer goroutines
	var queue = make(chan Logline)

	// let the OS tell us to shutdown
	go osSignalHandler(sig_shutdown)

	// tell goroutine to save state before shutdown
	var savestate = make(chan bool)
	go readLogsFromFile(*options.filename, queue, file_shutdown, savestate)

	go writeLogsToKafka(queue, conn_shutdown)

	// keep track of last offset
	ticker := time.NewTicker(time.Second * 2)
	go func() {
		for _ = range ticker.C {
			savestate <- true
		}
	}()

	select {
	case message := <-sig_shutdown:
		if *options.verbose {
			log.Println("sig_shutdown:", message)
		}
	case message := <-file_shutdown:
		if *options.verbose {
			log.Println("file_shutdown:", message)
		}
	case message := <-conn_shutdown:
		if *options.verbose {
			log.Println("connection_shutdown:", message)
		}
		savestate <- true // file reader still alive
	}
	if *options.verbose {
		log.Println("The End.")
	}
}
<|endoftext|>


<commit_before>/*
 *  _
 * | | __ _ _ __  _   _  ___  _ __
 * | |/ _` | '_ \| | | |/ _ \| '_ \
 * | | (_| | | | | |_| | (_) | | | |
 * |_|\__,_|_| |_|\__, |\___/|_| |_|
 *                |___/
 *
 * markdown web server
 *
 * Author : Marcus Kazmierczak
 * Source : http://github.com/mkaz/lanyon
 * License: MIT
 *
 */

package main

import (
  "bytes"
  "encoding/json"
  "flag"
  "fmt"
  "io"
  "io/ioutil"
  "log"
  "net/http"
  "os"
  "os/exec"
  "path/filepath"
  "sort"
  "strings"
  "text/template"
  "time"
  "mime"
  "compress/gzip"
  //"reflect"
  "math"

  "github.com/russross/blackfriday"
)

type gzipResponseWriter struct {
  io.Writer
  http.ResponseWriter
}

func (w *gzipResponseWriter) Write(b []byte) (int, error) {
  return w.Writer.Write(b)
}

type expireDateConfig struct {
  Html       int
  Css        int
  Javascript int
  Image      int
  Index      int
}

// globals
var config struct {
  PortNum        int
  PublicDir      string
  TemplateDir    string
  FormatDate     string
  RedirectDomain []string
  Less           []string
  ExpireTime     expireDateConfig
}

var configFile string

var defaultDays = 30

var ServerVersion = "0.3.0"

var ts *template.Template

type PagesSlice []Page

func (p PagesSlice) Get(i int) Page         { return p[i] }
func (p PagesSlice) Len() int               { return len(p) }
func (p PagesSlice) Less(i, j int) bool     { return p[i].Date.Unix() > p[j].Date.Unix() }
func (p PagesSlice) Swap(i, j int)          { p[i], p[j] = p[j], p[i] }
func (p PagesSlice) Sort()                  { sort.Sort(p) }
func (p PagesSlice) Limit(n int) PagesSlice { return p[0:n] }

// Log
type LanyonLog struct { }

func (ll LanyonLog) Text(text string) { log.Printf("%s ", text) }

func (ll LanyonLog) Request(req *http.Request) {
  text := []string{"->", req.Method, req.URL.RequestURI(), req.Proto}

  ll.Text(strings.Join(text, " "))
}

var Log = LanyonLog{}

// Page struct holds all data required for a web page
// Pages array is used for category pages with list of pages
type Page struct {
  Title, Content, Category, Layout, Url string
  Date                                  time.Time
  Pages                                 PagesSlice
  Params                                map[string]string
}

func startup() {
  loadConfig()

  log.Printf("Lanyon listening on http://localhost:%d", config.PortNum)

  // add trailing slashes to config directories
  if !strings.HasSuffix(config.PublicDir, "/") {
    config.PublicDir = config.PublicDir + "/"
  }

  // verify public directory exists
  if _, err := os.Stat(config.PublicDir); err != nil {
    log.Fatalln("Public directory does not exist")
  }

  // add trailing slashes to config directories
  if !strings.HasSuffix(config.TemplateDir, "/") {
    config.TemplateDir = config.TemplateDir + "/"
  }

  // verify template directory exists
  if _, err := os.Stat(config.TemplateDir); err != nil {
    log.Fatalln("Template directory does not exist")
  }

  if err := loadTemplates(); err != nil {
    log.Fatalln("Error Parsing Templates: ", err.Error())
  }

}

func loadTemplates() (err error) {
  funcMap := template.FuncMap{
    "remainder": func (a, b int) int {
      return int( math.Remainder(float64(a + 1), float64(b)) )
    },

    "isPost": func (page Page) bool {
      return page.Layout == "post"
    },

    "isCategory": func (page Page) bool {
      return page.Layout == "category"
    },
  }

  ts = template.New("")

  ts, err = ts.Funcs(funcMap).ParseGlob(config.TemplateDir + "*.html")

  return err
}

func main() {

  flag.StringVar(&configFile, "config", "lanyon.json", "specify a config file")
  flag.Parse()
  startup()

  http.HandleFunc("/", getRequest)

  colonport := fmt.Sprintf(":%d", config.PortNum)
  log.Fatal(http.ListenAndServe(colonport, nil))
}

// handler for all requests
func getRequest(w http.ResponseWriter, r *http.Request) {

  // check domain redirect returns true on redirect
  if domainRedirect(w, r) {
    return
  }

  // add default headers
  w.Header().Add("Server", "Lanyon " + ServerVersion)
  w.Header().Add("Vary", "Accept-Encoding")

  isIndex := false
  html := ""
  fullpath := filepath.Clean(config.PublicDir + r.URL.Path)
  ext := filepath.Ext(fullpath)
  modTime := time.Now()

  if ext == "" {
    // probaly, a directory list (index)
    ext = ".html"
    isIndex = true
  }

  setCacheExpirationDays(w, ext, isIndex)

  w.Header().Set("Content-Type", mime.TypeByExtension(ext))

  Log.Request(r)

  // check if file exists on filesystem
  if fi, err := os.Stat(fullpath); err == nil {
    if fi.IsDir() {
      modTime, html, err = getDirectoryListing(fullpath)
      if err != nil {
        showFourOhFour(w, r)
        return
      }
    } else {
      http.ServeFile(w, r, fullpath)

      return
    }
  } else { // file does not exist

    // check if false extension and an actual
    // file should be generated, or 404
    switch ext {
    case ".html":
      modTime, html, err = getMarkdownFile(fullpath)
      if err != nil {
        showFourOhFour(w, r)
        return
      }

    case ".css":
      modTime, html = getLessFile(fullpath)

    default:
      showFourOhFour(w, r)
      return
    }
  }

  w.Header().Set("Last-Modified", modTime.Format(time.RFC1123))

  if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
    fmt.Fprint(w, html)
    return
  }

  w.Header().Set("Content-Encoding", "gzip")

  gz := gzip.NewWriter(w)
  defer gz.Close()

  gzw := gzipResponseWriter{Writer: gz, ResponseWriter: w}

  gzw.Writer.Write( []byte(html) )
}

func readDirListAndAppend(dir string) []string {
  var files []string

  dirlist, _ := ioutil.ReadDir(dir)
  for _, fi := range dirlist {
    f := filepath.Join(dir, fi.Name())
    ext := filepath.Ext(f)

    if ext == ".html" || ext == ".md" {
      files = append(files, f)
    } else {
      // recursively
      files = append(files, readDirListAndAppend(f)...)
    }
  }

  return files
}

// directory listing checks for existing index file
// and if exists processes like any other markdown file
// otherwise gets directory listing of html and md files
// and creates a "category" page using the category.html
// template file with array of .Pages
func getDirectoryListing(dir string) (lastModified time.Time, html string, err error) {
  modTime := time.Now();

  // check for index.md
  indexfile := dir + "/index.md"
  if _, err := os.Stat(indexfile); err == nil {
    return getMarkdownFile(indexfile)
  }

  page := Page{}
  page.Title = filepath.Base(dir)
  page.Layout = "category"
  page.Category = filepath.Base(dir)

  var files []string

  files = readDirListAndAppend(dir)

  // read markdown files to get title, date
  for fileIndex, f := range files {
    pg := readParseFile(f)
    filename := strings.Replace(f, ".md", ".html", 1)
    pg.Url = "/" + strings.Replace(filename, config.PublicDir, "", 1)
    page.Pages = append(page.Pages, pg)

    if fileIndex == 0 {
      modTime = pg.Date
    }
  }

  page.Pages.Sort()
  html = applyTemplates(page)

  return modTime, html, err
}

// reads markdown file, parse front matter and render content
// using template file defined by layout: param or by default
// uses post.html template
func getMarkdownFile(fullpath string) (lastModified time.Time, html string, err error) {
  mdfile := strings.Replace(fullpath, ".html", ".md", 1)
  page := Page{}

  statinfo, err := os.Stat(mdfile)

  if err == nil {
    page = readParseFile(mdfile)
  } else {
    return time.Now(), "", fmt.Errorf("Error reading file %s ", mdfile)
  }

  html = applyTemplates(page)
  return statinfo.ModTime(), html, err
}

func getLessFile(fullpath string) (time.Time, string) {
  lessfile := strings.Replace(fullpath, ".css", ".less", 1)

  path, err := exec.LookPath("lessc")

  if err != nil {
    return time.Now(), "/* Less is not installed */"
  }

  // requires lessc binary
  cmd := exec.Command(path, strings.Join(config.Less, " "), lessfile)
  output, err := cmd.Output()

  if err != nil {
    log.Println("Less error. Trying without parameters. [Try to update your lessc version]")

    // try without parameters
    cmd = exec.Command(path, lessfile)
    output, err = cmd.Output()

    if err != nil {
      return time.Now(), "/* Less error */"
    }
  }

  if statinfo, err := os.Stat(lessfile); err != nil {
    return time.Now(), "/* Error getting statinfo */"
  } else {
    return statinfo.ModTime(), string(output)
  }

}

func showFourOhFour(w http.ResponseWriter, r *http.Request) {
  md404 := config.PublicDir + "404.md"

  _, html, err := getMarkdownFile(md404)
  if err != nil {
    http.NotFound(w, r)
    return
  }

  w.WriteHeader(http.StatusNotFound)
  fmt.Fprint(w, html)
}

// checks config.RedirectDomain for ["domain1", "domain2"]
// If config is set, checks if request domain matches domain1
// if request does not match, issues 301 redirect to "domain2"
// Used to handle non "www.mkaz.com" requests to redirect
// to www.mkaz.com
// @return bool - true if redirected, false otherwise
//
func domainRedirect(w http.ResponseWriter, r *http.Request) bool {
  if len(config.RedirectDomain) != 2 {
    return false
  }

  if r.Host == config.RedirectDomain[0] {
    return false
  }
  redirect_url := fmt.Sprintf("http://%s/%s", config.RedirectDomain[1], r.RequestURI)
  http.Redirect(w, r, redirect_url, 301)
  return true
}

// read and parse markdown filename
func readParseFile(filename string) (page Page) {

  // setup default page struct
  page = Page{
    Title:    "",
    Content:  "",
    Category: getDirName(filename),
    Layout:   "post",
    Date:     time.Now(),
    Params:   make(map[string]string),
  }

  var data, err = ioutil.ReadFile(filename)
  if err != nil {
    return
  }

  // parse front matter from --- to ---
  var lines = strings.Split(string(data), "\n")
  var found = 0
  for i, line := range lines {
    line = strings.TrimSpace(line)

    if found == 1 {
      // parse line for param
      colonIndex := strings.Index(line, ":")
      if colonIndex > 0 {
        key := strings.TrimSpace(line[:colonIndex])
        value := strings.TrimSpace(line[colonIndex+1:])
        value = strings.Trim(value, "\"") //remove quotes
        switch key {
        case "title":
          page.Title = value
        case "layout":
          page.Layout = value
        case "date":
          page.Date, _ = time.Parse(config.FormatDate, value)
        default:
          page.Params[key] = value
        }
      }
    } else if found >= 2 {
      // params over
      lines = lines[i:]
      break
    }

    if line == "---" {
      found += 1
    }
  }

  // convert markdown content
  content := strings.Join(lines, "\n")
  output := markdownRender([]byte(content))
  page.Content = string(output)

  return page
}

func applyTemplates(page Page) string {
  buffer := new(bytes.Buffer)
  templateFile := ""
  if page.Layout == "" {
    templateFile = "post.html"
  } else {
    templateFile = page.Layout + ".html"
  }

  ts.ExecuteTemplate(buffer, templateFile, page)
  return buffer.String()
}

// configure markdown render options
// See blackfriday markdown source for details
func markdownRender(content []byte) []byte {
  htmlFlags := 0
  //htmlFlags |= blackfriday.HTML_SKIP_SCRIPT
  htmlFlags |= blackfriday.HTML_USE_XHTML
  htmlFlags |= blackfriday.HTML_USE_SMARTYPANTS
  htmlFlags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS
  htmlFlags |= blackfriday.HTML_SMARTYPANTS_LATEX_DASHES
  renderer := blackfriday.HtmlRenderer(htmlFlags, "", "")

  extensions := 0
  extensions |= blackfriday.EXTENSION_NO_INTRA_EMPHASIS
  extensions |= blackfriday.EXTENSION_TABLES
  extensions |= blackfriday.EXTENSION_FENCED_CODE
  extensions |= blackfriday.EXTENSION_AUTOLINK
  extensions |= blackfriday.EXTENSION_STRIKETHROUGH
  extensions |= blackfriday.EXTENSION_SPACE_HEADERS

  return blackfriday.Markdown(content, renderer, extensions)
}

func loadConfig() {

  // checks if specified file exists, either flag passed in
  // or default lanyon.json in current directory
  if _, err := os.Stat(configFile); os.IsNotExist(err) {

    // config file not found
    // lets check one more spot in /etc/
    configFile = "/etc/lanyon.json"
    if _, err := os.Stat(configFile); os.IsNotExist(err) {
      log.Fatalln("Config file not found, /etc/lanyon.json or specify with --config=FILENAME")
    }
  }

  file, err := ioutil.ReadFile(configFile)
  if err != nil {
    log.Fatalln("Error reading config file:", err)
  }

  if err := json.Unmarshal(file, &config); err != nil {
    log.Fatalln("Error parsing config lanyon.json: ", err)
  }
}

// returns directory name from filepath
func getDirName(fullpath string) (dir string) {
  dir = filepath.Dir(fullpath)
  dir = filepath.Base(dir)
  if strings.Trim(dir, "/") == strings.Trim(config.PublicDir, "/") {
    dir = ""
  }
  return
}

func setCacheExpirationDays(w http.ResponseWriter, ext string, isIndex bool) {
  var days int

  if isIndex == true {
    days = config.ExpireTime.Index
  } else {
    switch mime.TypeByExtension(ext) {
    case "text/html":
      days = config.ExpireTime.Html
    case "text/css":
      days = config.ExpireTime.Css
    case "application/javascript":
      days = config.ExpireTime.Javascript
    case "image/jpeg", "image/gif", "image/webm":
      days = config.ExpireTime.Image
    default:
      days = defaultDays
    }
  }

  if days == 0 {
    days = defaultDays
  }

  d := time.Duration(int64(time.Hour) * 24 * int64(days))
  expireDate := time.Now().Add(d)
  expireSecs := days * 24 * 60 * 60
  w.Header().Add("Cache-Control", fmt.Sprintf("max-age=%d, public", expireSecs))
  w.Header().Add("Expires", expireDate.Format(time.RFC1123))
}

<commit_msg>Send Last-Modified header to static files<commit_after>/*
 *  _
 * | | __ _ _ __  _   _  ___  _ __
 * | |/ _` | '_ \| | | |/ _ \| '_ \
 * | | (_| | | | | |_| | (_) | | | |
 * |_|\__,_|_| |_|\__, |\___/|_| |_|
 *                |___/
 *
 * markdown web server
 *
 * Author : Marcus Kazmierczak
 * Source : http://github.com/mkaz/lanyon
 * License: MIT
 *
 */

package main

import (
  "bytes"
  "encoding/json"
  "flag"
  "fmt"
  "io"
  "io/ioutil"
  "log"
  "net/http"
  "os"
  "os/exec"
  "path/filepath"
  "sort"
  "strings"
  "text/template"
  "time"
  "mime"
  "compress/gzip"
  //"reflect"
  "math"

  "github.com/russross/blackfriday"
)

type gzipResponseWriter struct {
  io.Writer
  http.ResponseWriter
}

func (w *gzipResponseWriter) Write(b []byte) (int, error) {
  return w.Writer.Write(b)
}

type expireDateConfig struct {
  Html       int
  Css        int
  Javascript int
  Image      int
  Index      int
}

// globals
var config struct {
  PortNum        int
  PublicDir      string
  TemplateDir    string
  FormatDate     string
  RedirectDomain []string
  Less           []string
  ExpireTime     expireDateConfig
}

var configFile string

var defaultDays = 30

var ServerVersion = "0.3.0"

var ts *template.Template

type PagesSlice []Page

func (p PagesSlice) Get(i int) Page         { return p[i] }
func (p PagesSlice) Len() int               { return len(p) }
func (p PagesSlice) Less(i, j int) bool     { return p[i].Date.Unix() > p[j].Date.Unix() }
func (p PagesSlice) Swap(i, j int)          { p[i], p[j] = p[j], p[i] }
func (p PagesSlice) Sort()                  { sort.Sort(p) }
func (p PagesSlice) Limit(n int) PagesSlice { return p[0:n] }

// Log
type LanyonLog struct { }

func (ll LanyonLog) Text(text string) { log.Printf("%s ", text) }

func (ll LanyonLog) Request(req *http.Request) {
  text := []string{"->", req.Method, req.URL.RequestURI(), req.Proto}

  ll.Text(strings.Join(text, " "))
}

var Log = LanyonLog{}

// Page struct holds all data required for a web page
// Pages array is used for category pages with list of pages
type Page struct {
  Title, Content, Category, Layout, Url string
  Date                                  time.Time
  Pages                                 PagesSlice
  Params                                map[string]string
}

func startup() {
  loadConfig()

  log.Printf("Lanyon listening on http://localhost:%d", config.PortNum)

  // add trailing slashes to config directories
  if !strings.HasSuffix(config.PublicDir, "/") {
    config.PublicDir = config.PublicDir + "/"
  }

  // verify public directory exists
  if _, err := os.Stat(config.PublicDir); err != nil {
    log.Fatalln("Public directory does not exist")
  }

  // add trailing slashes to config directories
  if !strings.HasSuffix(config.TemplateDir, "/") {
    config.TemplateDir = config.TemplateDir + "/"
  }

  // verify template directory exists
  if _, err := os.Stat(config.TemplateDir); err != nil {
    log.Fatalln("Template directory does not exist")
  }

  if err := loadTemplates(); err != nil {
    log.Fatalln("Error Parsing Templates: ", err.Error())
  }

}

func loadTemplates() (err error) {
  funcMap := template.FuncMap{
    "remainder": func (a, b int) int {
      return int( math.Remainder(float64(a + 1), float64(b)) )
    },

    "isPost": func (page Page) bool {
      return page.Layout == "post"
    },

    "isCategory": func (page Page) bool {
      return page.Layout == "category"
    },
  }

  ts = template.New("")

  ts, err = ts.Funcs(funcMap).ParseGlob(config.TemplateDir + "*.html")

  return err
}

func main() {

  flag.StringVar(&configFile, "config", "lanyon.json", "specify a config file")
  flag.Parse()
  startup()

  http.HandleFunc("/", getRequest)

  colonport := fmt.Sprintf(":%d", config.PortNum)
  log.Fatal(http.ListenAndServe(colonport, nil))
}

// handler for all requests
func getRequest(w http.ResponseWriter, r *http.Request) {

  // check domain redirect returns true on redirect
  if domainRedirect(w, r) {
    return
  }

  // add default headers
  w.Header().Add("Server", "Lanyon " + ServerVersion)
  w.Header().Add("Vary", "Accept-Encoding")

  isIndex := false
  html := ""
  fullpath := filepath.Clean(config.PublicDir + r.URL.Path)
  ext := filepath.Ext(fullpath)
  modTime := time.Now()

  if ext == "" {
    // probaly, a directory list (index)
    ext = ".html"
    isIndex = true
  }

  setCacheExpirationDays(w, ext, isIndex)

  w.Header().Set("Content-Type", mime.TypeByExtension(ext))

  Log.Request(r)

  // check if file exists on filesystem
  if fi, err := os.Stat(fullpath); err == nil {
    if fi.IsDir() {
      modTime, html, err = getDirectoryListing(fullpath)
      if err != nil {
        showFourOhFour(w, r)
        return
      }
    } else {
      statinfo, err := os.Stat(fullpath)

      if err != nil {
        modTime = statinfo.ModTime()
        w.Header().Set("Last-Modified", modTime.Format(time.RFC1123))
      }

      http.ServeFile(w, r, fullpath)

      return
    }
  } else { // file does not exist

    // check if false extension and an actual
    // file should be generated, or 404
    switch ext {
    case ".html":
      modTime, html, err = getMarkdownFile(fullpath)
      if err != nil {
        showFourOhFour(w, r)
        return
      }

    case ".css":
      modTime, html = getLessFile(fullpath)

    default:
      showFourOhFour(w, r)
      return
    }
  }

  w.Header().Set("Last-Modified", modTime.Format(time.RFC1123))

  if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
    fmt.Fprint(w, html)
    return
  }

  w.Header().Set("Content-Encoding", "gzip")

  gz := gzip.NewWriter(w)
  defer gz.Close()

  gzw := gzipResponseWriter{Writer: gz, ResponseWriter: w}

  gzw.Writer.Write( []byte(html) )
}

func readDirListAndAppend(dir string) []string {
  var files []string

  dirlist, _ := ioutil.ReadDir(dir)
  for _, fi := range dirlist {
    f := filepath.Join(dir, fi.Name())
    ext := filepath.Ext(f)

    if ext == ".html" || ext == ".md" {
      files = append(files, f)
    } else {
      // recursively
      files = append(files, readDirListAndAppend(f)...)
    }
  }

  return files
}

// directory listing checks for existing index file
// and if exists processes like any other markdown file
// otherwise gets directory listing of html and md files
// and creates a "category" page using the category.html
// template file with array of .Pages
func getDirectoryListing(dir string) (lastModified time.Time, html string, err error) {
  modTime := time.Now();

  // check for index.md
  indexfile := dir + "/index.md"
  if _, err := os.Stat(indexfile); err == nil {
    return getMarkdownFile(indexfile)
  }

  page := Page{}
  page.Title = filepath.Base(dir)
  page.Layout = "category"
  page.Category = filepath.Base(dir)

  var files []string

  files = readDirListAndAppend(dir)

  // read markdown files to get title, date
  for fileIndex, f := range files {
    pg := readParseFile(f)
    filename := strings.Replace(f, ".md", ".html", 1)
    pg.Url = "/" + strings.Replace(filename, config.PublicDir, "", 1)
    page.Pages = append(page.Pages, pg)

    if fileIndex == 0 {
      modTime = pg.Date
    }
  }

  page.Pages.Sort()
  html = applyTemplates(page)

  return modTime, html, err
}

// reads markdown file, parse front matter and render content
// using template file defined by layout: param or by default
// uses post.html template
func getMarkdownFile(fullpath string) (lastModified time.Time, html string, err error) {
  mdfile := strings.Replace(fullpath, ".html", ".md", 1)
  page := Page{}

  statinfo, err := os.Stat(mdfile)

  if err == nil {
    page = readParseFile(mdfile)
  } else {
    return time.Now(), "", fmt.Errorf("Error reading file %s ", mdfile)
  }

  html = applyTemplates(page)
  return statinfo.ModTime(), html, err
}

func getLessFile(fullpath string) (time.Time, string) {
  lessfile := strings.Replace(fullpath, ".css", ".less", 1)

  path, err := exec.LookPath("lessc")

  if err != nil {
    return time.Now(), "/* Less is not installed */"
  }

  // requires lessc binary
  cmd := exec.Command(path, strings.Join(config.Less, " "), lessfile)
  output, err := cmd.Output()

  if err != nil {
    log.Println("Less error. Trying without parameters. [Try to update your lessc version]")

    // try without parameters
    cmd = exec.Command(path, lessfile)
    output, err = cmd.Output()

    if err != nil {
      return time.Now(), "/* Less error */"
    }
  }

  if statinfo, err := os.Stat(lessfile); err != nil {
    return time.Now(), "/* Error getting statinfo */"
  } else {
    return statinfo.ModTime(), string(output)
  }

}

func showFourOhFour(w http.ResponseWriter, r *http.Request) {
  md404 := config.PublicDir + "404.md"

  _, html, err := getMarkdownFile(md404)
  if err != nil {
    http.NotFound(w, r)
    return
  }

  w.WriteHeader(http.StatusNotFound)
  fmt.Fprint(w, html)
}

// checks config.RedirectDomain for ["domain1", "domain2"]
// If config is set, checks if request domain matches domain1
// if request does not match, issues 301 redirect to "domain2"
// Used to handle non "www.mkaz.com" requests to redirect
// to www.mkaz.com
// @return bool - true if redirected, false otherwise
//
func domainRedirect(w http.ResponseWriter, r *http.Request) bool {
  if len(config.RedirectDomain) != 2 {
    return false
  }

  if r.Host == config.RedirectDomain[0] {
    return false
  }
  redirect_url := fmt.Sprintf("http://%s/%s", config.RedirectDomain[1], r.RequestURI)
  http.Redirect(w, r, redirect_url, 301)
  return true
}

// read and parse markdown filename
func readParseFile(filename string) (page Page) {

  // setup default page struct
  page = Page{
    Title:    "",
    Content:  "",
    Category: getDirName(filename),
    Layout:   "post",
    Date:     time.Now(),
    Params:   make(map[string]string),
  }

  var data, err = ioutil.ReadFile(filename)
  if err != nil {
    return
  }

  // parse front matter from --- to ---
  var lines = strings.Split(string(data), "\n")
  var found = 0
  for i, line := range lines {
    line = strings.TrimSpace(line)

    if found == 1 {
      // parse line for param
      colonIndex := strings.Index(line, ":")
      if colonIndex > 0 {
        key := strings.TrimSpace(line[:colonIndex])
        value := strings.TrimSpace(line[colonIndex+1:])
        value = strings.Trim(value, "\"") //remove quotes
        switch key {
        case "title":
          page.Title = value
        case "layout":
          page.Layout = value
        case "date":
          page.Date, _ = time.Parse(config.FormatDate, value)
        default:
          page.Params[key] = value
        }
      }
    } else if found >= 2 {
      // params over
      lines = lines[i:]
      break
    }

    if line == "---" {
      found += 1
    }
  }

  // convert markdown content
  content := strings.Join(lines, "\n")
  output := markdownRender([]byte(content))
  page.Content = string(output)

  return page
}

func applyTemplates(page Page) string {
  buffer := new(bytes.Buffer)
  templateFile := ""
  if page.Layout == "" {
    templateFile = "post.html"
  } else {
    templateFile = page.Layout + ".html"
  }

  ts.ExecuteTemplate(buffer, templateFile, page)
  return buffer.String()
}

// configure markdown render options
// See blackfriday markdown source for details
func markdownRender(content []byte) []byte {
  htmlFlags := 0
  //htmlFlags |= blackfriday.HTML_SKIP_SCRIPT
  htmlFlags |= blackfriday.HTML_USE_XHTML
  htmlFlags |= blackfriday.HTML_USE_SMARTYPANTS
  htmlFlags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS
  htmlFlags |= blackfriday.HTML_SMARTYPANTS_LATEX_DASHES
  renderer := blackfriday.HtmlRenderer(htmlFlags, "", "")

  extensions := 0
  extensions |= blackfriday.EXTENSION_NO_INTRA_EMPHASIS
  extensions |= blackfriday.EXTENSION_TABLES
  extensions |= blackfriday.EXTENSION_FENCED_CODE
  extensions |= blackfriday.EXTENSION_AUTOLINK
  extensions |= blackfriday.EXTENSION_STRIKETHROUGH
  extensions |= blackfriday.EXTENSION_SPACE_HEADERS

  return blackfriday.Markdown(content, renderer, extensions)
}

func loadConfig() {

  // checks if specified file exists, either flag passed in
  // or default lanyon.json in current directory
  if _, err := os.Stat(configFile); os.IsNotExist(err) {

    // config file not found
    // lets check one more spot in /etc/
    configFile = "/etc/lanyon.json"
    if _, err := os.Stat(configFile); os.IsNotExist(err) {
      log.Fatalln("Config file not found, /etc/lanyon.json or specify with --config=FILENAME")
    }
  }

  file, err := ioutil.ReadFile(configFile)
  if err != nil {
    log.Fatalln("Error reading config file:", err)
  }

  if err := json.Unmarshal(file, &config); err != nil {
    log.Fatalln("Error parsing config lanyon.json: ", err)
  }
}

// returns directory name from filepath
func getDirName(fullpath string) (dir string) {
  dir = filepath.Dir(fullpath)
  dir = filepath.Base(dir)
  if strings.Trim(dir, "/") == strings.Trim(config.PublicDir, "/") {
    dir = ""
  }
  return
}

func setCacheExpirationDays(w http.ResponseWriter, ext string, isIndex bool) {
  var days int

  if isIndex == true {
    days = config.ExpireTime.Index
  } else {
    switch mime.TypeByExtension(ext) {
    case "text/html":
      days = config.ExpireTime.Html
    case "text/css":
      days = config.ExpireTime.Css
    case "application/javascript":
      days = config.ExpireTime.Javascript
    case "image/jpeg", "image/gif", "image/webm":
      days = config.ExpireTime.Image
    default:
      days = defaultDays
    }
  }

  if days == 0 {
    days = defaultDays
  }

  d := time.Duration(int64(time.Hour) * 24 * int64(days))
  expireDate := time.Now().Add(d)
  expireSecs := days * 24 * 60 * 60
  w.Header().Add("Cache-Control", fmt.Sprintf("max-age=%d, public", expireSecs))
  w.Header().Add("Expires", expireDate.Format(time.RFC1123))
}

<|endoftext|>


<commit_before>package main

import (
    "flag"
    "fmt"
    "qmsk.net/clusterf/ipvs"
    "log"
    "net"
    "os"
    "qmsk.net/clusterf/server"
    "syscall"
)

var (
    etcdConfig server.EtcdConfig
    ipvsDebug   bool
)

func init() {
    flag.StringVar(&etcdConfig.Machines, "etcd-machines", "http://127.0.0.1:2379",
        "Client endpoint for etcd")
    flag.StringVar(&etcdConfig.Prefix, "etcd-prefix", "/clusterf",
        "Etcd tree prefix")

    flag.BoolVar(&ipvsDebug, "ipvs-debug", false,
        "IPVS debugging")
}

type serviceType struct {
    af      uint16
    proto   uint16
}

type ipvsKey struct {
    serviceName string
    serviceType serviceType

    backendName string
}

type ipvsState struct {
    client      *ipvs.Client

    services    map[ipvsKey]ipvs.Service
    dests       map[ipvsKey]ipvs.Dest
}

func (self *ipvsState) init() {
    self.services = make(map[ipvsKey]ipvs.Service)
    self.dests = make(map[ipvsKey]ipvs.Dest)
}

type Server struct {
    etcd        *server.Etcd

    ipvs        ipvsState
}

var serviceTypes = []serviceType {
    { syscall.AF_INET,      syscall.IPPROTO_TCP },
    { syscall.AF_INET6,     syscall.IPPROTO_TCP },
    { syscall.AF_INET,      syscall.IPPROTO_UDP },
    { syscall.AF_INET6,     syscall.IPPROTO_UDP },
}

func protoString (proto uint16) string {
    switch (proto) {
    case syscall.IPPROTO_TCP:   return "TCP"
    case syscall.IPPROTO_UDP:   return "UDP"
    default: return fmt.Sprintf("%d", proto)
    }
}

func (self *ipvsState) printIPVS () {
    if services, err := self.client.ListServices(); err != nil {
        log.Fatalf("ipvs.ListServices: %v\n", err)
    } else {
        fmt.Printf("Proto                           Addr:Port\n")
        for _, service := range services {
            fmt.Printf("%-5s %30s:%-5d %s\n",
                protoString(service.Protocol),
                service.Addr, service.Port,
                service.SchedName,
            )

            if dests, err := self.client.ListDests(service); err != nil {
                log.Fatalf("ipvs.ListDests: %v\n", err)
            } else {
                for _, dest := range dests {
                    fmt.Printf("%5s %30s:%-5d %#04x\n",
                        "",
                        dest.Addr, dest.Port,
                        dest.FwdMethod,
                    )
                }
            }
        }
    }
}

func syncServiceType (ipvsService *ipvs.Service, frontend *server.ServiceFrontend) (bool, error) {
    switch ipvsService.Af {
    case syscall.AF_INET:
        if frontend.IPv4 == "" {
            return false, nil
        } else if ip := net.ParseIP(frontend.IPv4); ip == nil {
            return false, fmt.Errorf("Invalid IPv4: %s", frontend.IPv4)
        } else if ip4 := ip.To4(); ip4 == nil {
            return false, fmt.Errorf("Invalid IPv4: %s", ip)
        } else {
            ipvsService.Addr = ip4
        }
    case syscall.AF_INET6:
        if frontend.IPv6 == "" {
            return false, nil
        } else if ip := net.ParseIP(frontend.IPv6); ip == nil {
            return false, fmt.Errorf("Invalid IPv6: %s", frontend.IPv6)
        } else if ip16 := ip.To16(); ip16 == nil {
            return false, fmt.Errorf("Invalid IPv6: %s", ip)
        } else {
            ipvsService.Addr = ip16
        }
    }

    switch ipvsService.Protocol {
    case syscall.IPPROTO_TCP:
        if frontend.TCP == 0 {
            return false, nil
        } else {
            ipvsService.Port = frontend.TCP
        }
    case syscall.IPPROTO_UDP:
        if frontend.UDP == 0 {
            return false, nil
        } else {
            ipvsService.Port = frontend.UDP
        }
    default:
        panic("invalid proto")
    }

    return true, nil
}

func syncDestType (ipvsDest *ipvs.Dest, backend *server.ServiceBackend, ipvsService ipvs.Service) (bool, error) {
    // only set the dest if the service is set
    if ipvsService.Addr == nil || ipvsService.Port == 0 {
        ipvsDest.Addr = nil
        ipvsDest.Port = 0

        return false, nil
    }

    switch ipvsService.Af {
    case syscall.AF_INET:
        if backend.IPv4 == "" {
            return false, nil
        } else if ip := net.ParseIP(backend.IPv4); ip == nil {
            return false, fmt.Errorf("Invalid IPv4: %s", backend.IPv4)
        } else if ip4 := ip.To4(); ip4 == nil {
            return false, fmt.Errorf("Invalid IPv4: %s", ip)
        } else {
            ipvsDest.Addr = ip4
        }
    case syscall.AF_INET6:
        if backend.IPv6 == "" {
            return false, nil
        } else if ip := net.ParseIP(backend.IPv6); ip == nil {
            return false, fmt.Errorf("Invalid IPv6: %s", backend.IPv6)
        } else if ip16 := ip.To16(); ip16 == nil {
            return false, fmt.Errorf("Invalid IPv6: %s", ip)
        } else {
            ipvsDest.Addr = ip16
        }
    default:
        panic("invalid af")
    }

    switch ipvsService.Protocol {
    case syscall.IPPROTO_TCP:
        if backend.TCP == 0 {
            return false, nil
        } else {
            ipvsDest.Port = backend.TCP
        }
    case syscall.IPPROTO_UDP:
        if backend.UDP == 0 {
            return false, nil
        } else {
            ipvsDest.Port = backend.UDP
        }
    default:
        panic("invalid proto")
    }

    return true, nil
}

func (self *ipvsState) syncService (service *server.Service, frontend *server.ServiceFrontend) {
    log.Printf("Sync %s: Frontend %+v\n", service.Name, frontend)

    ipvsService := ipvs.Service{
        SchedName:  "wlc",
        Timeout:    0,
        Flags:      ipvs.Flags{Flags: 0, Mask: 0xffffffff},
        Netmask:    0xffffffff,
    }

    for _, serviceType := range serviceTypes {
        ipvsService.Af = serviceType.af
        ipvsService.Protocol = serviceType.proto

        serviceActive, err := syncServiceType(&ipvsService, frontend)
        if err != nil {
            log.Printf("syncServiceType %s (%+v): %s\n", service.Name, serviceType, err)
            continue
        }

        serviceKey := ipvsKey{serviceName: service.Name, serviceType: serviceType}
        getService, serviceExists := self.services[serviceKey]

        if serviceExists && !serviceActive {
            // clear any existing instance
            if err := self.client.DelService(getService); err != nil  {
                log.Printf("ipvs.DelService %s: %s\n", service.Name, err)
            }

            delete(self.services, serviceKey)

        } else if serviceActive { // XXX: && (!serviceExists || ipvsService != getService) {
            // ensure service exists
            if err := self.client.NewService(ipvsService); err != nil  {
                log.Printf("ipvs.NewService %s: %s\n", service.Name, err)
            }

            self.services[serviceKey] = ipvsService

            // backing servers
            for backendName, backend := range service.Backends {
                // XXX: not with inner serviceType's
                self.syncServiceBackend(service, backendName, backend)
            }

            // flush old
            if serviceExists {
                if err := self.client.DelService(getService); err != nil  {
                    log.Printf("ipvs.DelService %s: %s\n", service.Name, err)
                }
            }
        }
    }
}

func (self *ipvsState) delService (service *server.Service, frontend *server.ServiceFrontend) {
    log.Printf("Delete %s: Frontend %+v\n", service.Name, frontend)

    for _, serviceType := range serviceTypes {
        serviceKey := ipvsKey{serviceName: service.Name, serviceType: serviceType}
        getService, serviceExists := self.services[serviceKey]

        if serviceExists {
            // clear any existing instance
            if err := self.client.DelService(getService); err != nil  {
                log.Printf("ipvs.DelService %s: %s\n", service.Name, err)
            }

            delete(self.services, serviceKey)
        }
    }
}

func (self *ipvsState) syncServiceBackend (service *server.Service, backendName string, backend *server.ServiceBackend) {
    log.Printf("Sync Service %s: Backend %s: %+v\n", service.Name, backendName, backend)

    ipvsDest := ipvs.Dest{
        FwdMethod:  ipvs.IP_VS_CONN_F_MASQ,
        Weight:     10,
    }

    for _, serviceType := range serviceTypes {
        serviceKey := ipvsKey{serviceName: service.Name, serviceType: serviceType}
        ipvsService := self.services[serviceKey]

        destKey := ipvsKey{serviceName: service.Name, serviceType: serviceType, backendName: backendName}
        getDest, destExists := self.dests[destKey]

        destActive, err := syncDestType(&ipvsDest, backend, ipvsService) // XXX: serviceExists
        if err != nil {
            log.Printf("syncDestType %s/%s (%+v): %s\n", service.Name, backendName, serviceType, err)
            continue
        }

        if destExists && !destActive {
            if err := self.client.DelDest(ipvsService, getDest); err != nil  {
                log.Printf("ipvs.DelDest %s %s: %s\n", service.Name, backendName, err)
            }

            delete(self.services, destKey)
        } else if destActive { // XXX: && (!destExists || ipvsDest != getDest) {
            if err := self.client.NewDest(ipvsService, ipvsDest); err != nil {
                log.Printf("ipvs.NewDest %s %s: %s\n", service.Name, backendName, err)
                continue
            }

            self.dests[destKey] = ipvsDest

            // flush old
            if destExists {
                if err := self.client.DelDest(ipvsService, getDest); err != nil  {
                    log.Printf("ipvs.DelDest %s %s: %s\n", service.Name, backendName, err)
                }
            }
        }
    }
}

func (self *ipvsState) delServiceBackend (service *server.Service, backendName string, backend *server.ServiceBackend) {
    log.Printf("Delete Service %s: Backend %s: %+v\n", service.Name, backendName, backend)

    for _, serviceType := range serviceTypes {
        serviceKey := ipvsKey{serviceName: service.Name, serviceType: serviceType}
        ipvsService := self.services[serviceKey]

        destKey := ipvsKey{serviceName: service.Name, serviceType: serviceType, backendName: backendName}
        getDest, destExists := self.dests[destKey]

        if destExists {
            if err := self.client.DelDest(ipvsService, getDest); err != nil  {
                log.Printf("ipvs.DelDest %s %s: %s\n", service.Name, backendName, err)
            }

            delete(self.services, destKey)
        }
    }
}

func main() {
    flag.Parse()

    if len(flag.Args()) > 0 {
        flag.Usage()
        os.Exit(1)
    }

    var self Server
    self.ipvs.init()

    // IPVS
    if ipvsClient, err := ipvs.Open(); err != nil {
        log.Fatalf("ipvs.Open: %v\n", err)
    } else {
        log.Printf("ipvs.Open\n")

        if ipvsDebug {
            ipvsClient.SetDebug()
        }

        if info, err := ipvsClient.GetInfo(); err != nil {
            log.Fatalf("ipvs.GetInfo: %v\n", err)
        } else {
            log.Printf("ipvs.GetInfo: version=%s, conn_tab_size=%d\n", info.Version, info.ConnTabSize)
        }

        self.ipvs.client = ipvsClient
    }

    if err := self.ipvs.client.Flush(); err != nil {
        log.Fatalf("ipvs.Flush: %v\n", err)
    }

    // etcd
    if etcdClient, err := etcdConfig.Open(); err != nil {
        log.Fatalf("etcd.Open: %s\n", err)
    } else {
        log.Printf("etcd.Open: %s\n", etcdClient)

        self.etcd = etcdClient

        // start
        if services, err := self.etcd.Scan(); err != nil {
            log.Fatalf("etcd.Sync: %s\n", err)
        } else {
            // iterate initial set of services
            for _, service := range services {
                self.ipvs.syncService(service, service.Frontend)
            }
        }

        // read channel for changes
        for event := range self.etcd.Sync() {
            log.Printf("etcd.Sync: %+v\n", event)
            switch event.Type {
            case server.NewService, server.SetService:
                self.ipvs.syncService(event.Service, event.Frontend)
            case server.DelService:
                self.ipvs.delService(event.Service, event.Frontend)
            case server.NewBackend, server.SetBackend:
                self.ipvs.syncServiceBackend(event.Service, event.BackendName, event.Backend)
            case server.DelBackend:
                self.ipvs.delServiceBackend(event.Service, event.BackendName, event.Backend)
            }
        }
    }

    // self.printIPVS()

    log.Printf("Exit\n")
}
<commit_msg>main: log<commit_after>package main

import (
    "flag"
    "fmt"
    "qmsk.net/clusterf/ipvs"
    "log"
    "net"
    "os"
    "qmsk.net/clusterf/server"
    "syscall"
)

var (
    etcdConfig server.EtcdConfig
    ipvsDebug   bool
)

func init() {
    flag.StringVar(&etcdConfig.Machines, "etcd-machines", "http://127.0.0.1:2379",
        "Client endpoint for etcd")
    flag.StringVar(&etcdConfig.Prefix, "etcd-prefix", "/clusterf",
        "Etcd tree prefix")

    flag.BoolVar(&ipvsDebug, "ipvs-debug", false,
        "IPVS debugging")
}

type serviceType struct {
    af      uint16
    proto   uint16
}

type ipvsKey struct {
    serviceName string
    serviceType serviceType

    backendName string
}

type ipvsState struct {
    client      *ipvs.Client

    services    map[ipvsKey]ipvs.Service
    dests       map[ipvsKey]ipvs.Dest
}

func (self *ipvsState) init() {
    self.services = make(map[ipvsKey]ipvs.Service)
    self.dests = make(map[ipvsKey]ipvs.Dest)
}

type Server struct {
    etcd        *server.Etcd

    ipvs        ipvsState
}

var serviceTypes = []serviceType {
    { syscall.AF_INET,      syscall.IPPROTO_TCP },
    { syscall.AF_INET6,     syscall.IPPROTO_TCP },
    { syscall.AF_INET,      syscall.IPPROTO_UDP },
    { syscall.AF_INET6,     syscall.IPPROTO_UDP },
}

func protoString (proto uint16) string {
    switch (proto) {
    case syscall.IPPROTO_TCP:   return "TCP"
    case syscall.IPPROTO_UDP:   return "UDP"
    default: return fmt.Sprintf("%d", proto)
    }
}

func (self *ipvsState) printIPVS () {
    if services, err := self.client.ListServices(); err != nil {
        log.Fatalf("ipvs.ListServices: %v\n", err)
    } else {
        fmt.Printf("Proto                           Addr:Port\n")
        for _, service := range services {
            fmt.Printf("%-5s %30s:%-5d %s\n",
                protoString(service.Protocol),
                service.Addr, service.Port,
                service.SchedName,
            )

            if dests, err := self.client.ListDests(service); err != nil {
                log.Fatalf("ipvs.ListDests: %v\n", err)
            } else {
                for _, dest := range dests {
                    fmt.Printf("%5s %30s:%-5d %#04x\n",
                        "",
                        dest.Addr, dest.Port,
                        dest.FwdMethod,
                    )
                }
            }
        }
    }
}

func syncServiceType (ipvsService *ipvs.Service, frontend *server.ServiceFrontend) (bool, error) {
    switch ipvsService.Af {
    case syscall.AF_INET:
        if frontend.IPv4 == "" {
            return false, nil
        } else if ip := net.ParseIP(frontend.IPv4); ip == nil {
            return false, fmt.Errorf("Invalid IPv4: %s", frontend.IPv4)
        } else if ip4 := ip.To4(); ip4 == nil {
            return false, fmt.Errorf("Invalid IPv4: %s", ip)
        } else {
            ipvsService.Addr = ip4
        }
    case syscall.AF_INET6:
        if frontend.IPv6 == "" {
            return false, nil
        } else if ip := net.ParseIP(frontend.IPv6); ip == nil {
            return false, fmt.Errorf("Invalid IPv6: %s", frontend.IPv6)
        } else if ip16 := ip.To16(); ip16 == nil {
            return false, fmt.Errorf("Invalid IPv6: %s", ip)
        } else {
            ipvsService.Addr = ip16
        }
    }

    switch ipvsService.Protocol {
    case syscall.IPPROTO_TCP:
        if frontend.TCP == 0 {
            return false, nil
        } else {
            ipvsService.Port = frontend.TCP
        }
    case syscall.IPPROTO_UDP:
        if frontend.UDP == 0 {
            return false, nil
        } else {
            ipvsService.Port = frontend.UDP
        }
    default:
        panic("invalid proto")
    }

    return true, nil
}

func syncDestType (ipvsDest *ipvs.Dest, backend *server.ServiceBackend, ipvsService ipvs.Service) (bool, error) {
    // only set the dest if the service is set
    if ipvsService.Addr == nil || ipvsService.Port == 0 {
        ipvsDest.Addr = nil
        ipvsDest.Port = 0

        return false, nil
    }

    switch ipvsService.Af {
    case syscall.AF_INET:
        if backend.IPv4 == "" {
            return false, nil
        } else if ip := net.ParseIP(backend.IPv4); ip == nil {
            return false, fmt.Errorf("Invalid IPv4: %s", backend.IPv4)
        } else if ip4 := ip.To4(); ip4 == nil {
            return false, fmt.Errorf("Invalid IPv4: %s", ip)
        } else {
            ipvsDest.Addr = ip4
        }
    case syscall.AF_INET6:
        if backend.IPv6 == "" {
            return false, nil
        } else if ip := net.ParseIP(backend.IPv6); ip == nil {
            return false, fmt.Errorf("Invalid IPv6: %s", backend.IPv6)
        } else if ip16 := ip.To16(); ip16 == nil {
            return false, fmt.Errorf("Invalid IPv6: %s", ip)
        } else {
            ipvsDest.Addr = ip16
        }
    default:
        panic("invalid af")
    }

    switch ipvsService.Protocol {
    case syscall.IPPROTO_TCP:
        if backend.TCP == 0 {
            return false, nil
        } else {
            ipvsDest.Port = backend.TCP
        }
    case syscall.IPPROTO_UDP:
        if backend.UDP == 0 {
            return false, nil
        } else {
            ipvsDest.Port = backend.UDP
        }
    default:
        panic("invalid proto")
    }

    return true, nil
}

func (self *ipvsState) syncService (service *server.Service, frontend *server.ServiceFrontend) {
    log.Printf("Sync %s: Frontend %+v\n", service.Name, frontend)

    ipvsService := ipvs.Service{
        SchedName:  "wlc",
        Timeout:    0,
        Flags:      ipvs.Flags{Flags: 0, Mask: 0xffffffff},
        Netmask:    0xffffffff,
    }

    for _, serviceType := range serviceTypes {
        ipvsService.Af = serviceType.af
        ipvsService.Protocol = serviceType.proto

        serviceActive, err := syncServiceType(&ipvsService, frontend)
        if err != nil {
            log.Printf("syncServiceType %s (%+v): %s\n", service.Name, serviceType, err)
            continue
        }

        serviceKey := ipvsKey{serviceName: service.Name, serviceType: serviceType}
        getService, serviceExists := self.services[serviceKey]

        if serviceExists && !serviceActive {
            // clear any existing instance
            if err := self.client.DelService(getService); err != nil  {
                log.Printf("ipvs.DelService %s: %s\n", service.Name, err)
            }

            delete(self.services, serviceKey)

        } else if serviceActive { // XXX: && (!serviceExists || ipvsService != getService) {
            // ensure service exists
            if err := self.client.NewService(ipvsService); err != nil  {
                log.Printf("ipvs.NewService %s: %s\n", service.Name, err)
            }

            self.services[serviceKey] = ipvsService

            // backing servers
            for backendName, backend := range service.Backends {
                // XXX: not with inner serviceType's
                self.syncServiceBackend(service, backendName, backend)
            }

            // flush old
            if serviceExists {
                if err := self.client.DelService(getService); err != nil  {
                    log.Printf("ipvs.DelService %s: %s\n", service.Name, err)
                }
            }
        }
    }
}

func (self *ipvsState) delService (service *server.Service, frontend *server.ServiceFrontend) {
    log.Printf("Delete %s: Frontend %+v\n", service.Name, frontend)

    for _, serviceType := range serviceTypes {
        serviceKey := ipvsKey{serviceName: service.Name, serviceType: serviceType}
        getService, serviceExists := self.services[serviceKey]

        if serviceExists {
            // clear any existing instance
            if err := self.client.DelService(getService); err != nil  {
                log.Printf("ipvs.DelService %s: %s\n", service.Name, err)
            }

            delete(self.services, serviceKey)
        }
    }
}

func (self *ipvsState) syncServiceBackend (service *server.Service, backendName string, backend *server.ServiceBackend) {
    log.Printf("Sync Service %s: Backend %s: %+v\n", service.Name, backendName, backend)

    ipvsDest := ipvs.Dest{
        FwdMethod:  ipvs.IP_VS_CONN_F_MASQ,
        Weight:     10,
    }

    for _, serviceType := range serviceTypes {
        serviceKey := ipvsKey{serviceName: service.Name, serviceType: serviceType}
        ipvsService := self.services[serviceKey]

        destKey := ipvsKey{serviceName: service.Name, serviceType: serviceType, backendName: backendName}
        getDest, destExists := self.dests[destKey]

        destActive, err := syncDestType(&ipvsDest, backend, ipvsService) // XXX: serviceExists
        if err != nil {
            log.Printf("syncDestType %s/%s (%+v): %s\n", service.Name, backendName, serviceType, err)
            continue
        }

        if destExists && !destActive {
            if err := self.client.DelDest(ipvsService, getDest); err != nil  {
                log.Printf("ipvs.DelDest %s %s: %s\n", service.Name, backendName, err)
            }

            delete(self.services, destKey)
        } else if destActive { // XXX: && (!destExists || ipvsDest != getDest) {
            if err := self.client.NewDest(ipvsService, ipvsDest); err != nil {
                log.Printf("ipvs.NewDest %s %s: %s\n", service.Name, backendName, err)
                continue
            }

            self.dests[destKey] = ipvsDest

            // flush old
            if destExists {
                if err := self.client.DelDest(ipvsService, getDest); err != nil  {
                    log.Printf("ipvs.DelDest %s %s: %s\n", service.Name, backendName, err)
                }
            }
        }
    }
}

func (self *ipvsState) delServiceBackend (service *server.Service, backendName string, backend *server.ServiceBackend) {
    log.Printf("Delete Service %s: Backend %s: %+v\n", service.Name, backendName, backend)

    for _, serviceType := range serviceTypes {
        serviceKey := ipvsKey{serviceName: service.Name, serviceType: serviceType}
        ipvsService := self.services[serviceKey]

        destKey := ipvsKey{serviceName: service.Name, serviceType: serviceType, backendName: backendName}
        getDest, destExists := self.dests[destKey]

        if destExists {
            if err := self.client.DelDest(ipvsService, getDest); err != nil  {
                log.Printf("ipvs.DelDest %s %s: %s\n", service.Name, backendName, err)
            }

            delete(self.services, destKey)
        }
    }
}

func main() {
    flag.Parse()

    if len(flag.Args()) > 0 {
        flag.Usage()
        os.Exit(1)
    }

    var self Server
    self.ipvs.init()

    // IPVS
    if ipvsClient, err := ipvs.Open(); err != nil {
        log.Fatalf("ipvs.Open: %v\n", err)
    } else {
        log.Printf("ipvs.Open\n")

        if ipvsDebug {
            ipvsClient.SetDebug()
        }

        if info, err := ipvsClient.GetInfo(); err != nil {
            log.Fatalf("ipvs.GetInfo: %v\n", err)
        } else {
            log.Printf("ipvs.GetInfo: version=%s, conn_tab_size=%d\n", info.Version, info.ConnTabSize)
        }

        self.ipvs.client = ipvsClient
    }

    if err := self.ipvs.client.Flush(); err != nil {
        log.Fatalf("ipvs.Flush: %v\n", err)
    } else {
        log.Printf("ipvs.Flush")
    }

    // etcd
    if etcdClient, err := etcdConfig.Open(); err != nil {
        log.Fatalf("etcd.Open: %s\n", err)
    } else {
        log.Printf("etcd.Open: %s\n", etcdClient)

        self.etcd = etcdClient

        // start
        if services, err := self.etcd.Scan(); err != nil {
            log.Fatalf("etcd.Scan: %s\n", err)
        } else {
            log.Printf("etcd.Scan: %d services\n", len(services))

            // iterate initial set of services
            for _, service := range services {
                self.ipvs.syncService(service, service.Frontend)
            }
        }

        // read channel for changes
        log.Printf("etcd.Sync...\n")

        for event := range self.etcd.Sync() {
            log.Printf("etcd.Sync: %+v\n", event)
            switch event.Type {
            case server.NewService, server.SetService:
                self.ipvs.syncService(event.Service, event.Frontend)
            case server.DelService:
                self.ipvs.delService(event.Service, event.Frontend)
            case server.NewBackend, server.SetBackend:
                self.ipvs.syncServiceBackend(event.Service, event.BackendName, event.Backend)
            case server.DelBackend:
                self.ipvs.delServiceBackend(event.Service, event.BackendName, event.Backend)
            }
        }
    }

    // self.printIPVS()

    log.Printf("Exit\n")
}
<|endoftext|>


<commit_before>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

// 也可以以服务的方式启动nging
// 服务支持的操作有：
// nging service install  	-- 安装服务
// nging service uninstall  -- 卸载服务
// nging service start 		-- 启动服务
// nging service stop 		-- 停止服务
// nging service restart 	-- 重启服务
package main

//go:generate go install github.com/admpub/bindata/v3/go-bindata@latest
//go:generate go-bindata -fs -o bindata_assetfs.go -prefix "vendor/github.com/nging-plugins/caddymanager/|vendor/github.com/nging-plugins/collector/|vendor/github.com/nging-plugins/dbmanager/|vendor/github.com/nging-plugins/ddnsmanager/|vendor/github.com/nging-plugins/dlmanager/|vendor/github.com/nging-plugins/frpmanager/|vendor/github.com/nging-plugins/ftpmanager/|vendor/github.com/nging-plugins/servermanager/|vendor/github.com/nging-plugins/sshmanager/" -ignore "\\.(git|svn|DS_Store|less|scss|gitkeep)$" -minify "\\.(js|css)$" -tags bindata public/assets/... template/... config/i18n/... vendor/github.com/nging-plugins/caddymanager/template/... vendor/github.com/nging-plugins/collector/template/... vendor/github.com/nging-plugins/dbmanager/template/... vendor/github.com/nging-plugins/ddnsmanager/template/... vendor/github.com/nging-plugins/dlmanager/template/... vendor/github.com/nging-plugins/frpmanager/template/... vendor/github.com/nging-plugins/ftpmanager/template/... vendor/github.com/nging-plugins/servermanager/template/... vendor/github.com/nging-plugins/sshmanager/template/...

import (
	"time"

	_ "github.com/admpub/bindata/v3"
	"github.com/admpub/log"
	"github.com/admpub/nging/v4/application/cmd"
	_ "github.com/admpub/nging/v4/application/ico"
	_ "github.com/admpub/nging/v4/upgrade"

	//"github.com/admpub/nging/v4/application/library/loader"
	"github.com/webx-top/com"
	"github.com/webx-top/echo"

	//register

	_ "github.com/admpub/nging/v4/application"
	_ "github.com/admpub/nging/v4/application/initialize/manager"
	"github.com/admpub/nging/v4/application/library/module"
	_ "github.com/admpub/nging/v4/application/library/sqlite"

	"github.com/admpub/nging/v4/application/version"

	// module
	"github.com/admpub/nging/v4/application/handler/cloud"
	"github.com/admpub/nging/v4/application/handler/task"
	"github.com/nging-plugins/caddymanager"
	"github.com/nging-plugins/collector"
	"github.com/nging-plugins/dbmanager"
	"github.com/nging-plugins/ddnsmanager"
	"github.com/nging-plugins/dlmanager"
	"github.com/nging-plugins/frpmanager"
	"github.com/nging-plugins/ftpmanager"
	"github.com/nging-plugins/servermanager"
	"github.com/nging-plugins/sshmanager"
)

var (
	BUILD_TIME string
	BUILD_OS   string
	BUILD_ARCH string
	CLOUD_GOX  string
	COMMIT     string
	LABEL      = `dev` //beta/alpha/stable
	VERSION    = `4.1.0`
	PACKAGE    = `free`

	schemaVer = version.DBSCHEMA //数据表结构版本
)

func main() {
	log.SetEmoji(com.IsMac)
	defer log.Close()
	// if err := loader.LoadPlugins(); err != nil {
	// 	panic(err)
	// }
	echo.Set(`BUILD_TIME`, BUILD_TIME)
	echo.Set(`BUILD_OS`, BUILD_OS)
	echo.Set(`BUILD_ARCH`, BUILD_ARCH)
	echo.Set(`COMMIT`, COMMIT)
	echo.Set(`LABEL`, LABEL)
	echo.Set(`VERSION`, VERSION)
	echo.Set(`PACKAGE`, PACKAGE)
	echo.Set(`SCHEMA_VER`, schemaVer)
	if com.FileExists(`config/install.sql`) {
		com.Rename(`config/install.sql`, `config/install.sql.`+time.Now().Format(`20060102150405.000`))
	}
	initModule()
	exec()
}

func exec() {
	cmd.Execute()
}

func initModule() {
	module.Register(
		&caddymanager.Module,
		&servermanager.Module,
		&ftpmanager.Module,
		&collector.Module,
		&task.Module,
		&dlmanager.Module,
		&cloud.Module,
		&dbmanager.Module,
		&frpmanager.Module,
		&sshmanager.Module,
		&ddnsmanager.Module,
	)
}
<commit_msg>update<commit_after>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

// 也可以以服务的方式启动nging
// 服务支持的操作有：
// nging service install  	-- 安装服务
// nging service uninstall  -- 卸载服务
// nging service start 		-- 启动服务
// nging service stop 		-- 停止服务
// nging service restart 	-- 重启服务
package main

//go:generate go install github.com/admpub/bindata/v3/go-bindata@latest
//go:generate go-bindata -fs -o bindata_assetfs.go -prefix "vendor/github.com/nging-plugins/caddymanager/|vendor/github.com/nging-plugins/collector/|vendor/github.com/nging-plugins/dbmanager/|vendor/github.com/nging-plugins/ddnsmanager/|vendor/github.com/nging-plugins/dlmanager/|vendor/github.com/nging-plugins/frpmanager/|vendor/github.com/nging-plugins/ftpmanager/|vendor/github.com/nging-plugins/servermanager/|vendor/github.com/nging-plugins/sshmanager/" -ignore "\\.(git|svn|DS_Store|less|scss|gitkeep)$" -minify "\\.(js|css)$" -tags bindata public/assets/... template/... config/i18n/... vendor/github.com/nging-plugins/caddymanager/template/... vendor/github.com/nging-plugins/collector/template/... vendor/github.com/nging-plugins/dbmanager/template/... vendor/github.com/nging-plugins/ddnsmanager/template/... vendor/github.com/nging-plugins/dlmanager/template/... vendor/github.com/nging-plugins/frpmanager/template/... vendor/github.com/nging-plugins/ftpmanager/template/... vendor/github.com/nging-plugins/servermanager/template/... vendor/github.com/nging-plugins/sshmanager/template/...

import (
	"time"

	_ "github.com/admpub/bindata/v3"
	"github.com/admpub/log"
	"github.com/admpub/nging/v4/application/cmd"
	_ "github.com/admpub/nging/v4/application/ico"
	_ "github.com/admpub/nging/v4/upgrade"

	//"github.com/admpub/nging/v4/application/library/loader"
	"github.com/webx-top/com"
	"github.com/webx-top/echo"

	//register

	_ "github.com/admpub/nging/v4/application"
	_ "github.com/admpub/nging/v4/application/initialize/manager"
	"github.com/admpub/nging/v4/application/library/module"
	_ "github.com/admpub/nging/v4/application/library/sqlite"

	"github.com/admpub/nging/v4/application/version"

	// module
	"github.com/admpub/nging/v4/application/handler/cloud"
	"github.com/admpub/nging/v4/application/handler/task"
	"github.com/nging-plugins/caddymanager"
	"github.com/nging-plugins/collector"
	"github.com/nging-plugins/dbmanager"
	"github.com/nging-plugins/ddnsmanager"
	"github.com/nging-plugins/dlmanager"
	"github.com/nging-plugins/frpmanager"
	"github.com/nging-plugins/ftpmanager"
	"github.com/nging-plugins/servermanager"
	"github.com/nging-plugins/sshmanager"
)

var (
	BUILD_TIME string
	BUILD_OS   string
	BUILD_ARCH string
	CLOUD_GOX  string
	COMMIT     string
	LABEL      = `dev` //beta/alpha/stable
	VERSION    = `4.1.3`
	PACKAGE    = `free`

	schemaVer = version.DBSCHEMA //数据表结构版本
)

func main() {
	log.SetEmoji(com.IsMac)
	defer log.Close()
	// if err := loader.LoadPlugins(); err != nil {
	// 	panic(err)
	// }
	echo.Set(`BUILD_TIME`, BUILD_TIME)
	echo.Set(`BUILD_OS`, BUILD_OS)
	echo.Set(`BUILD_ARCH`, BUILD_ARCH)
	echo.Set(`COMMIT`, COMMIT)
	echo.Set(`LABEL`, LABEL)
	echo.Set(`VERSION`, VERSION)
	echo.Set(`PACKAGE`, PACKAGE)
	echo.Set(`SCHEMA_VER`, schemaVer)
	if com.FileExists(`config/install.sql`) {
		com.Rename(`config/install.sql`, `config/install.sql.`+time.Now().Format(`20060102150405.000`))
	}
	initModule()
	exec()
}

func exec() {
	cmd.Execute()
}

func initModule() {
	module.Register(
		&caddymanager.Module,
		&servermanager.Module,
		&ftpmanager.Module,
		&collector.Module,
		&task.Module,
		&dlmanager.Module,
		&cloud.Module,
		&dbmanager.Module,
		&frpmanager.Module,
		&sshmanager.Module,
		&ddnsmanager.Module,
	)
}
<|endoftext|>