<commit_before>package osversion

const (
	// RS1 (version 1607, codename "Redstone 1") corresponds to Windows Server
	// 2016 (ltsc2016) and Windows 10 (Anniversary Update).
	RS1 = 14393

	// RS2 (version 1703, codename "Redstone 2") was a client-only update, and
	// corresponds to Windows 10 (Creators Update).
	RS2 = 15063

	// RS3 (version 1709, codename "Redstone 3") corresponds to Windows Server
	// 1709 (Semi-Annual Channel (SAC)), and Windows 10 (Fall Creators Update).
	RS3 = 16299

	// RS4 (version 1803, codename "Redstone 4") corresponds to Windows Server
	// 1809 (Semi-Annual Channel (SAC)), and Windows 10 (April 2018 Update).
	RS4 = 17134

	// RS5 (version 1809, codename "Redstone 5") corresponds to Windows Server
	// 2019 (ltsc2019), and Windows 10 (October 2018 Update).
	RS5 = 17763
)
<commit_msg>Fix Windows Server version for RS4 in comment<commit_after>package osversion

const (
	// RS1 (version 1607, codename "Redstone 1") corresponds to Windows Server
	// 2016 (ltsc2016) and Windows 10 (Anniversary Update).
	RS1 = 14393

	// RS2 (version 1703, codename "Redstone 2") was a client-only update, and
	// corresponds to Windows 10 (Creators Update).
	RS2 = 15063

	// RS3 (version 1709, codename "Redstone 3") corresponds to Windows Server
	// 1709 (Semi-Annual Channel (SAC)), and Windows 10 (Fall Creators Update).
	RS3 = 16299

	// RS4 (version 1803, codename "Redstone 4") corresponds to Windows Server
	// 1803 (Semi-Annual Channel (SAC)), and Windows 10 (April 2018 Update).
	RS4 = 17134

	// RS5 (version 1809, codename "Redstone 5") corresponds to Windows Server
	// 2019 (ltsc2019), and Windows 10 (October 2018 Update).
	RS5 = 17763
)
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"net/url"
	"strings"
	"testing"

	"github.com/jamesog/scan/internal/sqlite"
)

func TestLoadJobsWithNoResults(t *testing.T) {
	db := createDB("TestLoadJobsWithNoResults")
	defer db.Close()
	data, err := db.LoadJobs(sqlite.SQLFilter{})
	if err != nil {
		t.Fatalf("error from loadJobs: %v", err)
	}
	if len(data) != 0 {
		t.Errorf("expected len 0, got %v", len(data))
	}
}

func TestSaveJob(t *testing.T) {
	db := createDB("TestSaveJob")
	defer db.Close()
	id, err := db.SaveJob("192.0.2.0/24", "80,443", "tcp", "sysadmin@example.com")
	if err != nil {
		t.Fatal(err)
	}
	if id != 1 {
		t.Errorf("expected ID 1, got %d", id)
	}
}

func TestUpdateJob(t *testing.T) {
	db := createDB("TestUpdateJob")
	defer db.Close()
	id, err := db.SaveJob("192.0.2.0/24", "80,443", "tcp", "sysadmin@example.com")
	if err != nil {
		t.Fatal(err)
	}
	err = db.UpdateJob(string(id), 999)
	if err != nil {
		t.Errorf("error updating job: %v", err)
	}
}

func TestJobHandler(t *testing.T) {
	db := createDB("TestJobHandler")
	defer db.Close()
	app := App{db: db}

	r := httptest.NewRequest("GET", "/job", nil)
	w := httptest.NewRecorder()
	app.newJob(w, r)

	resp := w.Result()
	body, _ := ioutil.ReadAll(resp.Body)
	if resp.StatusCode != http.StatusOK {
		t.Errorf("expected status 200, got %v: %s", resp.StatusCode, body)
	}

	v := url.Values{}
	v.Set("cidr", "192.0.2.0/24")
	v.Set("ports", "1-1024")
	v.Set("proto", "tcp")

	r = httptest.NewRequest("POST", "/job", strings.NewReader(v.Encode()))
	w = httptest.NewRecorder()
	app.newJob(w, r)

	resp = w.Result()
	body, _ = ioutil.ReadAll(resp.Body)
	if resp.StatusCode != http.StatusOK {
		t.Errorf("expected status 200, got %v: %s", resp.StatusCode, body)
	}
}

func TestJobsHandler(t *testing.T) {
	db := createDB("TestJobsHandler")
	defer db.Close()
	app := App{db: db}

	r := httptest.NewRequest("GET", "/jobs", nil)
	w := httptest.NewRecorder()
	app.jobs(w, r)

	resp := w.Result()
	body, _ := ioutil.ReadAll(resp.Body)
	if resp.StatusCode != http.StatusOK {
		t.Errorf("expected status 200, got %v: %s", resp.StatusCode, body)
	}
	ct := resp.Header.Get("Content-Type")
	if ct != "application/json" {
		t.Errorf("expected Content-Type: %s, got %v", "application/json", ct)
	}
}

func TestJobResultsHandler(t *testing.T) {
	db := createDB("TestJobResultsHandler")
	defer db.Close()
	app := App{db: db}

	data := bytes.NewBuffer([]byte(`[{"ip":"192.0.2.1","ports":[{"port":80,"proto":"tcp","status":"open","reason":"syn-ack","ttl":57}]}]`))

	mux := app.setupRouter()
	ts := httptest.NewServer(mux)
	defer ts.Close()

	// We need to save some job data before trying to submit any
	app.db.SaveJob("192.0.2.1", "80", "tcp", "testuser@example.com")

	req, err := http.NewRequest("PUT", ts.URL+"/results/1", data)
	if err != nil {
		t.Fatal(err)
	}
	req.Header.Set("Content-Type", "application/json")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		t.Fatal(err)
	}
	if resp.StatusCode != http.StatusOK {
		t.Errorf("expected status 200, got %v", resp.StatusCode)
	}

	// Do it again - submitting the same job should be an error
	resp, err = http.DefaultClient.Do(req)
	if err != nil {
		t.Fatal(err)
	}
	if resp.StatusCode != http.StatusBadRequest {
		t.Errorf("expected status 400, got %v", resp.StatusCode)
	}
}
<commit_msg>Fix broken TestUpdateJob test<commit_after>package main

import (
	"bytes"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"net/url"
	"strconv"
	"strings"
	"testing"

	"github.com/jamesog/scan/internal/sqlite"
)

func TestLoadJobsWithNoResults(t *testing.T) {
	db := createDB("TestLoadJobsWithNoResults")
	defer db.Close()
	data, err := db.LoadJobs(sqlite.SQLFilter{})
	if err != nil {
		t.Fatalf("error from loadJobs: %v", err)
	}
	if len(data) != 0 {
		t.Errorf("expected len 0, got %v", len(data))
	}
}

func TestSaveJob(t *testing.T) {
	db := createDB("TestSaveJob")
	defer db.Close()
	id, err := db.SaveJob("192.0.2.0/24", "80,443", "tcp", "sysadmin@example.com")
	if err != nil {
		t.Fatal(err)
	}
	if id != 1 {
		t.Errorf("expected ID 1, got %d", id)
	}
}

func TestUpdateJob(t *testing.T) {
	db := createDB("TestUpdateJob")
	defer db.Close()
	id, err := db.SaveJob("192.0.2.0/24", "80,443", "tcp", "sysadmin@example.com")
	if err != nil {
		t.Fatal(err)
	}
	err = db.UpdateJob(strconv.FormatInt(id, 10), 999)
	if err != nil {
		t.Errorf("error updating job: %v", err)
	}
}

func TestJobHandler(t *testing.T) {
	db := createDB("TestJobHandler")
	defer db.Close()
	app := App{db: db}

	r := httptest.NewRequest("GET", "/job", nil)
	w := httptest.NewRecorder()
	app.newJob(w, r)

	resp := w.Result()
	body, _ := ioutil.ReadAll(resp.Body)
	if resp.StatusCode != http.StatusOK {
		t.Errorf("expected status 200, got %v: %s", resp.StatusCode, body)
	}

	v := url.Values{}
	v.Set("cidr", "192.0.2.0/24")
	v.Set("ports", "1-1024")
	v.Set("proto", "tcp")

	r = httptest.NewRequest("POST", "/job", strings.NewReader(v.Encode()))
	w = httptest.NewRecorder()
	app.newJob(w, r)

	resp = w.Result()
	body, _ = ioutil.ReadAll(resp.Body)
	if resp.StatusCode != http.StatusOK {
		t.Errorf("expected status 200, got %v: %s", resp.StatusCode, body)
	}
}

func TestJobsHandler(t *testing.T) {
	db := createDB("TestJobsHandler")
	defer db.Close()
	app := App{db: db}

	r := httptest.NewRequest("GET", "/jobs", nil)
	w := httptest.NewRecorder()
	app.jobs(w, r)

	resp := w.Result()
	body, _ := ioutil.ReadAll(resp.Body)
	if resp.StatusCode != http.StatusOK {
		t.Errorf("expected status 200, got %v: %s", resp.StatusCode, body)
	}
	ct := resp.Header.Get("Content-Type")
	if ct != "application/json" {
		t.Errorf("expected Content-Type: %s, got %v", "application/json", ct)
	}
}

func TestJobResultsHandler(t *testing.T) {
	db := createDB("TestJobResultsHandler")
	defer db.Close()
	app := App{db: db}

	data := bytes.NewBuffer([]byte(`[{"ip":"192.0.2.1","ports":[{"port":80,"proto":"tcp","status":"open","reason":"syn-ack","ttl":57}]}]`))

	mux := app.setupRouter()
	ts := httptest.NewServer(mux)
	defer ts.Close()

	// We need to save some job data before trying to submit any
	app.db.SaveJob("192.0.2.1", "80", "tcp", "testuser@example.com")

	req, err := http.NewRequest("PUT", ts.URL+"/results/1", data)
	if err != nil {
		t.Fatal(err)
	}
	req.Header.Set("Content-Type", "application/json")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		t.Fatal(err)
	}
	if resp.StatusCode != http.StatusOK {
		t.Errorf("expected status 200, got %v", resp.StatusCode)
	}

	// Do it again - submitting the same job should be an error
	resp, err = http.DefaultClient.Do(req)
	if err != nil {
		t.Fatal(err)
	}
	if resp.StatusCode != http.StatusBadRequest {
		t.Errorf("expected status 400, got %v", resp.StatusCode)
	}
}
<|endoftext|>


<commit_before>package jobs

import (
	"sort"
	"strconv"

	log "github.com/meifamily/logrus"

	"github.com/meifamily/ptt-alertor/models/top"
	user "github.com/meifamily/ptt-alertor/models/user/redis"
)

type Top struct{}

func NewTop() *Top {
	return &Top{}
}

func (t Top) Run() {
	log.Info("Top List Generated")
	keywordMap := make(map[top.BoardWord]int)
	authorMap := make(map[top.BoardWord]int)
	pushSumMap := make(map[top.BoardWord]int)
	us := new(user.User).All()
	for _, u := range us {
		for _, sub := range u.Subscribes {
			for _, keyword := range sub.Keywords {
				keywordMap[top.BoardWord{Board: sub.Board, Word: keyword}]++
			}
			for _, author := range sub.Authors {
				authorMap[top.BoardWord{Board: sub.Board, Word: author}]++
			}
			if sub.PushSum.Up != 0 {
				pushSumMap[top.BoardWord{Board: sub.Board, Word: strconv.Itoa(sub.PushSum.Up)}]++
			}
			if sub.PushSum.Down != 0 {
				pushSumMap[top.BoardWord{Board: sub.Board, Word: strconv.Itoa(sub.PushSum.Down * -1)}]++
			}
		}
	}
	topKeywords := rank(keywordMap)
	topKeywords.SaveKeywords()
	topAuthors := rank(authorMap)
	topAuthors.SaveAuthors()
	topPushSum := rank(pushSumMap)
	topPushSum.SavePushSum()
}

func rank(m map[top.BoardWord]int) (orderSlice top.WordOrders) {
	for key, count := range m {
		k := top.WordOrder{BoardWord: key, Count: count}
		orderSlice = append(orderSlice, k)
	}
	sort.Slice(orderSlice, func(i, j int) bool {
		return orderSlice[i].Count > orderSlice[j].Count
	})
	return orderSlice
}
<commit_msg>ignore case when calculate top<commit_after>package jobs

import (
	"sort"
	"strconv"

	log "github.com/meifamily/logrus"

	"strings"

	"github.com/meifamily/ptt-alertor/models/top"
	user "github.com/meifamily/ptt-alertor/models/user/redis"
)

type Top struct{}

func NewTop() *Top {
	return &Top{}
}

func (t Top) Run() {
	log.Info("Top List Generated")
	keywordMap := make(map[top.BoardWord]int)
	authorMap := make(map[top.BoardWord]int)
	pushSumMap := make(map[top.BoardWord]int)
	us := new(user.User).All()
	for _, u := range us {
		for _, sub := range u.Subscribes {
			for _, keyword := range sub.Keywords {
				keyword = strings.ToLower(keyword)
				keywordMap[top.BoardWord{Board: sub.Board, Word: keyword}]++
			}
			for _, author := range sub.Authors {
				author = strings.ToLower(author)
				authorMap[top.BoardWord{Board: sub.Board, Word: author}]++
			}
			if sub.PushSum.Up != 0 {
				pushSumMap[top.BoardWord{Board: sub.Board, Word: strconv.Itoa(sub.PushSum.Up)}]++
			}
			if sub.PushSum.Down != 0 {
				pushSumMap[top.BoardWord{Board: sub.Board, Word: strconv.Itoa(sub.PushSum.Down * -1)}]++
			}
		}
	}
	topKeywords := rank(keywordMap)
	topKeywords.SaveKeywords()
	topAuthors := rank(authorMap)
	topAuthors.SaveAuthors()
	topPushSum := rank(pushSumMap)
	topPushSum.SavePushSum()
}

func rank(m map[top.BoardWord]int) (orderSlice top.WordOrders) {
	for key, count := range m {
		k := top.WordOrder{BoardWord: key, Count: count}
		orderSlice = append(orderSlice, k)
	}
	sort.Slice(orderSlice, func(i, j int) bool {
		return orderSlice[i].Count > orderSlice[j].Count
	})
	return orderSlice
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e_node

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os/exec"
	"reflect"
	"strings"
	"time"

	"github.com/golang/glog"

	apiv1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/client-go/kubernetes/scheme"
	"k8s.io/kubernetes/pkg/features"
	"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
	kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
	kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
	stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
	kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics"
	"k8s.io/kubernetes/test/e2e/framework"
	"k8s.io/kubernetes/test/e2e/framework/metrics"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

// TODO(random-liu): Get this automatically from kubelet flag.
var kubeletAddress = flag.String("kubelet-address", "http://127.0.0.1:10255", "Host and port of the kubelet")

var startServices = flag.Bool("start-services", true, "If true, start local node services")
var stopServices = flag.Bool("stop-services", true, "If true, stop local node services after running tests")
var busyboxImage = "busybox"

func getNodeSummary() (*stats.Summary, error) {
	req, err := http.NewRequest("GET", *kubeletAddress+"/stats/summary", nil)
	if err != nil {
		return nil, fmt.Errorf("failed to build http request: %v", err)
	}
	req.Header.Add("Accept", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("failed to get /stats/summary: %v", err)
	}

	defer resp.Body.Close()
	contentsBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("failed to read /stats/summary: %+v", resp)
	}

	decoder := json.NewDecoder(strings.NewReader(string(contentsBytes)))
	summary := stats.Summary{}
	err = decoder.Decode(&summary)
	if err != nil {
		return nil, fmt.Errorf("failed to parse /stats/summary to go struct: %+v", resp)
	}
	return &summary, nil
}

// Returns the current KubeletConfiguration
func getCurrentKubeletConfig() (*kubeletconfig.KubeletConfiguration, error) {
	resp := pollConfigz(5*time.Minute, 5*time.Second)
	kubeCfg, err := decodeConfigz(resp)
	if err != nil {
		return nil, err
	}
	return kubeCfg, nil
}

// Must be called within a Context. Allows the function to modify the KubeletConfiguration during the BeforeEach of the context.
// The change is reverted in the AfterEach of the context.
// Returns true on success.
func tempSetCurrentKubeletConfig(f *framework.Framework, updateFunction func(initialConfig *kubeletconfig.KubeletConfiguration)) {
	var oldCfg *kubeletconfig.KubeletConfiguration
	BeforeEach(func() {
		configEnabled, err := isKubeletConfigEnabled(f)
		framework.ExpectNoError(err)
		Expect(configEnabled).To(BeTrue(), "The Dynamic Kubelet Configuration feature is not enabled.\n"+
			"Pass --feature-gates=DynamicKubeletConfig=true to the Kubelet to enable this feature.\n"+
			"For `make test-e2e-node`, you can set `TEST_ARGS='--feature-gates=DynamicKubeletConfig=true'`.")
		oldCfg, err = getCurrentKubeletConfig()
		framework.ExpectNoError(err)
		newCfg := oldCfg.DeepCopy()
		updateFunction(newCfg)
		framework.ExpectNoError(setKubeletConfiguration(f, newCfg))
	})
	AfterEach(func() {
		if oldCfg != nil {
			err := setKubeletConfiguration(f, oldCfg)
			framework.ExpectNoError(err)
		}
	})
}

// Returns true if kubeletConfig is enabled, false otherwise or if we cannot determine if it is.
func isKubeletConfigEnabled(f *framework.Framework) (bool, error) {
	cfgz, err := getCurrentKubeletConfig()
	if err != nil {
		return false, fmt.Errorf("could not determine whether 'DynamicKubeletConfig' feature is enabled, err: %v", err)
	}
	v, ok := cfgz.FeatureGates[string(features.DynamicKubeletConfig)]
	if !ok {
		return false, nil
	}
	return v, nil
}

// Creates or updates the configmap for KubeletConfiguration, waits for the Kubelet to restart
// with the new configuration. Returns an error if the configuration after waiting for restartGap
// doesn't match what you attempted to set, or if the dynamic configuration feature is disabled.
// You should only call this from serial tests.
func setKubeletConfiguration(f *framework.Framework, kubeCfg *kubeletconfig.KubeletConfiguration) error {
	const (
		restartGap   = 40 * time.Second
		pollInterval = 5 * time.Second
	)

	// make sure Dynamic Kubelet Configuration feature is enabled on the Kubelet we are about to reconfigure
	if configEnabled, err := isKubeletConfigEnabled(f); err != nil {
		return err
	} else if !configEnabled {
		return fmt.Errorf("The Dynamic Kubelet Configuration feature is not enabled.\n" +
			"Pass --feature-gates=DynamicKubeletConfig=true to the Kubelet to enable this feature.\n" +
			"For `make test-e2e-node`, you can set `TEST_ARGS='--feature-gates=DynamicKubeletConfig=true'`.")
	}

	// create the ConfigMap with the new configuration
	cm, err := createConfigMap(f, kubeCfg)
	if err != nil {
		return err
	}

	// create the reference and set Node.Spec.ConfigSource
	src := &apiv1.NodeConfigSource{
		ConfigMapRef: &apiv1.ObjectReference{
			Namespace: "kube-system",
			Name:      cm.Name,
			UID:       cm.UID,
		},
	}

	// set the source, retry a few times in case we are competing with other writers
	Eventually(func() error {
		if err := setNodeConfigSource(f, src); err != nil {
			return err
		}
		return nil
	}, time.Minute, time.Second).Should(BeNil())

	// poll for new config, for a maximum wait of restartGap
	Eventually(func() error {
		newKubeCfg, err := getCurrentKubeletConfig()
		if err != nil {
			return fmt.Errorf("failed trying to get current Kubelet config, will retry, error: %v", err)
		}
		if !reflect.DeepEqual(*kubeCfg, *newKubeCfg) {
			return fmt.Errorf("still waiting for new configuration to take effect, will continue to watch /configz")
		}
		glog.Infof("new configuration has taken effect")
		return nil
	}, restartGap, pollInterval).Should(BeNil())

	return nil
}

// sets the current node's configSource, this should only be called from Serial tests
func setNodeConfigSource(f *framework.Framework, source *apiv1.NodeConfigSource) error {
	// since this is a serial test, we just get the node, change the source, and then update it
	// this prevents any issues with the patch API from affecting the test results
	nodeclient := f.ClientSet.CoreV1().Nodes()

	// get the node
	node, err := nodeclient.Get(framework.TestContext.NodeName, metav1.GetOptions{})
	if err != nil {
		return err
	}

	// set new source
	node.Spec.ConfigSource = source

	// update to the new source
	_, err = nodeclient.Update(node)
	if err != nil {
		return err
	}

	return nil
}

// getConfigOK returns the first NodeCondition in `cs` with Type == apiv1.NodeConfigOK,
// or if no such condition exists, returns nil.
func getConfigOKCondition(cs []apiv1.NodeCondition) *apiv1.NodeCondition {
	for i := range cs {
		if cs[i].Type == apiv1.NodeConfigOK {
			return &cs[i]
		}
	}
	return nil
}

// Causes the test to fail, or returns a status 200 response from the /configz endpoint
func pollConfigz(timeout time.Duration, pollInterval time.Duration) *http.Response {
	endpoint := fmt.Sprintf("http://127.0.0.1:8080/api/v1/proxy/nodes/%s/configz", framework.TestContext.NodeName)
	client := &http.Client{}
	req, err := http.NewRequest("GET", endpoint, nil)
	framework.ExpectNoError(err)
	req.Header.Add("Accept", "application/json")

	var resp *http.Response
	Eventually(func() bool {
		resp, err = client.Do(req)
		if err != nil {
			glog.Errorf("Failed to get /configz, retrying. Error: %v", err)
			return false
		}
		if resp.StatusCode != 200 {
			glog.Errorf("/configz response status not 200, retrying. Response was: %+v", resp)
			return false
		}
		return true
	}, timeout, pollInterval).Should(Equal(true))
	return resp
}

// Decodes the http response from /configz and returns a kubeletconfig.KubeletConfiguration (internal type).
func decodeConfigz(resp *http.Response) (*kubeletconfig.KubeletConfiguration, error) {
	// This hack because /configz reports the following structure:
	// {"kubeletconfig": {the JSON representation of kubeletconfigv1alpha1.KubeletConfiguration}}
	type configzWrapper struct {
		ComponentConfig kubeletconfigv1alpha1.KubeletConfiguration `json:"kubeletconfig"`
	}

	configz := configzWrapper{}
	kubeCfg := kubeletconfig.KubeletConfiguration{}

	contentsBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(contentsBytes, &configz)
	if err != nil {
		return nil, err
	}

	err = scheme.Scheme.Convert(&configz.ComponentConfig, &kubeCfg, nil)
	if err != nil {
		return nil, err
	}

	return &kubeCfg, nil
}

// creates a configmap containing kubeCfg in kube-system namespace
func createConfigMap(f *framework.Framework, internalKC *kubeletconfig.KubeletConfiguration) (*apiv1.ConfigMap, error) {
	cmap := newKubeletConfigMap("testcfg", internalKC)
	cmap, err := f.ClientSet.CoreV1().ConfigMaps("kube-system").Create(cmap)
	if err != nil {
		return nil, err
	}
	return cmap, nil
}

// constructs a ConfigMap, populating one of its keys with the KubeletConfiguration. Always uses GenerateName to generate a suffix.
func newKubeletConfigMap(name string, internalKC *kubeletconfig.KubeletConfiguration) *apiv1.ConfigMap {
	scheme, _, err := kubeletscheme.NewSchemeAndCodecs()
	framework.ExpectNoError(err)

	versioned := &kubeletconfigv1alpha1.KubeletConfiguration{}
	err = scheme.Convert(internalKC, versioned, nil)
	framework.ExpectNoError(err)

	encoder, err := newKubeletConfigJSONEncoder()
	framework.ExpectNoError(err)

	data, err := runtime.Encode(encoder, versioned)
	framework.ExpectNoError(err)

	cmap := &apiv1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{GenerateName: name + "-"},
		Data: map[string]string{
			"kubelet": string(data),
		},
	}
	return cmap
}

func logPodEvents(f *framework.Framework) {
	framework.Logf("Summary of pod events during the test:")
	err := framework.ListNamespaceEvents(f.ClientSet, f.Namespace.Name)
	framework.ExpectNoError(err)
}

func logNodeEvents(f *framework.Framework) {
	framework.Logf("Summary of node events during the test:")
	err := framework.ListNamespaceEvents(f.ClientSet, "")
	framework.ExpectNoError(err)
}

func getLocalNode(f *framework.Framework) *apiv1.Node {
	nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
	Expect(len(nodeList.Items)).To(Equal(1), "Unexpected number of node objects for node e2e. Expects only one node.")
	return &nodeList.Items[0]
}

// logs prometheus metrics from the local kubelet.
func logKubeletMetrics(metricKeys ...string) {
	metricSet := sets.NewString()
	for _, key := range metricKeys {
		metricSet.Insert(kubeletmetrics.KubeletSubsystem + "_" + key)
	}
	metric, err := metrics.GrabKubeletMetricsWithoutProxy(framework.TestContext.NodeName + ":10255")
	if err != nil {
		framework.Logf("Error getting kubelet metrics: %v", err)
	} else {
		framework.Logf("Kubelet Metrics: %+v", framework.GetKubeletMetrics(metric, metricSet))
	}
}

func newKubeletConfigJSONEncoder() (runtime.Encoder, error) {
	_, kubeletCodecs, err := kubeletscheme.NewSchemeAndCodecs()
	if err != nil {
		return nil, err
	}

	mediaType := "application/json"
	info, ok := runtime.SerializerInfoForMediaType(kubeletCodecs.SupportedMediaTypes(), mediaType)
	if !ok {
		return nil, fmt.Errorf("unsupported media type %q", mediaType)
	}
	return kubeletCodecs.EncoderForVersion(info.Serializer, kubeletconfigv1alpha1.SchemeGroupVersion), nil
}

// runCommand runs the cmd and returns the combined stdout and stderr, or an
// error if the command failed.
func runCommand(cmd ...string) (string, error) {
	output, err := exec.Command(cmd[0], cmd[1:]...).CombinedOutput()
	if err != nil {
		return "", fmt.Errorf("failed to run %q: %s (%s)", strings.Join(cmd, " "), err, output)
	}
	return string(output), nil
}
<commit_msg>Double check before setKubeletConfiguration<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e_node

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os/exec"
	"reflect"
	"strings"
	"time"

	"github.com/golang/glog"

	apiv1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/client-go/kubernetes/scheme"
	"k8s.io/kubernetes/pkg/features"
	"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
	kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
	kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
	stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
	kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics"
	"k8s.io/kubernetes/test/e2e/framework"
	"k8s.io/kubernetes/test/e2e/framework/metrics"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

// TODO(random-liu): Get this automatically from kubelet flag.
var kubeletAddress = flag.String("kubelet-address", "http://127.0.0.1:10255", "Host and port of the kubelet")

var startServices = flag.Bool("start-services", true, "If true, start local node services")
var stopServices = flag.Bool("stop-services", true, "If true, stop local node services after running tests")
var busyboxImage = "busybox"

func getNodeSummary() (*stats.Summary, error) {
	req, err := http.NewRequest("GET", *kubeletAddress+"/stats/summary", nil)
	if err != nil {
		return nil, fmt.Errorf("failed to build http request: %v", err)
	}
	req.Header.Add("Accept", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("failed to get /stats/summary: %v", err)
	}

	defer resp.Body.Close()
	contentsBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("failed to read /stats/summary: %+v", resp)
	}

	decoder := json.NewDecoder(strings.NewReader(string(contentsBytes)))
	summary := stats.Summary{}
	err = decoder.Decode(&summary)
	if err != nil {
		return nil, fmt.Errorf("failed to parse /stats/summary to go struct: %+v", resp)
	}
	return &summary, nil
}

// Returns the current KubeletConfiguration
func getCurrentKubeletConfig() (*kubeletconfig.KubeletConfiguration, error) {
	resp := pollConfigz(5*time.Minute, 5*time.Second)
	kubeCfg, err := decodeConfigz(resp)
	if err != nil {
		return nil, err
	}
	return kubeCfg, nil
}

// Must be called within a Context. Allows the function to modify the KubeletConfiguration during the BeforeEach of the context.
// The change is reverted in the AfterEach of the context.
// Returns true on success.
func tempSetCurrentKubeletConfig(f *framework.Framework, updateFunction func(initialConfig *kubeletconfig.KubeletConfiguration)) {
	var oldCfg *kubeletconfig.KubeletConfiguration
	BeforeEach(func() {
		configEnabled, err := isKubeletConfigEnabled(f)
		framework.ExpectNoError(err)
		Expect(configEnabled).To(BeTrue(), "The Dynamic Kubelet Configuration feature is not enabled.\n"+
			"Pass --feature-gates=DynamicKubeletConfig=true to the Kubelet to enable this feature.\n"+
			"For `make test-e2e-node`, you can set `TEST_ARGS='--feature-gates=DynamicKubeletConfig=true'`.")
		oldCfg, err = getCurrentKubeletConfig()
		framework.ExpectNoError(err)
		newCfg := oldCfg.DeepCopy()
		updateFunction(newCfg)
		if reflect.DeepEqual(*newCfg, *oldCfg) {
			return
		}

		framework.ExpectNoError(setKubeletConfiguration(f, newCfg))
	})
	AfterEach(func() {
		if oldCfg != nil {
			err := setKubeletConfiguration(f, oldCfg)
			framework.ExpectNoError(err)
		}
	})
}

// Returns true if kubeletConfig is enabled, false otherwise or if we cannot determine if it is.
func isKubeletConfigEnabled(f *framework.Framework) (bool, error) {
	cfgz, err := getCurrentKubeletConfig()
	if err != nil {
		return false, fmt.Errorf("could not determine whether 'DynamicKubeletConfig' feature is enabled, err: %v", err)
	}
	v, ok := cfgz.FeatureGates[string(features.DynamicKubeletConfig)]
	if !ok {
		return false, nil
	}
	return v, nil
}

// Creates or updates the configmap for KubeletConfiguration, waits for the Kubelet to restart
// with the new configuration. Returns an error if the configuration after waiting for restartGap
// doesn't match what you attempted to set, or if the dynamic configuration feature is disabled.
// You should only call this from serial tests.
func setKubeletConfiguration(f *framework.Framework, kubeCfg *kubeletconfig.KubeletConfiguration) error {
	const (
		restartGap   = 40 * time.Second
		pollInterval = 5 * time.Second
	)

	// make sure Dynamic Kubelet Configuration feature is enabled on the Kubelet we are about to reconfigure
	if configEnabled, err := isKubeletConfigEnabled(f); err != nil {
		return err
	} else if !configEnabled {
		return fmt.Errorf("The Dynamic Kubelet Configuration feature is not enabled.\n" +
			"Pass --feature-gates=DynamicKubeletConfig=true to the Kubelet to enable this feature.\n" +
			"For `make test-e2e-node`, you can set `TEST_ARGS='--feature-gates=DynamicKubeletConfig=true'`.")
	}

	// create the ConfigMap with the new configuration
	cm, err := createConfigMap(f, kubeCfg)
	if err != nil {
		return err
	}

	// create the reference and set Node.Spec.ConfigSource
	src := &apiv1.NodeConfigSource{
		ConfigMapRef: &apiv1.ObjectReference{
			Namespace: "kube-system",
			Name:      cm.Name,
			UID:       cm.UID,
		},
	}

	// set the source, retry a few times in case we are competing with other writers
	Eventually(func() error {
		if err := setNodeConfigSource(f, src); err != nil {
			return err
		}
		return nil
	}, time.Minute, time.Second).Should(BeNil())

	// poll for new config, for a maximum wait of restartGap
	Eventually(func() error {
		newKubeCfg, err := getCurrentKubeletConfig()
		if err != nil {
			return fmt.Errorf("failed trying to get current Kubelet config, will retry, error: %v", err)
		}
		if !reflect.DeepEqual(*kubeCfg, *newKubeCfg) {
			return fmt.Errorf("still waiting for new configuration to take effect, will continue to watch /configz")
		}
		glog.Infof("new configuration has taken effect")
		return nil
	}, restartGap, pollInterval).Should(BeNil())

	return nil
}

// sets the current node's configSource, this should only be called from Serial tests
func setNodeConfigSource(f *framework.Framework, source *apiv1.NodeConfigSource) error {
	// since this is a serial test, we just get the node, change the source, and then update it
	// this prevents any issues with the patch API from affecting the test results
	nodeclient := f.ClientSet.CoreV1().Nodes()

	// get the node
	node, err := nodeclient.Get(framework.TestContext.NodeName, metav1.GetOptions{})
	if err != nil {
		return err
	}

	// set new source
	node.Spec.ConfigSource = source

	// update to the new source
	_, err = nodeclient.Update(node)
	if err != nil {
		return err
	}

	return nil
}

// getConfigOK returns the first NodeCondition in `cs` with Type == apiv1.NodeConfigOK,
// or if no such condition exists, returns nil.
func getConfigOKCondition(cs []apiv1.NodeCondition) *apiv1.NodeCondition {
	for i := range cs {
		if cs[i].Type == apiv1.NodeConfigOK {
			return &cs[i]
		}
	}
	return nil
}

// Causes the test to fail, or returns a status 200 response from the /configz endpoint
func pollConfigz(timeout time.Duration, pollInterval time.Duration) *http.Response {
	endpoint := fmt.Sprintf("http://127.0.0.1:8080/api/v1/proxy/nodes/%s/configz", framework.TestContext.NodeName)
	client := &http.Client{}
	req, err := http.NewRequest("GET", endpoint, nil)
	framework.ExpectNoError(err)
	req.Header.Add("Accept", "application/json")

	var resp *http.Response
	Eventually(func() bool {
		resp, err = client.Do(req)
		if err != nil {
			glog.Errorf("Failed to get /configz, retrying. Error: %v", err)
			return false
		}
		if resp.StatusCode != 200 {
			glog.Errorf("/configz response status not 200, retrying. Response was: %+v", resp)
			return false
		}
		return true
	}, timeout, pollInterval).Should(Equal(true))
	return resp
}

// Decodes the http response from /configz and returns a kubeletconfig.KubeletConfiguration (internal type).
func decodeConfigz(resp *http.Response) (*kubeletconfig.KubeletConfiguration, error) {
	// This hack because /configz reports the following structure:
	// {"kubeletconfig": {the JSON representation of kubeletconfigv1alpha1.KubeletConfiguration}}
	type configzWrapper struct {
		ComponentConfig kubeletconfigv1alpha1.KubeletConfiguration `json:"kubeletconfig"`
	}

	configz := configzWrapper{}
	kubeCfg := kubeletconfig.KubeletConfiguration{}

	contentsBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(contentsBytes, &configz)
	if err != nil {
		return nil, err
	}

	err = scheme.Scheme.Convert(&configz.ComponentConfig, &kubeCfg, nil)
	if err != nil {
		return nil, err
	}

	return &kubeCfg, nil
}

// creates a configmap containing kubeCfg in kube-system namespace
func createConfigMap(f *framework.Framework, internalKC *kubeletconfig.KubeletConfiguration) (*apiv1.ConfigMap, error) {
	cmap := newKubeletConfigMap("testcfg", internalKC)
	cmap, err := f.ClientSet.CoreV1().ConfigMaps("kube-system").Create(cmap)
	if err != nil {
		return nil, err
	}
	return cmap, nil
}

// constructs a ConfigMap, populating one of its keys with the KubeletConfiguration. Always uses GenerateName to generate a suffix.
func newKubeletConfigMap(name string, internalKC *kubeletconfig.KubeletConfiguration) *apiv1.ConfigMap {
	scheme, _, err := kubeletscheme.NewSchemeAndCodecs()
	framework.ExpectNoError(err)

	versioned := &kubeletconfigv1alpha1.KubeletConfiguration{}
	err = scheme.Convert(internalKC, versioned, nil)
	framework.ExpectNoError(err)

	encoder, err := newKubeletConfigJSONEncoder()
	framework.ExpectNoError(err)

	data, err := runtime.Encode(encoder, versioned)
	framework.ExpectNoError(err)

	cmap := &apiv1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{GenerateName: name + "-"},
		Data: map[string]string{
			"kubelet": string(data),
		},
	}
	return cmap
}

func logPodEvents(f *framework.Framework) {
	framework.Logf("Summary of pod events during the test:")
	err := framework.ListNamespaceEvents(f.ClientSet, f.Namespace.Name)
	framework.ExpectNoError(err)
}

func logNodeEvents(f *framework.Framework) {
	framework.Logf("Summary of node events during the test:")
	err := framework.ListNamespaceEvents(f.ClientSet, "")
	framework.ExpectNoError(err)
}

func getLocalNode(f *framework.Framework) *apiv1.Node {
	nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
	Expect(len(nodeList.Items)).To(Equal(1), "Unexpected number of node objects for node e2e. Expects only one node.")
	return &nodeList.Items[0]
}

// logs prometheus metrics from the local kubelet.
func logKubeletMetrics(metricKeys ...string) {
	metricSet := sets.NewString()
	for _, key := range metricKeys {
		metricSet.Insert(kubeletmetrics.KubeletSubsystem + "_" + key)
	}
	metric, err := metrics.GrabKubeletMetricsWithoutProxy(framework.TestContext.NodeName + ":10255")
	if err != nil {
		framework.Logf("Error getting kubelet metrics: %v", err)
	} else {
		framework.Logf("Kubelet Metrics: %+v", framework.GetKubeletMetrics(metric, metricSet))
	}
}

func newKubeletConfigJSONEncoder() (runtime.Encoder, error) {
	_, kubeletCodecs, err := kubeletscheme.NewSchemeAndCodecs()
	if err != nil {
		return nil, err
	}

	mediaType := "application/json"
	info, ok := runtime.SerializerInfoForMediaType(kubeletCodecs.SupportedMediaTypes(), mediaType)
	if !ok {
		return nil, fmt.Errorf("unsupported media type %q", mediaType)
	}
	return kubeletCodecs.EncoderForVersion(info.Serializer, kubeletconfigv1alpha1.SchemeGroupVersion), nil
}

// runCommand runs the cmd and returns the combined stdout and stderr, or an
// error if the command failed.
func runCommand(cmd ...string) (string, error) {
	output, err := exec.Command(cmd[0], cmd[1:]...).CombinedOutput()
	if err != nil {
		return "", fmt.Errorf("failed to run %q: %s (%s)", strings.Join(cmd, " "), err, output)
	}
	return string(output), nil
}
<|endoftext|>


<commit_before>package observable

import (
	"reflect"

	"github.com/raininfall/gorx"
	"github.com/raininfall/gorx/observer"
)

type observableMerge struct {
	observable
	inputs []rx.Observable
}

func (oba *observableMerge) Subscribe(out rx.InObserver) {
	go func() {
		defer out.Close()
		//The first elem holds for unsubscribe chan
		observers := make([]rx.Observer, len(oba.inputs)+1)
		cases := make([]reflect.SelectCase, len(oba.inputs)+1)

		for i, in := range oba.inputs {
			obs := observer.New(0)
			in.Subscribe(obs)
			defer obs.Unsubscribe()
			observers[i+1] = obs
			cases[i+1].Dir = reflect.SelectRecv
		}
		cases[0].Dir = reflect.SelectRecv

		for len(observers) > 1 {
			// Re-assign channel from func Out in case of change
			for i := 1; i < len(cases); i++ {
				cases[i].Chan = reflect.ValueOf(observers[i].Out())
			}
			cases[0].Chan = reflect.ValueOf(out.OnUnsubscribe())
			chosen, recv, ok := reflect.Select(cases)
			if !ok {
				//It should not be first elem(unsubscribe chan)
				observers = append(observers[:chosen], observers[chosen+1:]...)
				cases = append(cases[:chosen], cases[chosen+1:]...)
				continue
			}
			if chosen == 0 {
				return
			}
			out.In() <- recv.Interface()
			switch recv.Interface().(type) {
			case error:
				return
			}
		}
	}()
}

/*Merge all emitter into one observable*/
func Merge(inputs ...rx.Observable) rx.Observable {
	me := &observableMerge{
		inputs: inputs,
	}
	me.subscribe = me.Subscribe

	return me
}
<commit_msg>fix merge<commit_after>package observable

import (
	"reflect"

	"github.com/raininfall/gorx"
	"github.com/raininfall/gorx/observer"
)

type observableMerge struct {
	observable
	inputs []rx.Observable
}

func (oba *observableMerge) Subscribe(out rx.InObserver) {
	go func() {
		defer out.Close()
		//The first elem holds for unsubscribe chan
		observers := make([]rx.Observer, len(oba.inputs)+1)
		cases := make([]reflect.SelectCase, len(oba.inputs)+1)

		for i, in := range oba.inputs {
			obs := observer.New(0)
			in.Subscribe(obs)
			defer obs.Unsubscribe()
			observers[i+1] = obs
			cases[i+1].Dir = reflect.SelectRecv
		}
		cases[0].Dir = reflect.SelectRecv

		for len(observers) > 1 {
			// Re-assign channel from func Out in case of change
			for i := 1; i < len(cases); i++ {
				cases[i].Chan = reflect.ValueOf(observers[i].Out())
			}
			cases[0].Chan = reflect.ValueOf(out.OnUnsubscribe())
			chosen, recv, ok := reflect.Select(cases)
			/*Unsubscribe*/
			if chosen == 0 {
				return
			}
			/*One input Observable closed*/
			if !ok {
				//It should not be first elem(unsubscribe chan)
				observers = append(observers[:chosen], observers[chosen+1:]...)
				cases = append(cases[:chosen], cases[chosen+1:]...)
				continue
			}
			out.In() <- recv.Interface()
			switch recv.Interface().(type) {
			case error:
				return
			}
		}
	}()
}

/*Merge all emitter into one observable*/
func Merge(inputs ...rx.Observable) rx.Observable {
	me := &observableMerge{
		inputs: inputs,
	}
	me.subscribe = me.Subscribe

	return me
}
<|endoftext|>


<commit_before>package packer

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/mitchellh/mapstructure"
	"sort"
)

// The rawTemplate struct represents the structure of a template read
// directly from a file. The builders and other components map just to
// "interface{}" pointers since we actually don't know what their contents
// are until we read the "type" field.
type rawTemplate struct {
	Builders       []map[string]interface{}
	Hooks          map[string][]string
	Provisioners   []map[string]interface{}
	PostProcessors []interface{} `mapstructure:"post-processors"`
}

// The Template struct represents a parsed template, parsed into the most
// completed form it can be without additional processing by the caller.
type Template struct {
	Builders       map[string]rawBuilderConfig
	Hooks          map[string][]string
	PostProcessors [][]rawPostProcessorConfig
	Provisioners   []rawProvisionerConfig
}

// The rawBuilderConfig struct represents a raw, unprocessed builder
// configuration. It contains the name of the builder as well as the
// raw configuration. If requested, this is used to compile into a full
// builder configuration at some point.
type rawBuilderConfig struct {
	Name string
	Type string

	rawConfig interface{}
}

// rawPostProcessorConfig represents a raw, unprocessed post-processor
// configuration. It contains the type of the post processor as well as the
// raw configuration that is handed to the post-processor for it to process.
type rawPostProcessorConfig struct {
	Type              string
	KeepInputArtifact bool `mapstructure:"keep_input_artifact"`
	rawConfig         interface{}
}

// rawProvisionerConfig represents a raw, unprocessed provisioner configuration.
// It contains the type of the provisioner as well as the raw configuration
// that is handed to the provisioner for it to process.
type rawProvisionerConfig struct {
	Type     string
	Override map[string]interface{}

	rawConfig interface{}
}

// ParseTemplate takes a byte slice and parses a Template from it, returning
// the template and possibly errors while loading the template. The error
// could potentially be a MultiError, representing multiple errors. Knowing
// and checking for this can be useful, if you wish to format it in a certain
// way.
func ParseTemplate(data []byte) (t *Template, err error) {
	var rawTplInterface interface{}
	err = json.Unmarshal(data, &rawTplInterface)
	if err != nil {
		syntaxErr, ok := err.(*json.SyntaxError)
		if !ok {
			return
		}

		// We have a syntax error. Extract out the line number and friends.
		// https://groups.google.com/forum/#!topic/golang-nuts/fizimmXtVfc
		newline := []byte{'\x0a'}

		// Calculate the start/end position of the line where the error is
		start := bytes.LastIndex(data[:syntaxErr.Offset], newline) + 1
		end := len(data)
		if idx := bytes.Index(data[start:], newline); idx >= 0 {
			end = start + idx
		}

		// Count the line number we're on plus the offset in the line
		line := bytes.Count(data[:start], newline) + 1
		pos := int(syntaxErr.Offset) - start - 1

		err = fmt.Errorf("Error in line %d, char %d: %s\n%s",
			line, pos, syntaxErr, data[start:end])

		return
	}

	// Decode the raw template interface into the actual rawTemplate
	// structure, checking for any extranneous keys along the way.
	var md mapstructure.Metadata
	var rawTpl rawTemplate
	decoderConfig := &mapstructure.DecoderConfig{
		Metadata: &md,
		Result:   &rawTpl,
	}

	decoder, err := mapstructure.NewDecoder(decoderConfig)
	if err != nil {
		return
	}

	err = decoder.Decode(rawTplInterface)
	if err != nil {
		return
	}

	errors := make([]error, 0)

	if len(md.Unused) > 0 {
		sort.Strings(md.Unused)
		for _, unused := range md.Unused {
			errors = append(
				errors, fmt.Errorf("Unknown root level key in template: '%s'", unused))
		}
	}

	t = &Template{}
	t.Builders = make(map[string]rawBuilderConfig)
	t.Hooks = rawTpl.Hooks
	t.PostProcessors = make([][]rawPostProcessorConfig, len(rawTpl.PostProcessors))
	t.Provisioners = make([]rawProvisionerConfig, len(rawTpl.Provisioners))

	// Gather all the builders
	for i, v := range rawTpl.Builders {
		var raw rawBuilderConfig
		if err := mapstructure.Decode(v, &raw); err != nil {
			if merr, ok := err.(*mapstructure.Error); ok {
				for _, err := range merr.Errors {
					errors = append(errors, fmt.Errorf("builder %d: %s", i+1, err))
				}
			} else {
				errors = append(errors, fmt.Errorf("builder %d: %s", i+1, err))
			}

			continue
		}

		if raw.Type == "" {
			errors = append(errors, fmt.Errorf("builder %d: missing 'type'", i+1))
			continue
		}

		// Attempt to get the name of the builder. If the "name" key
		// missing, use the "type" field, which is guaranteed to exist
		// at this point.
		if raw.Name == "" {
			raw.Name = raw.Type
		}

		// Check if we already have a builder with this name and error if so
		if _, ok := t.Builders[raw.Name]; ok {
			errors = append(errors, fmt.Errorf("builder with name '%s' already exists", raw.Name))
			continue
		}

		raw.rawConfig = v

		t.Builders[raw.Name] = raw
	}

	// Gather all the post-processors. This is a complicated process since there
	// are actually three different formats that the user can use to define
	// a post-processor.
	for i, rawV := range rawTpl.PostProcessors {
		rawPP, err := parsePostProvisioner(i, rawV)
		if err != nil {
			errors = append(errors, err...)
			continue
		}

		t.PostProcessors[i] = make([]rawPostProcessorConfig, len(rawPP))
		configs := t.PostProcessors[i]
		for j, pp := range rawPP {
			config := &configs[j]
			if err := mapstructure.Decode(pp, config); err != nil {
				if merr, ok := err.(*mapstructure.Error); ok {
					for _, err := range merr.Errors {
						errors = append(errors, fmt.Errorf("Post-processor #%d.%d: %s", i+1, j+1, err))
					}
				} else {
					errors = append(errors, fmt.Errorf("Post-processor %d.%d: %s", i+1, j+1, err))
				}

				continue
			}

			if config.Type == "" {
				errors = append(errors, fmt.Errorf("Post-processor %d.%d: missing 'type'", i+1, j+1))
				continue
			}

			config.rawConfig = pp
		}
	}

	// Gather all the provisioners
	for i, v := range rawTpl.Provisioners {
		raw := &t.Provisioners[i]
		if err := mapstructure.Decode(v, raw); err != nil {
			if merr, ok := err.(*mapstructure.Error); ok {
				for _, err := range merr.Errors {
					errors = append(errors, fmt.Errorf("provisioner %d: %s", i+1, err))
				}
			} else {
				errors = append(errors, fmt.Errorf("provisioner %d: %s", i+1, err))
			}

			continue
		}

		if raw.Type == "" {
			errors = append(errors, fmt.Errorf("provisioner %d: missing 'type'", i+1))
			continue
		}

		raw.rawConfig = v
	}

	if len(t.Builders) == 0 {
		errors = append(errors, fmt.Errorf("No builders are defined in the template."))
	}

	// If there were errors, we put it into a MultiError and return
	if len(errors) > 0 {
		err = &MultiError{errors}
		t = nil
		return
	}

	return
}

func parsePostProvisioner(i int, rawV interface{}) (result []map[string]interface{}, errors []error) {
	switch v := rawV.(type) {
	case string:
		result = []map[string]interface{}{
			{"type": v},
		}
	case map[string]interface{}:
		result = []map[string]interface{}{v}
	case []interface{}:
		result = make([]map[string]interface{}, len(v))
		errors = make([]error, 0)
		for j, innerRawV := range v {
			switch innerV := innerRawV.(type) {
			case string:
				result[j] = map[string]interface{}{"type": innerV}
			case map[string]interface{}:
				result[j] = innerV
			case []interface{}:
				errors = append(
					errors,
					fmt.Errorf("Post-processor %d.%d: sequences not allowed to be nested in sequences", i+1, j+1))
			default:
				errors = append(errors, fmt.Errorf("Post-processor %d.%d is in a bad format.", i+1, j+1))
			}
		}

		if len(errors) == 0 {
			errors = nil
		}
	default:
		result = nil
		errors = []error{fmt.Errorf("Post-processor %d is in a bad format.", i+1)}
	}

	return
}

// BuildNames returns a slice of the available names of builds that
// this template represents.
func (t *Template) BuildNames() []string {
	names := make([]string, 0, len(t.Builders))
	for name, _ := range t.Builders {
		names = append(names, name)
	}

	return names
}

// Build returns a Build for the given name.
//
// If the build does not exist as part of this template, an error is
// returned.
func (t *Template) Build(name string, components *ComponentFinder) (b Build, err error) {
	// Setup the Builder
	builderConfig, ok := t.Builders[name]
	if !ok {
		err = fmt.Errorf("No such build found in template: %s", name)
		return
	}

	// We panic if there is no builder function because this is really
	// an internal bug that always needs to be fixed, not an error.
	if components.Builder == nil {
		panic("no builder function")
	}

	// Panic if there are provisioners on the template but no provisioner
	// component finder. This is always an internal error, so we panic.
	if len(t.Provisioners) > 0 && components.Provisioner == nil {
		panic("no provisioner function")
	}

	builder, err := components.Builder(builderConfig.Type)
	if err != nil {
		return
	}

	if builder == nil {
		err = fmt.Errorf("Builder type not found: %s", builderConfig.Type)
		return
	}

	// Gather the Hooks
	hooks := make(map[string][]Hook)
	for tplEvent, tplHooks := range t.Hooks {
		curHooks := make([]Hook, 0, len(tplHooks))

		for _, hookName := range tplHooks {
			var hook Hook
			hook, err = components.Hook(hookName)
			if err != nil {
				return
			}

			if hook == nil {
				err = fmt.Errorf("Hook not found: %s", hookName)
				return
			}

			curHooks = append(curHooks, hook)
		}

		hooks[tplEvent] = curHooks
	}

	// Prepare the post-processors
	postProcessors := make([][]coreBuildPostProcessor, 0, len(t.PostProcessors))
	for _, rawPPs := range t.PostProcessors {
		current := make([]coreBuildPostProcessor, len(rawPPs))
		for i, rawPP := range rawPPs {
			pp, err := components.PostProcessor(rawPP.Type)
			if err != nil {
				return nil, err
			}

			if pp == nil {
				return nil, fmt.Errorf("PostProcessor type not found: %s", rawPP.Type)
			}

			current[i] = coreBuildPostProcessor{
				processor:         pp,
				processorType:     rawPP.Type,
				config:            rawPP.rawConfig,
				keepInputArtifact: rawPP.KeepInputArtifact,
			}
		}

		postProcessors = append(postProcessors, current)
	}

	// Prepare the provisioners
	provisioners := make([]coreBuildProvisioner, 0, len(t.Provisioners))
	for _, rawProvisioner := range t.Provisioners {
		var provisioner Provisioner
		provisioner, err = components.Provisioner(rawProvisioner.Type)
		if err != nil {
			return
		}

		if provisioner == nil {
			err = fmt.Errorf("Provisioner type not found: %s", rawProvisioner.Type)
			return
		}

		configs := make([]interface{}, 1, 2)
		configs[0] = rawProvisioner.rawConfig

		if rawProvisioner.Override != nil {
			if override, ok := rawProvisioner.Override[name]; ok {
				configs = append(configs, override)
			}
		}

		coreProv := coreBuildProvisioner{provisioner, configs}
		provisioners = append(provisioners, coreProv)
	}

	b = &coreBuild{
		name:           name,
		builder:        builder,
		builderConfig:  builderConfig.rawConfig,
		builderType:    builderConfig.Type,
		hooks:          hooks,
		postProcessors: postProcessors,
		provisioners:   provisioners,
	}

	return
}
<commit_msg>Don't expose overrides directly to provisioners<commit_after>package packer

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/mitchellh/mapstructure"
	"sort"
)

// The rawTemplate struct represents the structure of a template read
// directly from a file. The builders and other components map just to
// "interface{}" pointers since we actually don't know what their contents
// are until we read the "type" field.
type rawTemplate struct {
	Builders       []map[string]interface{}
	Hooks          map[string][]string
	Provisioners   []map[string]interface{}
	PostProcessors []interface{} `mapstructure:"post-processors"`
}

// The Template struct represents a parsed template, parsed into the most
// completed form it can be without additional processing by the caller.
type Template struct {
	Builders       map[string]rawBuilderConfig
	Hooks          map[string][]string
	PostProcessors [][]rawPostProcessorConfig
	Provisioners   []rawProvisionerConfig
}

// The rawBuilderConfig struct represents a raw, unprocessed builder
// configuration. It contains the name of the builder as well as the
// raw configuration. If requested, this is used to compile into a full
// builder configuration at some point.
type rawBuilderConfig struct {
	Name string
	Type string

	rawConfig interface{}
}

// rawPostProcessorConfig represents a raw, unprocessed post-processor
// configuration. It contains the type of the post processor as well as the
// raw configuration that is handed to the post-processor for it to process.
type rawPostProcessorConfig struct {
	Type              string
	KeepInputArtifact bool `mapstructure:"keep_input_artifact"`
	rawConfig         interface{}
}

// rawProvisionerConfig represents a raw, unprocessed provisioner configuration.
// It contains the type of the provisioner as well as the raw configuration
// that is handed to the provisioner for it to process.
type rawProvisionerConfig struct {
	Type     string
	Override map[string]interface{}

	rawConfig interface{}
}

// ParseTemplate takes a byte slice and parses a Template from it, returning
// the template and possibly errors while loading the template. The error
// could potentially be a MultiError, representing multiple errors. Knowing
// and checking for this can be useful, if you wish to format it in a certain
// way.
func ParseTemplate(data []byte) (t *Template, err error) {
	var rawTplInterface interface{}
	err = json.Unmarshal(data, &rawTplInterface)
	if err != nil {
		syntaxErr, ok := err.(*json.SyntaxError)
		if !ok {
			return
		}

		// We have a syntax error. Extract out the line number and friends.
		// https://groups.google.com/forum/#!topic/golang-nuts/fizimmXtVfc
		newline := []byte{'\x0a'}

		// Calculate the start/end position of the line where the error is
		start := bytes.LastIndex(data[:syntaxErr.Offset], newline) + 1
		end := len(data)
		if idx := bytes.Index(data[start:], newline); idx >= 0 {
			end = start + idx
		}

		// Count the line number we're on plus the offset in the line
		line := bytes.Count(data[:start], newline) + 1
		pos := int(syntaxErr.Offset) - start - 1

		err = fmt.Errorf("Error in line %d, char %d: %s\n%s",
			line, pos, syntaxErr, data[start:end])

		return
	}

	// Decode the raw template interface into the actual rawTemplate
	// structure, checking for any extranneous keys along the way.
	var md mapstructure.Metadata
	var rawTpl rawTemplate
	decoderConfig := &mapstructure.DecoderConfig{
		Metadata: &md,
		Result:   &rawTpl,
	}

	decoder, err := mapstructure.NewDecoder(decoderConfig)
	if err != nil {
		return
	}

	err = decoder.Decode(rawTplInterface)
	if err != nil {
		return
	}

	errors := make([]error, 0)

	if len(md.Unused) > 0 {
		sort.Strings(md.Unused)
		for _, unused := range md.Unused {
			errors = append(
				errors, fmt.Errorf("Unknown root level key in template: '%s'", unused))
		}
	}

	t = &Template{}
	t.Builders = make(map[string]rawBuilderConfig)
	t.Hooks = rawTpl.Hooks
	t.PostProcessors = make([][]rawPostProcessorConfig, len(rawTpl.PostProcessors))
	t.Provisioners = make([]rawProvisionerConfig, len(rawTpl.Provisioners))

	// Gather all the builders
	for i, v := range rawTpl.Builders {
		var raw rawBuilderConfig
		if err := mapstructure.Decode(v, &raw); err != nil {
			if merr, ok := err.(*mapstructure.Error); ok {
				for _, err := range merr.Errors {
					errors = append(errors, fmt.Errorf("builder %d: %s", i+1, err))
				}
			} else {
				errors = append(errors, fmt.Errorf("builder %d: %s", i+1, err))
			}

			continue
		}

		if raw.Type == "" {
			errors = append(errors, fmt.Errorf("builder %d: missing 'type'", i+1))
			continue
		}

		// Attempt to get the name of the builder. If the "name" key
		// missing, use the "type" field, which is guaranteed to exist
		// at this point.
		if raw.Name == "" {
			raw.Name = raw.Type
		}

		// Check if we already have a builder with this name and error if so
		if _, ok := t.Builders[raw.Name]; ok {
			errors = append(errors, fmt.Errorf("builder with name '%s' already exists", raw.Name))
			continue
		}

		raw.rawConfig = v

		t.Builders[raw.Name] = raw
	}

	// Gather all the post-processors. This is a complicated process since there
	// are actually three different formats that the user can use to define
	// a post-processor.
	for i, rawV := range rawTpl.PostProcessors {
		rawPP, err := parsePostProvisioner(i, rawV)
		if err != nil {
			errors = append(errors, err...)
			continue
		}

		t.PostProcessors[i] = make([]rawPostProcessorConfig, len(rawPP))
		configs := t.PostProcessors[i]
		for j, pp := range rawPP {
			config := &configs[j]
			if err := mapstructure.Decode(pp, config); err != nil {
				if merr, ok := err.(*mapstructure.Error); ok {
					for _, err := range merr.Errors {
						errors = append(errors, fmt.Errorf("Post-processor #%d.%d: %s", i+1, j+1, err))
					}
				} else {
					errors = append(errors, fmt.Errorf("Post-processor %d.%d: %s", i+1, j+1, err))
				}

				continue
			}

			if config.Type == "" {
				errors = append(errors, fmt.Errorf("Post-processor %d.%d: missing 'type'", i+1, j+1))
				continue
			}

			config.rawConfig = pp
		}
	}

	// Gather all the provisioners
	for i, v := range rawTpl.Provisioners {
		raw := &t.Provisioners[i]
		if err := mapstructure.Decode(v, raw); err != nil {
			if merr, ok := err.(*mapstructure.Error); ok {
				for _, err := range merr.Errors {
					errors = append(errors, fmt.Errorf("provisioner %d: %s", i+1, err))
				}
			} else {
				errors = append(errors, fmt.Errorf("provisioner %d: %s", i+1, err))
			}

			continue
		}

		if raw.Type == "" {
			errors = append(errors, fmt.Errorf("provisioner %d: missing 'type'", i+1))
			continue
		}

		// The provisioners not only don't need or want the override settings
		// (as they are processed as part of the preparation below), but will
		// actively reject them as invalid configuration.
		delete(v, "override")

		raw.rawConfig = v
	}

	if len(t.Builders) == 0 {
		errors = append(errors, fmt.Errorf("No builders are defined in the template."))
	}

	// If there were errors, we put it into a MultiError and return
	if len(errors) > 0 {
		err = &MultiError{errors}
		t = nil
		return
	}

	return
}

func parsePostProvisioner(i int, rawV interface{}) (result []map[string]interface{}, errors []error) {
	switch v := rawV.(type) {
	case string:
		result = []map[string]interface{}{
			{"type": v},
		}
	case map[string]interface{}:
		result = []map[string]interface{}{v}
	case []interface{}:
		result = make([]map[string]interface{}, len(v))
		errors = make([]error, 0)
		for j, innerRawV := range v {
			switch innerV := innerRawV.(type) {
			case string:
				result[j] = map[string]interface{}{"type": innerV}
			case map[string]interface{}:
				result[j] = innerV
			case []interface{}:
				errors = append(
					errors,
					fmt.Errorf("Post-processor %d.%d: sequences not allowed to be nested in sequences", i+1, j+1))
			default:
				errors = append(errors, fmt.Errorf("Post-processor %d.%d is in a bad format.", i+1, j+1))
			}
		}

		if len(errors) == 0 {
			errors = nil
		}
	default:
		result = nil
		errors = []error{fmt.Errorf("Post-processor %d is in a bad format.", i+1)}
	}

	return
}

// BuildNames returns a slice of the available names of builds that
// this template represents.
func (t *Template) BuildNames() []string {
	names := make([]string, 0, len(t.Builders))
	for name, _ := range t.Builders {
		names = append(names, name)
	}

	return names
}

// Build returns a Build for the given name.
//
// If the build does not exist as part of this template, an error is
// returned.
func (t *Template) Build(name string, components *ComponentFinder) (b Build, err error) {
	// Setup the Builder
	builderConfig, ok := t.Builders[name]
	if !ok {
		err = fmt.Errorf("No such build found in template: %s", name)
		return
	}

	// We panic if there is no builder function because this is really
	// an internal bug that always needs to be fixed, not an error.
	if components.Builder == nil {
		panic("no builder function")
	}

	// Panic if there are provisioners on the template but no provisioner
	// component finder. This is always an internal error, so we panic.
	if len(t.Provisioners) > 0 && components.Provisioner == nil {
		panic("no provisioner function")
	}

	builder, err := components.Builder(builderConfig.Type)
	if err != nil {
		return
	}

	if builder == nil {
		err = fmt.Errorf("Builder type not found: %s", builderConfig.Type)
		return
	}

	// Gather the Hooks
	hooks := make(map[string][]Hook)
	for tplEvent, tplHooks := range t.Hooks {
		curHooks := make([]Hook, 0, len(tplHooks))

		for _, hookName := range tplHooks {
			var hook Hook
			hook, err = components.Hook(hookName)
			if err != nil {
				return
			}

			if hook == nil {
				err = fmt.Errorf("Hook not found: %s", hookName)
				return
			}

			curHooks = append(curHooks, hook)
		}

		hooks[tplEvent] = curHooks
	}

	// Prepare the post-processors
	postProcessors := make([][]coreBuildPostProcessor, 0, len(t.PostProcessors))
	for _, rawPPs := range t.PostProcessors {
		current := make([]coreBuildPostProcessor, len(rawPPs))
		for i, rawPP := range rawPPs {
			pp, err := components.PostProcessor(rawPP.Type)
			if err != nil {
				return nil, err
			}

			if pp == nil {
				return nil, fmt.Errorf("PostProcessor type not found: %s", rawPP.Type)
			}

			current[i] = coreBuildPostProcessor{
				processor:         pp,
				processorType:     rawPP.Type,
				config:            rawPP.rawConfig,
				keepInputArtifact: rawPP.KeepInputArtifact,
			}
		}

		postProcessors = append(postProcessors, current)
	}

	// Prepare the provisioners
	provisioners := make([]coreBuildProvisioner, 0, len(t.Provisioners))
	for _, rawProvisioner := range t.Provisioners {
		var provisioner Provisioner
		provisioner, err = components.Provisioner(rawProvisioner.Type)
		if err != nil {
			return
		}

		if provisioner == nil {
			err = fmt.Errorf("Provisioner type not found: %s", rawProvisioner.Type)
			return
		}

		configs := make([]interface{}, 1, 2)
		configs[0] = rawProvisioner.rawConfig

		if rawProvisioner.Override != nil {
			if override, ok := rawProvisioner.Override[name]; ok {
				configs = append(configs, override)
			}
		}

		coreProv := coreBuildProvisioner{provisioner, configs}
		provisioners = append(provisioners, coreProv)
	}

	b = &coreBuild{
		name:           name,
		builder:        builder,
		builderConfig:  builderConfig.rawConfig,
		builderType:    builderConfig.Type,
		hooks:          hooks,
		postProcessors: postProcessors,
		provisioners:   provisioners,
	}

	return
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/the42/ogdat/ckan"
	"github.com/the42/ogdat/ogdatv21"
	"github.com/the42/ogdat/schedule"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

const AppID = "a6545f8f-e0c9-4917-83c7-3e47bd1e0247"

var logger *log.Logger
var db *DBConn
var portal *ckan.Portal

var resettdb = flag.Bool("resetdb", false, "Delete the tracking database. You will be prompted before actual deletion. Process will terminate afterwards.")
var inittdb = flag.Bool("initdb", false, "Initialize the tracking database. In case there are old entries in the tracking database, use init in conjunction with reset. Process will terminate afterwards.")
var servetdb = flag.Bool("serve", false, "Start in watchdog mode. Process will continue to run until it receives a (clean shutdown) or gets killed")
var DEBUG = flag.Bool("DEBUG", false, "DEBUG MODE")

func gotyesonprompt() bool {
	var prompt string
	fmt.Scanf("%s", &prompt)
	prompt = strings.ToLower(strings.TrimSpace(prompt))
	if len(prompt) > 0 {
		return prompt[0] == 'y'
	}
	return false
}

func getheartbeatinterval() int {

	if i, err := strconv.Atoi(os.Getenv("HEARTBEAT_INTERVAL")); err == nil {
		return i
	}
	return 10 // Minutes
}

func getnumworkers() int {
	if i, err := strconv.Atoi(os.Getenv("PARALLEL_FETCHNO")); err == nil {
		return i
	}
	return 4 // process four IDs in parallel
}

func getckanurl() (url string) {

	const CKAN_URL = "http://www.data.gv.at/katalog/api/"

	url = os.Getenv("CKAN_URL")
	if url == "" {
		url = CKAN_URL
	}
	return
}

func initdb() {
	if err := db.CreateDatabase(); err != nil {
		s := fmt.Sprintf("Database initialisation failed: %s", err)
		fmt.Println(s)
		logger.Panic(s)
	}
}

func resetdb() {
	logger.Println("Warning: Requesting database reset")
	fmt.Print("\n\nALL RECORDED DATA IN DATABASE  WILL BE DELETED.\nDO YOU REALLY WANT TO PROCEED? [N,y]\n")
	if !gotyesonprompt() {
		fmt.Print("\nABORTING\n\n")
		logger.Println("Info: Database reset canceled")
	} else {
		if err := db.ResetDatabase(); err != nil {
			s := fmt.Sprintf("Database reset failed: %s", err)
			fmt.Println(s)
			logger.Panic(s)
		}
	}
}

func stringslicetoiface(ss []string) []interface{} {
	slice := make([]interface{}, len(ss))
	for i, v := range ss {
		slice[i] = v
	}
	return slice
}

func ifaceslicetostring(ifs []interface{}) []string {
	slice := make([]string, len(ifs))
	for i, v := range ifs {
		s, ok := v.(string)
		if !ok {
			panic("Interface value not of string type")
		}
		slice[i] = s
	}
	return slice
}

func processmetadataids(conn *DBConn, processids []string) error {

	for _, id := range processids {

		logger.Println(fmt.Sprintf("Processing %v", id))

		mdjson, err := portal.GetJSONforID(id, true)
		if err != nil {
			return fmt.Errorf("Cannot fetch JSON for ID %v: %s", id, err)
		}

		md, err := ogdatv21.MetadatafromJSON(mdjson)
		if err != nil {
			return fmt.Errorf("Cannot access Metadata for ID %v: %s", id, err)
		}

		dbdatasetid, isnew, err := conn.InsertOrUpdateMetadataInfo(md)
		if err != nil {
			return fmt.Errorf("InsertOrUpdateMetadataInfo: Database Error at id %v: %s", id, err)
		}

		messages, err := md.Check(true)
		if err != nil {
			return fmt.Errorf("Metadata Check Error for id %v: %s", id, err)
		}

		if err = conn.ProtocollCheck(dbdatasetid, isnew, messages); err != nil {
			return fmt.Errorf("ProtocollCheck: Database Error at id %v: %s", id, err)
		}
	}
	return nil
}

func mymain() int {

	if flag.NFlag() == 0 {
		fmt.Println("No command line flags given. Usage:")
		flag.PrintDefaults()
		logger.Panicln("Fatal: No command line flags given")
	}

	lockfile := NewLockfile(lockfilename)
	defer lockfile.Delete()
	lockfile.WriteInfo()

	dbconnection := GetDatabaseConnection(AppID)
	db = &DBConn{dbconnection, AppID}
	defer dbconnection.Close()

	if *resettdb || *inittdb {
		if *inittdb {
			initdb()
		}
		if *resettdb {
			resetdb()
		}
		logger.Println("Info: Earyl exit due to maintainance switches")
		return 2
	}

	if *servetdb {

		portal = ckan.NewDataPortalAPIEndpoint(getckanurl(), "2/")
		heartbeatinterval := getheartbeatinterval()
		numworkers := getnumworkers()

		for {
			hit, err := db.GetLastHit()
			if err != nil {
				s := fmt.Sprintf("Cannot read last DBHit: %s", err)
				fmt.Println(s)
				logger.Panic(s)
			}

			var processids []string
			if hit == nil {
				processids, err = portal.GetAllMetaDataIDs()
			} else {
				processids, err = portal.GetChangedPackageIDsSince(*hit, numworkers)
			}

			if anzids := len(processids); anzids > 0 {

				tx, _ := dbconnection.Begin()
				scheduler := schedule.New(numworkers)
				conn := &DBConn{DBer: tx, appid: AppID}
				f := func(slice []interface{}) error {
					if err := processmetadataids(conn, ifaceslicetostring(slice)); err != nil {
						return err
					}
					return nil
				}

				db.LogMessage(fmt.Sprintf("%d Medadaten werden verarbeitet", anzids), StateOk, true)
				workchannel := scheduler.Schedule(f, stringslicetoiface(processids))
			workloop:
				for {
					select {
					case workreply := <-workchannel:
						if err := workreply.Err; err != nil {
							logger.Panicln("Scheduler didn't return success:", err)
						} else if workreply.Code == schedule.StateFinish {
							tx.Commit()
							db.LogMessage("Idle", StateOk, true)
							break workloop
						}
					case <-time.After(time.Duration(heartbeatinterval) * time.Minute):
						logger.Println("Alive")
						db.HeartBeat()
					}
				}

			}
			select {
			case <-time.After(time.Duration(heartbeatinterval) * time.Minute):
				logger.Println("Alive")
				db.HeartBeat()
			}

		}
	}
	return 0
}

func main() {
	os.Exit(mymain())
}

func init() {
	logger = log.New(os.Stderr, filepath.Base(os.Args[0])+": ", log.LstdFlags)
	flag.Parse()
}
<commit_msg>check for error when creating a database transaction<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/the42/ogdat/ckan"
	"github.com/the42/ogdat/ogdatv21"
	"github.com/the42/ogdat/schedule"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

const AppID = "a6545f8f-e0c9-4917-83c7-3e47bd1e0247"

var logger *log.Logger
var db *DBConn
var portal *ckan.Portal

var resettdb = flag.Bool("resetdb", false, "Delete the tracking database. You will be prompted before actual deletion. Process will terminate afterwards.")
var inittdb = flag.Bool("initdb", false, "Initialize the tracking database. In case there are old entries in the tracking database, use init in conjunction with reset. Process will terminate afterwards.")
var servetdb = flag.Bool("serve", false, "Start in watchdog mode. Process will continue to run until it receives a (clean shutdown) or gets killed")
var DEBUG = flag.Bool("DEBUG", false, "DEBUG MODE")

func gotyesonprompt() bool {
	var prompt string
	fmt.Scanf("%s", &prompt)
	prompt = strings.ToLower(strings.TrimSpace(prompt))
	if len(prompt) > 0 {
		return prompt[0] == 'y'
	}
	return false
}

func getheartbeatinterval() int {

	if i, err := strconv.Atoi(os.Getenv("HEARTBEAT_INTERVAL")); err == nil {
		return i
	}
	return 10 // Minutes
}

func getnumworkers() int {
	if i, err := strconv.Atoi(os.Getenv("PARALLEL_FETCHNO")); err == nil {
		return i
	}
	return 4 // process four IDs in parallel
}

func getckanurl() (url string) {

	const CKAN_URL = "http://www.data.gv.at/katalog/api/"

	url = os.Getenv("CKAN_URL")
	if url == "" {
		url = CKAN_URL
	}
	return
}

func initdb() {
	if err := db.CreateDatabase(); err != nil {
		s := fmt.Sprintf("Database initialisation failed: %s", err)
		fmt.Println(s)
		logger.Panic(s)
	}
}

func resetdb() {
	logger.Println("Warning: Requesting database reset")
	fmt.Print("\n\nALL RECORDED DATA IN DATABASE  WILL BE DELETED.\nDO YOU REALLY WANT TO PROCEED? [N,y]\n")
	if !gotyesonprompt() {
		fmt.Print("\nABORTING\n\n")
		logger.Println("Info: Database reset canceled")
	} else {
		if err := db.ResetDatabase(); err != nil {
			s := fmt.Sprintf("Database reset failed: %s", err)
			fmt.Println(s)
			logger.Panic(s)
		}
	}
}

func stringslicetoiface(ss []string) []interface{} {
	slice := make([]interface{}, len(ss))
	for i, v := range ss {
		slice[i] = v
	}
	return slice
}

func ifaceslicetostring(ifs []interface{}) []string {
	slice := make([]string, len(ifs))
	for i, v := range ifs {
		s, ok := v.(string)
		if !ok {
			panic("Interface value not of string type")
		}
		slice[i] = s
	}
	return slice
}

func processmetadataids(conn *DBConn, processids []string) error {

	for _, id := range processids {

		logger.Println(fmt.Sprintf("Processing %v", id))

		mdjson, err := portal.GetJSONforID(id, true)
		if err != nil {
			return fmt.Errorf("Cannot fetch JSON for ID %v: %s", id, err)
		}

		md, err := ogdatv21.MetadatafromJSON(mdjson)
		if err != nil {
			return fmt.Errorf("Cannot access Metadata for ID %v: %s", id, err)
		}

		dbdatasetid, isnew, err := conn.InsertOrUpdateMetadataInfo(md)
		if err != nil {
			return fmt.Errorf("InsertOrUpdateMetadataInfo: Database Error at id %v: %s", id, err)
		}

		messages, err := md.Check(true)
		if err != nil {
			return fmt.Errorf("Metadata Check Error for id %v: %s", id, err)
		}

		if err = conn.ProtocollCheck(dbdatasetid, isnew, messages); err != nil {
			return fmt.Errorf("ProtocollCheck: Database Error at id %v: %s", id, err)
		}
	}
	return nil
}

func mymain() int {

	if flag.NFlag() == 0 {
		fmt.Println("No command line flags given. Usage:")
		flag.PrintDefaults()
		logger.Panicln("Fatal: No command line flags given")
	}

	lockfile := NewLockfile(lockfilename)
	defer lockfile.Delete()
	lockfile.WriteInfo()

	dbconnection := GetDatabaseConnection(AppID)
	db = &DBConn{dbconnection, AppID}
	defer dbconnection.Close()

	if *resettdb || *inittdb {
		if *inittdb {
			initdb()
		}
		if *resettdb {
			resetdb()
		}
		logger.Println("Info: Earyl exit due to maintainance switches")
		return 2
	}

	if *servetdb {

		portal = ckan.NewDataPortalAPIEndpoint(getckanurl(), "2/")
		heartbeatinterval := getheartbeatinterval()
		numworkers := getnumworkers()

		for {
			hit, err := db.GetLastHit()
			if err != nil {
				s := fmt.Sprintf("Cannot read last DBHit: %s", err)
				fmt.Println(s)
				logger.Panic(s)
			}

			var processids []string
			if hit == nil {
				processids, err = portal.GetAllMetaDataIDs()
			} else {
				processids, err = portal.GetChangedPackageIDsSince(*hit, numworkers)
			}

			if anzids := len(processids); anzids > 0 {

				tx, err := dbconnection.Begin()
				if err != nil {
					logger.Panicln("Cannot create database transaction")
				}
				scheduler := schedule.New(numworkers)
				conn := &DBConn{DBer: tx, appid: AppID}
				f := func(slice []interface{}) error {
					if err := processmetadataids(conn, ifaceslicetostring(slice)); err != nil {
						return err
					}
					return nil
				}

				db.LogMessage(fmt.Sprintf("%d Medadaten werden verarbeitet", anzids), StateOk, true)
				workchannel := scheduler.Schedule(f, stringslicetoiface(processids))
			workloop:
				for {
					select {
					case workreply := <-workchannel:
						if err := workreply.Err; err != nil {
							logger.Panicln("Scheduler didn't return success:", err)
						} else if workreply.Code == schedule.StateFinish {
							tx.Commit()
							db.LogMessage("Idle", StateOk, true)
							break workloop
						}
					case <-time.After(time.Duration(heartbeatinterval) * time.Minute):
						logger.Println("Alive")
						db.HeartBeat()
					}
				}

			}
			select {
			case <-time.After(time.Duration(heartbeatinterval) * time.Minute):
				logger.Println("Alive")
				db.HeartBeat()
			}

		}
	}
	return 0
}

func main() {
	os.Exit(mymain())
}

func init() {
	logger = log.New(os.Stderr, filepath.Base(os.Args[0])+": ", log.LstdFlags)
	flag.Parse()
}
<|endoftext|>


<commit_before>package webpage

import (
	"bytes"
	"html/template"
	"io/ioutil"
	"net/http"

	"github.com/adampresley/webframework/httpService"
	"github.com/pkg/errors"
)

/*
GoLayout is a layout using Go templates
*/
type GoLayout struct {
	layout *template.Template
}

func (l *GoLayout) addHelperFunctions() {
	var funcMap = template.FuncMap{
		"eq": func(a, b interface{}) bool {
			return a == b
		},
		"neq": func(a, b interface{}) bool {
			return a != b
		},
	}

	l.layout.Funcs(funcMap)
}

/*
LoadLayoutFile loads a layout from a file
*/
func (l *GoLayout) LoadLayoutFile(fileName string) error {
	var err error
	var layoutContents []byte

	if layoutContents, err = ioutil.ReadFile(fileName); err != nil {
		return err
	}

	l.layout = template.Must(template.New("layout").Parse(string(layoutContents[:len(layoutContents)])))
	l.addHelperFunctions()
	return nil
}

/*
LoadLayoutString loads a layout from a passed-in byte array
*/
func (l *GoLayout) LoadLayoutString(contents []byte) error {
	l.layout = template.Must(template.New("layout").Parse(string(contents[:len(contents)])))
	l.addHelperFunctions()
	return nil
}

/*
RenderViewFile renders a view from a file into this layout
*/
func (l *GoLayout) RenderViewFile(fileName string, context interface{}) (string, error) {
	var viewContents []byte
	var err error
	var viewTemplate *template.Template
	stringWriter := bytes.NewBufferString("")

	if viewContents, err = ioutil.ReadFile(fileName); err != nil {
		return "", errors.Wrapf(err, "Unable to read the view file %s", fileName)
	}

	if viewTemplate, err = l.layout.Parse(string(viewContents[:len(viewContents)])); err != nil {
		return "", errors.Wrapf(err, "Unable to parse the view file %s", fileName)
	}

	if err = viewTemplate.Execute(stringWriter, context); err != nil {
		return "", errors.Wrapf(err, "Unable to render the file %s", fileName)
	}

	return stringWriter.String(), nil
}

/*
RenderViewFilef renders a view from a file into this layout, then
writes it out to the provider writer. Useful for HTTP responses
*/
func (l *GoLayout) RenderViewFilef(writer http.ResponseWriter, fileName string, context interface{}) error {
	var renderedContents string
	var err error

	if renderedContents, err = l.RenderViewFile(fileName, context); err != nil {
		return errors.Wrapf(err, "Unable to render the view file %s", fileName)
	}

	httpService.WriteHTML(writer, renderedContents, 200)
	return nil
}

/*
RenderViewString renders a view from byte array content into this layout
*/
func (l *GoLayout) RenderViewString(contents []byte, context interface{}) (string, error) {
	var err error

	viewTemplate := template.Must(template.New("view").Parse(string(contents[:len(contents)])))
	stringWriter := bytes.NewBufferString("")

	if err = viewTemplate.ExecuteTemplate(stringWriter, "layout", context); err != nil {
		return "", errors.Wrapf(err, "Unable to render the view")
	}

	return stringWriter.String(), nil
}

/*
RenderViewStringf renders a view from byte array content into this layout, and
writes it out to the provided writer. Useful for HTTP responses
*/
func (l *GoLayout) RenderViewStringf(writer http.ResponseWriter, contents []byte, context interface{}) error {
	var renderedContents string
	var err error

	if renderedContents, err = l.RenderViewString(contents, context); err != nil {
		return errors.Wrap(err, "Unable to render the view")
	}

	httpService.WriteHTML(writer, renderedContents, 200)
	return nil
}
<commit_msg>Fixed error when rendering a view from a string<commit_after>package webpage

import (
	"bytes"
	"html/template"
	"io/ioutil"
	"net/http"

	"github.com/adampresley/webframework/httpService"
	"github.com/pkg/errors"
)

/*
GoLayout is a layout using Go templates
*/
type GoLayout struct {
	layout *template.Template
}

func (l *GoLayout) addHelperFunctions() {
	var funcMap = template.FuncMap{
		"eq": func(a, b interface{}) bool {
			return a == b
		},
		"neq": func(a, b interface{}) bool {
			return a != b
		},
	}

	l.layout.Funcs(funcMap)
}

/*
LoadLayoutFile loads a layout from a file
*/
func (l *GoLayout) LoadLayoutFile(fileName string) error {
	var err error
	var layoutContents []byte

	if layoutContents, err = ioutil.ReadFile(fileName); err != nil {
		return err
	}

	l.layout = template.Must(template.New("layout").Parse(string(layoutContents[:len(layoutContents)])))
	l.addHelperFunctions()
	return nil
}

/*
LoadLayoutString loads a layout from a passed-in byte array
*/
func (l *GoLayout) LoadLayoutString(contents []byte) error {
	l.layout = template.Must(template.New("layout").Parse(string(contents[:len(contents)])))
	l.addHelperFunctions()
	return nil
}

/*
RenderViewFile renders a view from a file into this layout
*/
func (l *GoLayout) RenderViewFile(fileName string, context interface{}) (string, error) {
	var viewContents []byte
	var err error
	var viewTemplate *template.Template
	stringWriter := bytes.NewBufferString("")

	if viewContents, err = ioutil.ReadFile(fileName); err != nil {
		return "", errors.Wrapf(err, "Unable to read the view file %s", fileName)
	}

	if viewTemplate, err = l.layout.Parse(string(viewContents[:len(viewContents)])); err != nil {
		return "", errors.Wrapf(err, "Unable to parse the view file %s", fileName)
	}

	if err = viewTemplate.Execute(stringWriter, context); err != nil {
		return "", errors.Wrapf(err, "Unable to render the file %s", fileName)
	}

	return stringWriter.String(), nil
}

/*
RenderViewFilef renders a view from a file into this layout, then
writes it out to the provider writer. Useful for HTTP responses
*/
func (l *GoLayout) RenderViewFilef(writer http.ResponseWriter, fileName string, context interface{}) error {
	var renderedContents string
	var err error

	if renderedContents, err = l.RenderViewFile(fileName, context); err != nil {
		return errors.Wrapf(err, "Unable to render the view file %s", fileName)
	}

	httpService.WriteHTML(writer, renderedContents, 200)
	return nil
}

/*
RenderViewString renders a view from byte array content into this layout
*/
func (l *GoLayout) RenderViewString(contents []byte, context interface{}) (string, error) {
	var err error
	var viewTemplate *template.Template
	stringWriter := bytes.NewBufferString("")

	if viewTemplate, err = l.layout.Parse(string(contents[:len(contents)])); err != nil {
		return "", errors.Wrapf(err, "Unable to parse the view in RenderViewString()")
	}

	if err = viewTemplate.Execute(stringWriter, context); err != nil {
		return "", errors.Wrapf(err, "Unable to render view in RenderViewString()")
	}

	return stringWriter.String(), nil
}

/*
RenderViewStringf renders a view from byte array content into this layout, and
writes it out to the provided writer. Useful for HTTP responses
*/
func (l *GoLayout) RenderViewStringf(writer http.ResponseWriter, contents []byte, context interface{}) error {
	var renderedContents string
	var err error

	if renderedContents, err = l.RenderViewString(contents, context); err != nil {
		return errors.Wrap(err, "Unable to render the view")
	}

	httpService.WriteHTML(writer, renderedContents, 200)
	return nil
}
<|endoftext|>


<commit_before>package turnpike

import (
	"fmt"
	"net/http"

	"github.com/gorilla/websocket"
)

const (
	jsonWebsocketProtocol    = "wamp.2.json"
	msgpackWebsocketProtocol = "wamp.2.msgpack"
)

type invalidPayload byte

func (e invalidPayload) Error() string {
	return fmt.Sprintf("Invalid payloadType: %d", e)
}

type protocolExists string

func (e protocolExists) Error() string {
	return "This protocol has already been registered: " + string(e)
}

type protocol struct {
	payloadType int
	serializer  Serializer
}

// WebsocketServer handles websocket connections.
type WebsocketServer struct {
	Router
	upgrader *websocket.Upgrader

	protocols map[string]protocol

	// The serializer to use for text frames. Defaults to JSONSerializer.
	TextSerializer Serializer
	// The serializer to use for binary frames. Defaults to JSONSerializer.
	BinarySerializer Serializer
}

// Creates a new WebsocketServer from a map of realms
func NewWebsocketServer(realms map[string]Realm) (*WebsocketServer, error) {
	log.Println("NewWebsocketServer")
	r := NewDefaultRouter()
	for uri, realm := range realms {
		if err := r.RegisterRealm(URI(uri), realm); err != nil {
			return nil, err
		}
	}
	s := newWebsocketServer(r)
	return s, nil
}

// Creates a new WebsocketServer with a single basic realm
func NewBasicWebsocketServer(uri string) *WebsocketServer {
	log.Println("NewBasicWebsocketServer")
	s, _ := NewWebsocketServer(map[string]Realm{uri: {}})
	return s
}

func newWebsocketServer(r Router) *WebsocketServer {
	s := &WebsocketServer{
		Router:    r,
		protocols: make(map[string]protocol),
	}
	s.upgrader = &websocket.Upgrader{}
	s.RegisterProtocol(jsonWebsocketProtocol, websocket.TextMessage, new(JSONSerializer))
	s.RegisterProtocol(msgpackWebsocketProtocol, websocket.BinaryMessage, new(MessagePackSerializer))
	return s
}

// RegisterProtocol registers a serializer that should be used for a given protocol string and payload type.
func (s *WebsocketServer) RegisterProtocol(proto string, payloadType int, serializer Serializer) error {
	log.Println("RegisterProtocol:", proto)
	if payloadType != websocket.TextMessage && payloadType != websocket.BinaryMessage {
		return invalidPayload(payloadType)
	}
	if _, ok := s.protocols[proto]; ok {
		return protocolExists(proto)
	}
	s.protocols[proto] = protocol{payloadType, serializer}
	s.upgrader.Subprotocols = append(s.upgrader.Subprotocols, proto)
	return nil
}

// GetLocalClient returns a client connected to the specified realm
func (s *WebsocketServer) GetLocalClient(realm string) (*Client, error) {
	if peer, err := s.Router.GetLocalPeer(URI(realm)); err != nil {
		return nil, err
	} else {
		c := NewClient(peer)
		go c.Receive()
		return c, nil
	}
}

// ServeHTTP handles a new HTTP connection.
func (s *WebsocketServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	log.Println("WebsocketServer.ServeHTTP", r.Method, r.RequestURI)
	// TODO: subprotocol?
	conn, err := s.upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Println("Error upgrading to websocket connection:", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	s.handleWebsocket(conn)
}

func (s *WebsocketServer) handleWebsocket(conn *websocket.Conn) {
	var serializer Serializer
	var payloadType int
	if proto, ok := s.protocols[conn.Subprotocol()]; ok {
		serializer = proto.serializer
		payloadType = proto.payloadType
	} else {
		// TODO: this will not currently ever be hit because
		//       gorilla/websocket will reject the conncetion
		//       if the subprotocol isn't registered
		switch conn.Subprotocol() {
		case jsonWebsocketProtocol:
			serializer = new(JSONSerializer)
			payloadType = websocket.TextMessage
		case msgpackWebsocketProtocol:
			serializer = new(MessagePackSerializer)
			payloadType = websocket.BinaryMessage
		default:
			conn.Close()
			return
		}
	}

	peer := websocketPeer{
		conn:        conn,
		serializer:  serializer,
		messages:    make(chan Message, 10),
		payloadType: payloadType,
	}
	go func() {
		for {
			// TODO: use conn.NextMessage() and stream
			// TODO: do something different based on binary/text frames
			if _, b, err := conn.ReadMessage(); err != nil {
				conn.Close()
				break
			} else {
				msg, err := serializer.Deserialize(b)
				if err != nil {
					// TODO: handle error
				} else {
					peer.messages <- msg
				}
			}
		}
	}()
	s.Router.Accept(&peer)
}
<commit_msg>Making it easier to configure websocket.Upgrader<commit_after>package turnpike

import (
	"fmt"
	"net/http"

	"github.com/gorilla/websocket"
)

const (
	jsonWebsocketProtocol    = "wamp.2.json"
	msgpackWebsocketProtocol = "wamp.2.msgpack"
)

type invalidPayload byte

func (e invalidPayload) Error() string {
	return fmt.Sprintf("Invalid payloadType: %d", e)
}

type protocolExists string

func (e protocolExists) Error() string {
	return "This protocol has already been registered: " + string(e)
}

type protocol struct {
	payloadType int
	serializer  Serializer
}

// WebsocketServer handles websocket connections.
type WebsocketServer struct {
	Router
	Upgrader *websocket.Upgrader

	protocols map[string]protocol

	// The serializer to use for text frames. Defaults to JSONSerializer.
	TextSerializer Serializer
	// The serializer to use for binary frames. Defaults to JSONSerializer.
	BinarySerializer Serializer
}

// Creates a new WebsocketServer from a map of realms
func NewWebsocketServer(realms map[string]Realm) (*WebsocketServer, error) {
	log.Println("NewWebsocketServer")
	r := NewDefaultRouter()
	for uri, realm := range realms {
		if err := r.RegisterRealm(URI(uri), realm); err != nil {
			return nil, err
		}
	}
	s := newWebsocketServer(r)
	return s, nil
}

// Creates a new WebsocketServer with a single basic realm
func NewBasicWebsocketServer(uri string) *WebsocketServer {
	log.Println("NewBasicWebsocketServer")
	s, _ := NewWebsocketServer(map[string]Realm{uri: {}})
	return s
}

func newWebsocketServer(r Router) *WebsocketServer {
	s := &WebsocketServer{
		Router:    r,
		protocols: make(map[string]protocol),
	}
	s.Upgrader = &websocket.Upgrader{}
	s.RegisterProtocol(jsonWebsocketProtocol, websocket.TextMessage, new(JSONSerializer))
	s.RegisterProtocol(msgpackWebsocketProtocol, websocket.BinaryMessage, new(MessagePackSerializer))
	return s
}

// RegisterProtocol registers a serializer that should be used for a given protocol string and payload type.
func (s *WebsocketServer) RegisterProtocol(proto string, payloadType int, serializer Serializer) error {
	log.Println("RegisterProtocol:", proto)
	if payloadType != websocket.TextMessage && payloadType != websocket.BinaryMessage {
		return invalidPayload(payloadType)
	}
	if _, ok := s.protocols[proto]; ok {
		return protocolExists(proto)
	}
	s.protocols[proto] = protocol{payloadType, serializer}
	s.Upgrader.Subprotocols = append(s.Upgrader.Subprotocols, proto)
	return nil
}

// GetLocalClient returns a client connected to the specified realm
func (s *WebsocketServer) GetLocalClient(realm string) (*Client, error) {
	if peer, err := s.Router.GetLocalPeer(URI(realm)); err != nil {
		return nil, err
	} else {
		c := NewClient(peer)
		go c.Receive()
		return c, nil
	}
}

// ServeHTTP handles a new HTTP connection.
func (s *WebsocketServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	log.Println("WebsocketServer.ServeHTTP", r.Method, r.RequestURI)
	// TODO: subprotocol?
	conn, err := s.Upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Println("Error upgrading to websocket connection:", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	s.handleWebsocket(conn)
}

func (s *WebsocketServer) handleWebsocket(conn *websocket.Conn) {
	var serializer Serializer
	var payloadType int
	if proto, ok := s.protocols[conn.Subprotocol()]; ok {
		serializer = proto.serializer
		payloadType = proto.payloadType
	} else {
		// TODO: this will not currently ever be hit because
		//       gorilla/websocket will reject the conncetion
		//       if the subprotocol isn't registered
		switch conn.Subprotocol() {
		case jsonWebsocketProtocol:
			serializer = new(JSONSerializer)
			payloadType = websocket.TextMessage
		case msgpackWebsocketProtocol:
			serializer = new(MessagePackSerializer)
			payloadType = websocket.BinaryMessage
		default:
			conn.Close()
			return
		}
	}

	peer := websocketPeer{
		conn:        conn,
		serializer:  serializer,
		messages:    make(chan Message, 10),
		payloadType: payloadType,
	}
	go func() {
		for {
			// TODO: use conn.NextMessage() and stream
			// TODO: do something different based on binary/text frames
			if _, b, err := conn.ReadMessage(); err != nil {
				conn.Close()
				break
			} else {
				msg, err := serializer.Deserialize(b)
				if err != nil {
					// TODO: handle error
				} else {
					peer.messages <- msg
				}
			}
		}
	}()
	s.Router.Accept(&peer)
}
<|endoftext|>


<commit_before>package command

import (
	"os"
	"path"
	"strconv"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/storage"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
	"github.com/chrislusf/seaweedfs/weed/storage/super_block"
	"github.com/chrislusf/seaweedfs/weed/storage/types"
)

func init() {
	cmdFix.Run = runFix // break init cycle
}

var cmdFix = &Command{
	UsageLine: "fix -dir=/tmp -volumeId=234",
	Short:     "run weed tool fix on index file if corrupted",
	Long: `Fix runs the SeaweedFS fix command to re-create the index .idx file.

  `,
}

var (
	fixVolumePath       = cmdFix.Flag.String("dir", ".", "data directory to store files")
	fixVolumeCollection = cmdFix.Flag.String("collection", "", "the volume collection name")
	fixVolumeId         = cmdFix.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
)

type VolumeFileScanner4Fix struct {
	version needle.Version
	nm      *needle_map.MemDb
}

func (scanner *VolumeFileScanner4Fix) VisitSuperBlock(superBlock super_block.SuperBlock) error {
	scanner.version = superBlock.Version
	return nil

}
func (scanner *VolumeFileScanner4Fix) ReadNeedleBody() bool {
	return false
}

func (scanner *VolumeFileScanner4Fix) VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error {
	glog.V(2).Infof("key %d offset %d size %d disk_size %d gzip %v", n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsGzipped())
	if n.Size > 0 && n.Size != types.TombstoneFileSize {
		pe := scanner.nm.Set(n.Id, types.ToOffset(offset), n.Size)
		glog.V(2).Infof("saved %d with error %v", n.Size, pe)
	} else {
		glog.V(2).Infof("skipping deleted file ...")
		return scanner.nm.Delete(n.Id)
	}
	return nil
}

func runFix(cmd *Command, args []string) bool {

	if *fixVolumeId == -1 {
		return false
	}

	baseFileName := strconv.Itoa(*fixVolumeId)
	if *fixVolumeCollection != "" {
		baseFileName = *fixVolumeCollection + "_" + baseFileName
	}
	indexFileName := path.Join(*fixVolumePath, baseFileName+".idx")

	nm := needle_map.NewMemDb()

	vid := needle.VolumeId(*fixVolumeId)
	scanner := &VolumeFileScanner4Fix{
		nm: nm,
	}

	if err = storage.ScanVolumeFile(*fixVolumePath, *fixVolumeCollection, vid, storage.NeedleMapInMemory, scanner); err != nil {
		glog.Fatalf("scan .dat File: %v", err)
		os.Remove(indexFileName)
	}

	if err := nm.SaveToIdx(indexFileName); err != nil {
		glog.Fatalf("save to .idx File: %v", err)
		os.Remove(indexFileName)
	}

	return true
}
<commit_msg>fix compilation error<commit_after>package command

import (
	"os"
	"path"
	"strconv"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/storage"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
	"github.com/chrislusf/seaweedfs/weed/storage/super_block"
	"github.com/chrislusf/seaweedfs/weed/storage/types"
)

func init() {
	cmdFix.Run = runFix // break init cycle
}

var cmdFix = &Command{
	UsageLine: "fix -dir=/tmp -volumeId=234",
	Short:     "run weed tool fix on index file if corrupted",
	Long: `Fix runs the SeaweedFS fix command to re-create the index .idx file.

  `,
}

var (
	fixVolumePath       = cmdFix.Flag.String("dir", ".", "data directory to store files")
	fixVolumeCollection = cmdFix.Flag.String("collection", "", "the volume collection name")
	fixVolumeId         = cmdFix.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
)

type VolumeFileScanner4Fix struct {
	version needle.Version
	nm      *needle_map.MemDb
}

func (scanner *VolumeFileScanner4Fix) VisitSuperBlock(superBlock super_block.SuperBlock) error {
	scanner.version = superBlock.Version
	return nil

}
func (scanner *VolumeFileScanner4Fix) ReadNeedleBody() bool {
	return false
}

func (scanner *VolumeFileScanner4Fix) VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error {
	glog.V(2).Infof("key %d offset %d size %d disk_size %d gzip %v", n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsGzipped())
	if n.Size > 0 && n.Size != types.TombstoneFileSize {
		pe := scanner.nm.Set(n.Id, types.ToOffset(offset), n.Size)
		glog.V(2).Infof("saved %d with error %v", n.Size, pe)
	} else {
		glog.V(2).Infof("skipping deleted file ...")
		return scanner.nm.Delete(n.Id)
	}
	return nil
}

func runFix(cmd *Command, args []string) bool {

	if *fixVolumeId == -1 {
		return false
	}

	baseFileName := strconv.Itoa(*fixVolumeId)
	if *fixVolumeCollection != "" {
		baseFileName = *fixVolumeCollection + "_" + baseFileName
	}
	indexFileName := path.Join(*fixVolumePath, baseFileName+".idx")

	nm := needle_map.NewMemDb()

	vid := needle.VolumeId(*fixVolumeId)
	scanner := &VolumeFileScanner4Fix{
		nm: nm,
	}

	if err := storage.ScanVolumeFile(*fixVolumePath, *fixVolumeCollection, vid, storage.NeedleMapInMemory, scanner); err != nil {
		glog.Fatalf("scan .dat File: %v", err)
		os.Remove(indexFileName)
	}

	if err := nm.SaveToIdx(indexFileName); err != nil {
		glog.Fatalf("save to .idx File: %v", err)
		os.Remove(indexFileName)
	}

	return true
}
<|endoftext|>


<commit_before>package analytics

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/algolia/algoliasearch-client-go/algolia/search"
)

// ABTest presents an AB test input as sent to the Analytics API when creating a
// new AB test.
type ABTest struct {
	Name     string
	Variants []Variant
	EndAt    time.Time
}

const ISO8601 = "2006-01-02T15:04:05Z"

func (abTest ABTest) MarshalJSON() ([]byte, error) {
	jsonVariants, err := json.Marshal(abTest.Variants)
	if err != nil {
		return nil, fmt.Errorf("cannot marshal variants: %s", err)
	}
	jsonAbTest := fmt.Sprintf(
		`{"name":"%s","variants":%s,"endAt":"%s"}`,
		abTest.Name,
		jsonVariants,
		abTest.EndAt.In(time.UTC).Format(ISO8601),
	)
	return []byte(jsonAbTest), nil
}

// Variant presents a Variant input as sent to the Analytics API when creating a
// new AB test.
type Variant struct {
	Index                  string             `json:"index"`
	TrafficPercentage      int                `json:"trafficPercentage"`
	Description            string             `json:"description,omitempty"`
	CustomSearchParameters search.QueryParams `json:"customSearchParameters,omitempty"`
}
<commit_msg>changed: use pointer type for ABTest.Variant.CustomSearchParameters to prevent serialization when the field is empty<commit_after>package analytics

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/algolia/algoliasearch-client-go/algolia/search"
)

// ABTest presents an AB test input as sent to the Analytics API when creating a
// new AB test.
type ABTest struct {
	Name     string
	Variants []Variant
	EndAt    time.Time
}

const ISO8601 = "2006-01-02T15:04:05Z"

func (abTest ABTest) MarshalJSON() ([]byte, error) {
	jsonVariants, err := json.Marshal(abTest.Variants)
	if err != nil {
		return nil, fmt.Errorf("cannot marshal variants: %s", err)
	}
	jsonAbTest := fmt.Sprintf(
		`{"name":"%s","variants":%s,"endAt":"%s"}`,
		abTest.Name,
		jsonVariants,
		abTest.EndAt.In(time.UTC).Format(ISO8601),
	)
	return []byte(jsonAbTest), nil
}

// Variant presents a Variant input as sent to the Analytics API when creating a
// new AB test.
type Variant struct {
	Index                  string              `json:"index"`
	TrafficPercentage      int                 `json:"trafficPercentage"`
	Description            string              `json:"description,omitempty"`
	CustomSearchParameters *search.QueryParams `json:"customSearchParameters,omitempty"`
}
<|endoftext|>