<commit_before>// Copyright 2014 Google Inc. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to writing, software distributed
// under the License is distributed on a "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
package main

import (
	"testing"

	"sourcegraph.com/sourcegraph/go-selenium"
)

var caps selenium.Capabilities = selenium.Capabilities(map[string]interface{}{"browserName": "chrome"})
var executorURL = "http://127.0.0.1:4444/wd/hub"

func TestTodo(t *testing.T) {
	wd, _ := selenium.NewRemote(caps, executorURL)
	wdt := wd.T(t)
	wdt.Get("http://127.0.0.1:8080")

	title := wdt.Title()
	if title != "TODO" {
		t.Errorf("Wanted title TODO, got %s", title)
	}
}
<commit_msg>Update todo_test.go<commit_after>// Copyright 2014 Google Inc. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to writing, software distributed
// under the License is distributed on a "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
package main

import (
	"testing"

	"sourcegraph.com/sourcegraph/go-selenium"
)

var caps selenium.Capabilities = selenium.Capabilities(map[string]interface{}{"browserName": "chrome"})
var executorURL = "http://selenium:4444/wd/hub"

func TestTodo(t *testing.T) {
	wd, _ := selenium.NewRemote(caps, executorURL)
	wdt := wd.T(t)
	wdt.Get("http://127.0.0.1:8080")

	title := wdt.Title()
	if title != "TODO" {
		t.Errorf("Wanted title TODO, got %s", title)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/gorilla/context"
	"github.com/gorilla/mux"
	"github.com/tejpbit/talarlista/backend/backend"
	"github.com/urfave/negroni"
	"log"
	"net/http"
)

var hub backend.Hub

//func userWithSession(w http.ResponseWriter, req *http.Request, next http.HandlerFunc) {
//
//	session, err := hub.Get(req, SESSION_KEY)
//	if err != nil {
//		http.Error(w, err.Error(), http.StatusInternalServerError)
//		return
//	}
//
//	log.Printf("Requested url %s", req.URL)
//
//	if session.IsNew {
//		var id = uuid.New()
//		session.Values[UUID_KEY] = id.String()
//
//		log.Printf("New user id: %v\n", id)
//
//		state.addUser(&backend.User{"", false, id})
//		log.Printf("State after new user: %v", state.Users)
//
//		session.Options = &sessions.Options{
//			MaxAge:   86400,
//			HttpOnly: true,
//		}
//
//		err = session.Save(req, w)
//
//		if err != nil {
//			log.Printf("Error when saving session to storage: %v\n", err)
//		}
//	}
//
//	next(w, req)
//}

func main() {
	log.SetFlags(log.Lshortfile)
	hub = backend.CreateHub()
	hub.Start()

	n := negroni.Classic()

	r := mux.NewRouter()
	r.HandleFunc("/ws", hub.ServeWs)
	r.HandleFunc("/", func(rw http.ResponseWriter, r *http.Request) {
		log.Printf("helo")
		rw.Write([]byte("Helohelo /"))
	})
	n.UseHandler(r)
	handler := context.ClearHandler(n)

	log.Print("About to listen on 3001. Go to http://127.0.0.1:3001/")
	err := http.ListenAndServe(":3001", handler)
	log.Fatal(err)
}
<commit_msg>Remove old middleware code<commit_after>package main

import (
	"github.com/gorilla/context"
	"github.com/gorilla/mux"
	"github.com/tejpbit/talarlista/backend/backend"
	"github.com/urfave/negroni"
	"log"
	"net/http"
)

var hub backend.Hub

func main() {
	log.SetFlags(log.Lshortfile)
	hub = backend.CreateHub()
	hub.Start()

	n := negroni.Classic()

	r := mux.NewRouter()
	r.HandleFunc("/ws", hub.ServeWs)
	r.HandleFunc("/", func(rw http.ResponseWriter, r *http.Request) {
		log.Printf("helo")
		rw.Write([]byte("Helohelo /"))
	})
	n.UseHandler(r)
	handler := context.ClearHandler(n)

	log.Print("About to listen on 3001. Go to http://127.0.0.1:3001/")
	err := http.ListenAndServe(":3001", handler)
	log.Fatal(err)
}
<|endoftext|>


<commit_before>package main

import (
	"compress/gzip"
	"encoding/json"
	"errors"
	"io"
	"log"
	"math"
	"net/http"
	"net/url"
	"os"
	"sort"
	"strconv"
	"strings"
	"time"
)

var bsn bikeShareNetwork

func main() {
	log.Println("starting seagull")
	var err error
	bsn, err = getSeedData()
	if err != nil {
		log.Fatal(err)
	}

	http.Handle("/", http.FileServer(http.Dir("../frontend/www")))
	http.HandleFunc("/api/networks", GzipFunc(listNetworksHandler))

	if err := http.ListenAndServe(":8080", nil); err != nil {
		log.Fatal(err)
	}

}

func (l Location) distance(lat, lng float64) float64 {
	R := 6371e3 // radius

	φ1 := (math.Pi * lat) / 180
	φ2 := (math.Pi * lng) / 180

	Δφ := (l.Latitude - lat) * math.Pi / 180
	Δλ := (l.Longitude - lng) * math.Pi / 180

	a := math.Sin(Δφ/2)*math.Sin(Δφ/2) +
		math.Cos(φ1)*math.Cos(φ2)*
			math.Sin(Δλ/2)*math.Sin(Δλ/2)
	return R * 2 * math.Atan2(math.Sqrt(a), math.Sqrt(1-a))
}

var (
	ErrRangeQuery    = errors.New("failed to parse range query")
	ErrLocationQuery = errors.New("failed to parse location query")
)

func parseNetworksQuery(u url.Values) (float64, float64, float64, error) {
	_, latok := u["lat"]
	_, lngok := u["lng"]
	_, rngok := u["rng"]
	if !latok || !lngok {
		return 0, 0, 0, ErrLocationQuery
	}

	lat, err := strconv.ParseFloat(u["lat"][0], 10)
	if err != nil {
		return 0, 0, 0, err
	}

	lng, err := strconv.ParseFloat(u["lng"][0], 10)
	if err != nil {
		return lat, 0, 0, err
	}
	rng := 100000.0 // set default range
	if rngok {
		rng, err = strconv.ParseFloat(u["rng"][0], 10)
		if err != nil {
			return lat, lng, 100000, ErrRangeQuery
		}
	}

	return lat, lng, rng, nil
}

func listNetworksHandler(w http.ResponseWriter, r *http.Request) {
	log.Println("listNetworks", r.RemoteAddr, r.RequestURI)
	defer timeLog(time.Now(), "listNetworks")

	type Shortnet struct {
		ID       string `json:"id"`
		Name     string `json:"name"`
		Location `json:"location"`
	}

	type Response struct {
		Networks []Shortnet `json:"networks"`
	}

	var networks Response
	for _, v := range bsn.Networks {
		short := Shortnet{
			ID:       v.ID,
			Name:     v.Name,
			Location: v.Location,
		}
		networks.Networks = append(networks.Networks, short)
	}

	q := r.URL.Query()
	lat, lng, rng, err := parseNetworksQuery(q)

	if err != nil {
		if err != ErrRangeQuery {
			log.Println("oops", err)
			enc := json.NewEncoder(w)
			if err := enc.Encode(networks); err != nil {
				log.Println(err)
			}
			return
		}
		log.Println(err)
	}
	for k, v := range networks.Networks {
		networks.Networks[k].Location.Distance = int(v.Location.distance(lat, lng))
	}

	sort.Slice(networks.Networks, func(i, j int) bool {
		return networks.Networks[i].Location.Distance < networks.Networks[j].Location.Distance
	})

	var within Response
	for _, v := range networks.Networks {
		if float64(v.Location.Distance) < rng {
			within.Networks = append(within.Networks, v)
		} else {
			// since it is a sorted list we can just exit loop
			break
		}
	}

	enc := json.NewEncoder(w)
	if err := enc.Encode(within); err != nil {
		log.Println(err)
	}

}

func getSeedData() (bikeShareNetwork, error) {
	defer timeLog(time.Now(), "getSeedData")
	// check if we have it first
	if _, err := os.Stat("bsn.json"); os.IsNotExist(err) {
		const bikeShareAPI = "http://api.citybik.es/v2/networks"

		var bsn bikeShareNetwork
		resp, err := http.Get(bikeShareAPI)
		if err != nil {
			return bsn, err
		}
		defer resp.Body.Close()

		dec := json.NewDecoder(resp.Body)
		if err := dec.Decode(&bsn); err != nil {
			return bsn, err
		}
		networks := len(bsn.Networks)
		for k, v := range bsn.Networks {
			log.Printf("(%3d of %3d) detail of %s, \n", k+1, networks, v.ID)
			var detail networkDetail
			resp, err := http.Get(bikeShareAPI + "/" + v.ID)
			if err != nil {
				log.Printf("error reading network detail for %s: %v", v.ID, err)
				break
			}
			defer resp.Body.Close()
			dec := json.NewDecoder(resp.Body)
			if err := dec.Decode(&detail); err != nil {
				log.Printf("error decoding detail json for %s: %v", v.ID, err)
				break
			}
			bsn.Networks[k] = detail.Detail
		}

		out, err := os.Create("bsn.json")
		if err != nil {
			log.Fatal(err)
		}
		defer out.Close()

		enc := json.NewEncoder(out)
		err = enc.Encode(bsn)
		if err != nil {
			log.Println(err)
		}

	}

	var bsn bikeShareNetwork

	in, err := os.Open("bsn.json")
	if err != nil {
		return bsn, err
	}

	dec := json.NewDecoder(in)
	if err := dec.Decode(&bsn); err != nil {
		return bsn, err
	}
	// get data from file

	return bsn, nil

}

type bikeShareNetwork struct {
	Networks []Network `json:"networks"`
}

type Network struct {
	Company  []string `json:"company"`
	Href     string   `json:"href"`
	ID       string   `json:"id"`
	Location `json:"location"`
	Name     string    `json:"name"`
	Stations []Station `json:"stations,omitempty"`
}

type networkDetail struct {
	Detail Network `json:"network"`
}

type Station struct {
	EmptySlots int       `json:"empty_slots"`
	FreeBikes  int       `json:"free_bikes"`
	ID         string    `json:"id"`
	Latitude   float64   `json:"latitude"`
	Longitude  float64   `json:"longitude"`
	Name       string    ` json:"name"`
	Timestamp  time.Time `json:"timestamp"` // look up local offset at location
}

type Location struct {
	City      string  `json:"city"`
	Country   string  `json:"country"`
	Latitude  float64 `json:"latitude"`
	Longitude float64 `json:"longitude"`
	Distance  int     `json:"distance,omitempty"`
}

func (n *Network) UnmarshalJSON(data []byte) error {
	// Need too handle the one case company strings vs []string
	type ServerNetworks Network
	aux := &struct {
		Company interface{} `json:"company"`
		*ServerNetworks
	}{
		ServerNetworks: (*ServerNetworks)(n),
	}
	err := json.Unmarshal(data, &aux)
	if err != nil {
		return err
	}

	switch aux.Company.(type) {
	case string:
		c, ok := aux.Company.(string)
		if ok {
			n.Company = append(n.Company, c)
		}
	case []interface{}:
		cc, ok := aux.Company.([]interface{})
		if ok {
			for _, vv := range cc {
				switch vv.(type) {
				case string:
					c, ok := vv.(string)
					if ok {
						n.Company = append(n.Company, c)
					} else {
						log.Println("failed conversion to string")
					}
				}
			}
		}
	}
	return nil
}

type gzipResponseWriter struct {
	io.Writer
	http.ResponseWriter
}

func GzipFunc(fn http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
			fn(w, r)
			w.Header().Set("Pragma", "no-cache")
			w.Header().Set("Content-Type", "application/json")
			return
		}
		w.Header().Set("Content-Encoding", "gzip")
		w.Header().Set("Pragma", "no-cache")
		w.Header().Set("Content-Type", "application/json")
		gz := gzip.NewWriter(w)
		defer gz.Close()
		gzr := gzipResponseWriter{Writer: gz, ResponseWriter: w}
		fn(gzr, r)
	}
}

func (w gzipResponseWriter) Write(b []byte) (int, error) {
	return w.Writer.Write(b)
}

func timeLog(start time.Time, name string) {
	taken := time.Since(start)
	log.Printf("%s took %s", name, taken)
}
<commit_msg>build flag for sort.Slice<commit_after>// +build go1.8

package main

import (
	"compress/gzip"
	"encoding/json"
	"errors"
	"io"
	"log"
	"math"
	"net/http"
	"net/url"
	"os"
	"sort"
	"strconv"
	"strings"
	"time"
)

var bsn bikeShareNetwork

func main() {
	log.Println("starting seagull")
	var err error
	bsn, err = getSeedData()
	if err != nil {
		log.Fatal(err)
	}

	http.Handle("/", http.FileServer(http.Dir("../frontend/www")))
	http.HandleFunc("/api/networks", GzipFunc(listNetworksHandler))

	if err := http.ListenAndServe(":8080", nil); err != nil {
		log.Fatal(err)
	}

}

func (l Location) distance(lat, lng float64) float64 {
	R := 6371e3 // radius

	φ1 := (math.Pi * lat) / 180
	φ2 := (math.Pi * lng) / 180

	Δφ := (l.Latitude - lat) * math.Pi / 180
	Δλ := (l.Longitude - lng) * math.Pi / 180

	a := math.Sin(Δφ/2)*math.Sin(Δφ/2) +
		math.Cos(φ1)*math.Cos(φ2)*
			math.Sin(Δλ/2)*math.Sin(Δλ/2)
	return R * 2 * math.Atan2(math.Sqrt(a), math.Sqrt(1-a))
}

var (
	ErrRangeQuery    = errors.New("failed to parse range query")
	ErrLocationQuery = errors.New("failed to parse location query")
)

func parseNetworksQuery(u url.Values) (float64, float64, float64, error) {
	_, latok := u["lat"]
	_, lngok := u["lng"]
	_, rngok := u["rng"]
	if !latok || !lngok {
		return 0, 0, 0, ErrLocationQuery
	}

	lat, err := strconv.ParseFloat(u["lat"][0], 10)
	if err != nil {
		return 0, 0, 0, err
	}

	lng, err := strconv.ParseFloat(u["lng"][0], 10)
	if err != nil {
		return lat, 0, 0, err
	}
	rng := 100000.0 // set default range
	if rngok {
		rng, err = strconv.ParseFloat(u["rng"][0], 10)
		if err != nil {
			return lat, lng, 100000, ErrRangeQuery
		}
	}

	return lat, lng, rng, nil
}

func listNetworksHandler(w http.ResponseWriter, r *http.Request) {
	log.Println("listNetworks", r.RemoteAddr, r.RequestURI)
	defer timeLog(time.Now(), "listNetworks")

	type Shortnet struct {
		ID       string `json:"id"`
		Name     string `json:"name"`
		Location `json:"location"`
	}

	type Response struct {
		Networks []Shortnet `json:"networks"`
	}

	var networks Response
	for _, v := range bsn.Networks {
		short := Shortnet{
			ID:       v.ID,
			Name:     v.Name,
			Location: v.Location,
		}
		networks.Networks = append(networks.Networks, short)
	}

	q := r.URL.Query()
	lat, lng, rng, err := parseNetworksQuery(q)

	if err != nil {
		if err != ErrRangeQuery {
			log.Println("oops", err)
			enc := json.NewEncoder(w)
			if err := enc.Encode(networks); err != nil {
				log.Println(err)
			}
			return
		}
		log.Println(err)
	}
	for k, v := range networks.Networks {
		networks.Networks[k].Location.Distance = int(v.Location.distance(lat, lng))
	}

	sort.Slice(networks.Networks, func(i, j int) bool {
		return networks.Networks[i].Location.Distance < networks.Networks[j].Location.Distance
	})

	var within Response
	for _, v := range networks.Networks {
		if float64(v.Location.Distance) < rng {
			within.Networks = append(within.Networks, v)
		} else {
			// since it is a sorted list we can just exit loop
			break
		}
	}

	enc := json.NewEncoder(w)
	if err := enc.Encode(within); err != nil {
		log.Println(err)
	}

}

func getSeedData() (bikeShareNetwork, error) {
	defer timeLog(time.Now(), "getSeedData")
	// check if we have it first
	if _, err := os.Stat("bsn.json"); os.IsNotExist(err) {
		const bikeShareAPI = "http://api.citybik.es/v2/networks"

		var bsn bikeShareNetwork
		resp, err := http.Get(bikeShareAPI)
		if err != nil {
			return bsn, err
		}
		defer resp.Body.Close()

		dec := json.NewDecoder(resp.Body)
		if err := dec.Decode(&bsn); err != nil {
			return bsn, err
		}
		networks := len(bsn.Networks)
		for k, v := range bsn.Networks {
			log.Printf("(%3d of %3d) detail of %s, \n", k+1, networks, v.ID)
			var detail networkDetail
			resp, err := http.Get(bikeShareAPI + "/" + v.ID)
			if err != nil {
				log.Printf("error reading network detail for %s: %v", v.ID, err)
				break
			}
			defer resp.Body.Close()
			dec := json.NewDecoder(resp.Body)
			if err := dec.Decode(&detail); err != nil {
				log.Printf("error decoding detail json for %s: %v", v.ID, err)
				break
			}
			bsn.Networks[k] = detail.Detail
		}

		out, err := os.Create("bsn.json")
		if err != nil {
			log.Fatal(err)
		}
		defer out.Close()

		enc := json.NewEncoder(out)
		err = enc.Encode(bsn)
		if err != nil {
			log.Println(err)
		}

	}

	var bsn bikeShareNetwork

	in, err := os.Open("bsn.json")
	if err != nil {
		return bsn, err
	}

	dec := json.NewDecoder(in)
	if err := dec.Decode(&bsn); err != nil {
		return bsn, err
	}
	// get data from file

	return bsn, nil

}

type bikeShareNetwork struct {
	Networks []Network `json:"networks"`
}

type Network struct {
	Company  []string `json:"company"`
	Href     string   `json:"href"`
	ID       string   `json:"id"`
	Location `json:"location"`
	Name     string    `json:"name"`
	Stations []Station `json:"stations,omitempty"`
}

type networkDetail struct {
	Detail Network `json:"network"`
}

type Station struct {
	EmptySlots int       `json:"empty_slots"`
	FreeBikes  int       `json:"free_bikes"`
	ID         string    `json:"id"`
	Latitude   float64   `json:"latitude"`
	Longitude  float64   `json:"longitude"`
	Name       string    ` json:"name"`
	Timestamp  time.Time `json:"timestamp"` // look up local offset at location
}

type Location struct {
	City      string  `json:"city"`
	Country   string  `json:"country"`
	Latitude  float64 `json:"latitude"`
	Longitude float64 `json:"longitude"`
	Distance  int     `json:"distance,omitempty"`
}

func (n *Network) UnmarshalJSON(data []byte) error {
	// Need too handle the one case company strings vs []string
	type ServerNetworks Network
	aux := &struct {
		Company interface{} `json:"company"`
		*ServerNetworks
	}{
		ServerNetworks: (*ServerNetworks)(n),
	}
	err := json.Unmarshal(data, &aux)
	if err != nil {
		return err
	}

	switch aux.Company.(type) {
	case string:
		c, ok := aux.Company.(string)
		if ok {
			n.Company = append(n.Company, c)
		}
	case []interface{}:
		cc, ok := aux.Company.([]interface{})
		if ok {
			for _, vv := range cc {
				switch vv.(type) {
				case string:
					c, ok := vv.(string)
					if ok {
						n.Company = append(n.Company, c)
					} else {
						log.Println("failed conversion to string")
					}
				}
			}
		}
	}
	return nil
}

type gzipResponseWriter struct {
	io.Writer
	http.ResponseWriter
}

func GzipFunc(fn http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
			fn(w, r)
			w.Header().Set("Pragma", "no-cache")
			w.Header().Set("Content-Type", "application/json")
			return
		}
		w.Header().Set("Content-Encoding", "gzip")
		w.Header().Set("Pragma", "no-cache")
		w.Header().Set("Content-Type", "application/json")
		gz := gzip.NewWriter(w)
		defer gz.Close()
		gzr := gzipResponseWriter{Writer: gz, ResponseWriter: w}
		fn(gzr, r)
	}
}

func (w gzipResponseWriter) Write(b []byte) (int, error) {
	return w.Writer.Write(b)
}

func timeLog(start time.Time, name string) {
	taken := time.Since(start)
	log.Printf("%s took %s", name, taken)
}
<|endoftext|>


<commit_before>// Copyright 2015 ISRG.  All rights reserved
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package publisher

import (
	"bytes"
	"crypto/ecdsa"
	"crypto/sha256"
	"crypto/x509"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/letsencrypt/boulder/core"
	blog "github.com/letsencrypt/boulder/log"
	"github.com/letsencrypt/boulder/sa"
)

type LogDescription struct {
	ID        string
	URI       string
	PublicKey *ecdsa.PublicKey
}

type rawLogDescription struct {
	URI       string `json:"uri"`
	PublicKey string `json:"key"`
}

func (logDesc *LogDescription) UnmarshalJSON(data []byte) error {
	var rawLogDesc rawLogDescription
	if err := json.Unmarshal(data, &rawLogDesc); err != nil {
		return fmt.Errorf("Failed to unmarshal log description, %s", err)
	}
	logDesc.URI = rawLogDesc.URI
	// Load Key
	pkBytes, err := base64.StdEncoding.DecodeString(rawLogDesc.PublicKey)
	if err != nil {
		return fmt.Errorf("")
	}
	pk, err := x509.ParsePKIXPublicKey(pkBytes)
	if err != nil {
		return fmt.Errorf("")
	}
	ecdsaKey, ok := pk.(*ecdsa.PublicKey)
	if !ok {
		return fmt.Errorf("Failed to unmarshal log description for %s, unsupported public key type", logDesc.URI)
	}
	logDesc.PublicKey = ecdsaKey

	// Generate key hash for log ID
	pkHash := sha256.Sum256(pkBytes)
	logDesc.ID = base64.StdEncoding.EncodeToString(pkHash[:])
	if len(logDesc.ID) != 44 {
		return fmt.Errorf("Invalid log ID length [%d]", len(logDesc.ID))
	}

	return nil
}

// CTConfig defines the JSON configuration file schema
type CTConfig struct {
	Logs                       []LogDescription `json:"logs"`
	SubmissionRetries          int              `json:"submissionRetries"`
	SubmissionBackoffString    string           `json:"submissionBackoff"`
	IntermediateBundleFilename string           `json:"intermediateBundleFilename"`
}

type ctSubmissionRequest struct {
	Chain []string `json:"chain"`
}

const (
	sctVersion       = 0
	sctSigType       = 0
	sctX509EntryType = 0
)

// PublisherImpl defines a Publisher
type PublisherImpl struct {
	log               *blog.AuditLogger
	client            *http.Client
	submissionBackoff time.Duration
	submissionRetries int
	issuerBundle      []string
	ctLogs            []LogDescription

	SA core.StorageAuthority
}

// NewPublisherImpl creates a Publisher that will submit certificates
// to any CT logs configured in CTConfig
func NewPublisherImpl(ctConfig CTConfig) (pub PublisherImpl, err error) {
	logger := blog.GetAuditLogger()
	logger.Notice("Publisher Authority Starting")

	if ctConfig.IntermediateBundleFilename == "" {
		err = fmt.Errorf("No CT submission bundle provided")
		return
	}
	bundle, err := core.LoadCertBundle(ctConfig.IntermediateBundleFilename)
	if err != nil {
		return
	}
	for _, cert := range bundle {
		pub.issuerBundle = append(pub.issuerBundle, base64.StdEncoding.EncodeToString(cert.Raw))
	}
	ctBackoff, err := time.ParseDuration(ctConfig.SubmissionBackoffString)
	if err != nil {
		return
	}
	for _, log := range ctConfig.Logs {
		if !strings.HasPrefix(log.URI, "https://") && !strings.HasPrefix(log.URI, "http://") {
			err = fmt.Errorf("Log URI [%s] is not absolute", log.URI)
			return
		}
	}

	pub.log = logger
	pub.client = &http.Client{}
	pub.submissionBackoff = ctBackoff
	pub.submissionRetries = ctConfig.SubmissionRetries
	pub.ctLogs = ctConfig.Logs

	return
}

func (pub *PublisherImpl) submitToCTLog(serial string, jsonSubmission []byte, log LogDescription) error {
	done := false
	var sct core.SignedCertificateTimestamp
	backoff := pub.submissionBackoff
	var retries int
	for retries = 0; retries <= pub.submissionRetries; retries++ {
		if retries > 0 {
			time.Sleep(backoff)
		}
		resp, err := postJSON(pub.client, fmt.Sprintf("%s%s", log.URI, "/ct/v1/add-chain"), jsonSubmission, &sct)
		if err != nil {
			// Retry the request, log the error
			pub.log.Warning(fmt.Sprintf("Error POSTing JSON to CT log submission endpoint [%s]: %s", log.URI, err))
			backoff = pub.submissionBackoff
			continue
		} else if resp.StatusCode == http.StatusRequestTimeout || resp.StatusCode == http.StatusServiceUnavailable {
			// Retry the request after either the configured backoff period or the period
			// specified by the Retry-After header
			backoff = pub.submissionBackoff
			if seconds, err := strconv.Atoi(resp.Header.Get("Retry-After")); err == nil {
				backoff = time.Second * time.Duration(seconds)
			}
			continue
		} else if resp.StatusCode != http.StatusOK {
			// If we hit an otherwise unexpected status code break the loop and return
			// an error immediately
			return fmt.Errorf("Unexpected status code returned from CT log: %d", resp.StatusCode)
		}
		done = true
		break
	}

	if !done {
		err := fmt.Errorf(
			"Unable to submit certificate to CT log [Serial: %s, Log URI: %s, Retries: %d]",
			serial,
			log.URI,
			retries,
		)
		return err
	}

	if err := sct.CheckSignature(); err != nil {
		return err
	}

	pub.log.Debug(fmt.Sprintf(
		"Submitted certificate to CT log [Serial: %s, Log URI: %s, Retries: %d, Signature: %x]",
		serial,
		log.URI,
		retries, sct.Signature,
	))

	// Set certificate serial and add SCT to DB
	sct.CertificateSerial = serial
	err := pub.SA.AddSCTReceipt(sct)
	if err != nil {
		if _, ok := err.(sa.ErrDuplicateReceipt); ok {
			pub.log.Warning(fmt.Sprintf(
				"SCT receipt for [Serial: %s, Log URI: %s] already exists in database",
				serial,
				log.URI,
			))
			return nil
		} else if err != nil {
			err = fmt.Errorf(
				"Error adding SCT receipt for [%s to %s]: %s",
				sct.CertificateSerial,
				log.URI,
				err,
			)
			return err
		}
	}
	pub.log.Info(fmt.Sprintf(
		"Stored SCT receipt from CT log submission [Serial: %s, Log URI: %s]",
		serial,
		log.URI,
	))
	return nil
}

// SubmitToCT will submit the certificate represented by certDER to any CT
// logs configured in pub.CT.Logs
func (pub *PublisherImpl) SubmitToCT(der []byte) error {
	cert, err := x509.ParseCertificate(der)
	if err != nil {
		pub.log.Err(fmt.Sprintf("Unable to parse certificate, %s", err))
		return err
	}

	submission := ctSubmissionRequest{Chain: []string{base64.StdEncoding.EncodeToString(cert.Raw)}}
	// Add all intermediate certificates needed for submission
	submission.Chain = append(submission.Chain, pub.issuerBundle...)
	jsonSubmission, err := json.Marshal(submission)
	if err != nil {
		pub.log.Err(fmt.Sprintf("Unable to marshal CT submission, %s", err))
		return err
	}

	for _, ctLog := range pub.ctLogs {
		err = pub.submitToCTLog(core.SerialToString(cert.SerialNumber), jsonSubmission, ctLog)
		if err != nil {
			// AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3
			pub.log.AuditErr(err)
			continue
		}
	}

	return nil
}

func postJSON(client *http.Client, uri string, data []byte, respObj interface{}) (*http.Response, error) {
	req, err := http.NewRequest("POST", uri, bytes.NewBuffer(data))
	if err != nil {
		return nil, fmt.Errorf("Creating request failed, %s", err)
	}
	req.Header.Set("Keep-Alive", "timeout=15, max=100")
	req.Header.Set("Content-Type", "application/json")

	resp, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("Request failed, %s", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode == http.StatusOK {
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, fmt.Errorf("Failed to read response body, %s", err)
		}

		err = json.Unmarshal(body, respObj)
		if err != nil {
			return nil, fmt.Errorf("Failed to unmarshal SCT receipt, %s", err)
		}
	}
	return resp, nil
}
<commit_msg>Add missing ct publisher errors<commit_after>// Copyright 2015 ISRG.  All rights reserved
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package publisher

import (
	"bytes"
	"crypto/ecdsa"
	"crypto/sha256"
	"crypto/x509"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/letsencrypt/boulder/core"
	blog "github.com/letsencrypt/boulder/log"
	"github.com/letsencrypt/boulder/sa"
)

type LogDescription struct {
	ID        string
	URI       string
	PublicKey *ecdsa.PublicKey
}

type rawLogDescription struct {
	URI       string `json:"uri"`
	PublicKey string `json:"key"`
}

func (logDesc *LogDescription) UnmarshalJSON(data []byte) error {
	var rawLogDesc rawLogDescription
	if err := json.Unmarshal(data, &rawLogDesc); err != nil {
		return fmt.Errorf("Failed to unmarshal log description, %s", err)
	}
	logDesc.URI = rawLogDesc.URI
	// Load Key
	pkBytes, err := base64.StdEncoding.DecodeString(rawLogDesc.PublicKey)
	if err != nil {
		return fmt.Errorf("Failed to decode base64 log public key")
	}
	pk, err := x509.ParsePKIXPublicKey(pkBytes)
	if err != nil {
		return fmt.Errorf("Failed to parse log public key")
	}
	ecdsaKey, ok := pk.(*ecdsa.PublicKey)
	if !ok {
		return fmt.Errorf("Failed to unmarshal log description for %s, unsupported public key type", logDesc.URI)
	}
	logDesc.PublicKey = ecdsaKey

	// Generate key hash for log ID
	pkHash := sha256.Sum256(pkBytes)
	logDesc.ID = base64.StdEncoding.EncodeToString(pkHash[:])
	if len(logDesc.ID) != 44 {
		return fmt.Errorf("Invalid log ID length [%d]", len(logDesc.ID))
	}

	return nil
}

// CTConfig defines the JSON configuration file schema
type CTConfig struct {
	Logs                       []LogDescription `json:"logs"`
	SubmissionRetries          int              `json:"submissionRetries"`
	SubmissionBackoffString    string           `json:"submissionBackoff"`
	IntermediateBundleFilename string           `json:"intermediateBundleFilename"`
}

type ctSubmissionRequest struct {
	Chain []string `json:"chain"`
}

const (
	sctVersion       = 0
	sctSigType       = 0
	sctX509EntryType = 0
)

// PublisherImpl defines a Publisher
type PublisherImpl struct {
	log               *blog.AuditLogger
	client            *http.Client
	submissionBackoff time.Duration
	submissionRetries int
	issuerBundle      []string
	ctLogs            []LogDescription

	SA core.StorageAuthority
}

// NewPublisherImpl creates a Publisher that will submit certificates
// to any CT logs configured in CTConfig
func NewPublisherImpl(ctConfig CTConfig) (pub PublisherImpl, err error) {
	logger := blog.GetAuditLogger()
	logger.Notice("Publisher Authority Starting")

	if ctConfig.IntermediateBundleFilename == "" {
		err = fmt.Errorf("No CT submission bundle provided")
		return
	}
	bundle, err := core.LoadCertBundle(ctConfig.IntermediateBundleFilename)
	if err != nil {
		return
	}
	for _, cert := range bundle {
		pub.issuerBundle = append(pub.issuerBundle, base64.StdEncoding.EncodeToString(cert.Raw))
	}
	ctBackoff, err := time.ParseDuration(ctConfig.SubmissionBackoffString)
	if err != nil {
		return
	}
	for _, log := range ctConfig.Logs {
		if !strings.HasPrefix(log.URI, "https://") && !strings.HasPrefix(log.URI, "http://") {
			err = fmt.Errorf("Log URI [%s] is not absolute", log.URI)
			return
		}
	}

	pub.log = logger
	pub.client = &http.Client{}
	pub.submissionBackoff = ctBackoff
	pub.submissionRetries = ctConfig.SubmissionRetries
	pub.ctLogs = ctConfig.Logs

	return
}

func (pub *PublisherImpl) submitToCTLog(serial string, jsonSubmission []byte, log LogDescription) error {
	done := false
	var sct core.SignedCertificateTimestamp
	backoff := pub.submissionBackoff
	var retries int
	for retries = 0; retries <= pub.submissionRetries; retries++ {
		if retries > 0 {
			time.Sleep(backoff)
		}
		resp, err := postJSON(pub.client, fmt.Sprintf("%s%s", log.URI, "/ct/v1/add-chain"), jsonSubmission, &sct)
		if err != nil {
			// Retry the request, log the error
			pub.log.Warning(fmt.Sprintf("Error POSTing JSON to CT log submission endpoint [%s]: %s", log.URI, err))
			backoff = pub.submissionBackoff
			continue
		} else if resp.StatusCode == http.StatusRequestTimeout || resp.StatusCode == http.StatusServiceUnavailable {
			// Retry the request after either the configured backoff period or the period
			// specified by the Retry-After header
			backoff = pub.submissionBackoff
			if seconds, err := strconv.Atoi(resp.Header.Get("Retry-After")); err == nil {
				backoff = time.Second * time.Duration(seconds)
			}
			continue
		} else if resp.StatusCode != http.StatusOK {
			// If we hit an otherwise unexpected status code break the loop and return
			// an error immediately
			return fmt.Errorf("Unexpected status code returned from CT log: %d", resp.StatusCode)
		}
		done = true
		break
	}

	if !done {
		err := fmt.Errorf(
			"Unable to submit certificate to CT log [Serial: %s, Log URI: %s, Retries: %d]",
			serial,
			log.URI,
			retries,
		)
		return err
	}

	if err := sct.CheckSignature(); err != nil {
		return err
	}

	pub.log.Debug(fmt.Sprintf(
		"Submitted certificate to CT log [Serial: %s, Log URI: %s, Retries: %d, Signature: %x]",
		serial,
		log.URI,
		retries, sct.Signature,
	))

	// Set certificate serial and add SCT to DB
	sct.CertificateSerial = serial
	err := pub.SA.AddSCTReceipt(sct)
	if err != nil {
		if _, ok := err.(sa.ErrDuplicateReceipt); ok {
			pub.log.Warning(fmt.Sprintf(
				"SCT receipt for [Serial: %s, Log URI: %s] already exists in database",
				serial,
				log.URI,
			))
			return nil
		} else if err != nil {
			err = fmt.Errorf(
				"Error adding SCT receipt for [%s to %s]: %s",
				sct.CertificateSerial,
				log.URI,
				err,
			)
			return err
		}
	}
	pub.log.Info(fmt.Sprintf(
		"Stored SCT receipt from CT log submission [Serial: %s, Log URI: %s]",
		serial,
		log.URI,
	))
	return nil
}

// SubmitToCT will submit the certificate represented by certDER to any CT
// logs configured in pub.CT.Logs
func (pub *PublisherImpl) SubmitToCT(der []byte) error {
	cert, err := x509.ParseCertificate(der)
	if err != nil {
		pub.log.Err(fmt.Sprintf("Unable to parse certificate, %s", err))
		return err
	}

	submission := ctSubmissionRequest{Chain: []string{base64.StdEncoding.EncodeToString(cert.Raw)}}
	// Add all intermediate certificates needed for submission
	submission.Chain = append(submission.Chain, pub.issuerBundle...)
	jsonSubmission, err := json.Marshal(submission)
	if err != nil {
		pub.log.Err(fmt.Sprintf("Unable to marshal CT submission, %s", err))
		return err
	}

	for _, ctLog := range pub.ctLogs {
		err = pub.submitToCTLog(core.SerialToString(cert.SerialNumber), jsonSubmission, ctLog)
		if err != nil {
			// AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3
			pub.log.AuditErr(err)
			continue
		}
	}

	return nil
}

func postJSON(client *http.Client, uri string, data []byte, respObj interface{}) (*http.Response, error) {
	req, err := http.NewRequest("POST", uri, bytes.NewBuffer(data))
	if err != nil {
		return nil, fmt.Errorf("Creating request failed, %s", err)
	}
	req.Header.Set("Keep-Alive", "timeout=15, max=100")
	req.Header.Set("Content-Type", "application/json")

	resp, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("Request failed, %s", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode == http.StatusOK {
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, fmt.Errorf("Failed to read response body, %s", err)
		}

		err = json.Unmarshal(body, respObj)
		if err != nil {
			return nil, fmt.Errorf("Failed to unmarshal SCT receipt, %s", err)
		}
	}
	return resp, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package zoekt

import (
	"fmt"
	"os"
	"syscall"
)

type mmapedIndexFile struct {
	name string
	size uint32
	data []byte
}

func (f *mmapedIndexFile) Read(off, sz uint32) ([]byte, error) {
	if off+sz > uint32(len(f.data)) {
		return nil, fmt.Errorf("out of bounds: %d, len %d", off+sz, len(f.data))
	}
	return f.data[off : off+sz], nil
}

func (f *mmapedIndexFile) Name() string {
	return f.Name()
}

func (f *mmapedIndexFile) Size() (uint32, error) {
	return f.size, nil
}

func (f *mmapedIndexFile) Close() {
	syscall.Munmap(f.data)
}

// NewIndexFile returns a new index file. The index file takes
// ownership of the passed in file, and may close it.
func NewIndexFile(f *os.File) (IndexFile, error) {
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		return nil, err
	}

	sz := fi.Size()
	if sz >= maxUInt32 {
		return nil, fmt.Errorf("file %s too large: %d", f.Name(), sz)
	}
	r := &mmapedIndexFile{
		name: f.Name(),
		size: uint32(sz),
	}

	rounded := (r.size + 4095) &^ 4095
	r.data, err = syscall.Mmap(int(f.Fd()), 0, int(rounded), syscall.PROT_READ, syscall.MAP_SHARED)
	if err != nil {
		return nil, err
	}

	return r, err
}
<commit_msg>Fix infinite recursion in indexfile_linux.go.<commit_after>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package zoekt

import (
	"fmt"
	"os"
	"syscall"
)

type mmapedIndexFile struct {
	name string
	size uint32
	data []byte
}

func (f *mmapedIndexFile) Read(off, sz uint32) ([]byte, error) {
	if off+sz > uint32(len(f.data)) {
		return nil, fmt.Errorf("out of bounds: %d, len %d", off+sz, len(f.data))
	}
	return f.data[off : off+sz], nil
}

func (f *mmapedIndexFile) Name() string {
	return f.name
}

func (f *mmapedIndexFile) Size() (uint32, error) {
	return f.size, nil
}

func (f *mmapedIndexFile) Close() {
	syscall.Munmap(f.data)
}

// NewIndexFile returns a new index file. The index file takes
// ownership of the passed in file, and may close it.
func NewIndexFile(f *os.File) (IndexFile, error) {
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		return nil, err
	}

	sz := fi.Size()
	if sz >= maxUInt32 {
		return nil, fmt.Errorf("file %s too large: %d", f.Name(), sz)
	}
	r := &mmapedIndexFile{
		name: f.Name(),
		size: uint32(sz),
	}

	rounded := (r.size + 4095) &^ 4095
	r.data, err = syscall.Mmap(int(f.Fd()), 0, int(rounded), syscall.PROT_READ, syscall.MAP_SHARED)
	if err != nil {
		return nil, err
	}

	return r, err
}
<|endoftext|>


<commit_before>package pipe_test

import (
	"io"
	"sync"
	"testing"
	"time"

	"v2ray.com/core/common"
	"v2ray.com/core/common/buf"
	"v2ray.com/core/common/task"
	. "v2ray.com/core/transport/pipe"
	. "v2ray.com/ext/assert"
)

func TestPipeReadWrite(t *testing.T) {
	assert := With(t)

	pReader, pWriter := New(WithSizeLimit(1024))
	payload := []byte{'a', 'b', 'c', 'd'}
	b := buf.New()
	b.Write(payload)
	assert(pWriter.WriteMultiBuffer(buf.MultiBuffer{b}), IsNil)

	rb, err := pReader.ReadMultiBuffer()
	assert(err, IsNil)
	assert(rb.String(), Equals, b.String())
}

func TestPipeCloseError(t *testing.T) {
	assert := With(t)

	pReader, pWriter := New(WithSizeLimit(1024))
	payload := []byte{'a', 'b', 'c', 'd'}
	b := buf.New()
	b.Write(payload)
	assert(pWriter.WriteMultiBuffer(buf.MultiBuffer{b}), IsNil)
	pWriter.CloseError()

	rb, err := pReader.ReadMultiBuffer()
	assert(err, Equals, io.ErrClosedPipe)
	assert(rb.IsEmpty(), IsTrue)
}

func TestPipeClose(t *testing.T) {
	assert := With(t)

	pReader, pWriter := New(WithSizeLimit(1024))
	payload := []byte{'a', 'b', 'c', 'd'}
	b := buf.New()
	b.Write(payload)
	assert(pWriter.WriteMultiBuffer(buf.MultiBuffer{b}), IsNil)
	assert(pWriter.Close(), IsNil)

	rb, err := pReader.ReadMultiBuffer()
	assert(err, IsNil)
	assert(rb.String(), Equals, b.String())

	rb, err = pReader.ReadMultiBuffer()
	assert(err, Equals, io.EOF)
	assert(rb.IsEmpty(), IsTrue)
}

func TestPipeLimitZero(t *testing.T) {
	assert := With(t)

	pReader, pWriter := New(WithSizeLimit(0))
	bb := buf.New()
	bb.Write([]byte{'a', 'b'})
	assert(pWriter.WriteMultiBuffer(buf.MultiBuffer{bb}), IsNil)

	err := task.Run(task.Parallel(func() error {
		b := buf.New()
		b.Write([]byte{'c', 'd'})
		return pWriter.WriteMultiBuffer(buf.MultiBuffer{b})
	}, func() error {
		time.Sleep(time.Second)

		rb, err := pReader.ReadMultiBuffer()
		if err != nil {
			return err
		}
		assert(rb.String(), Equals, "ab")

		rb, err = pReader.ReadMultiBuffer()
		if err != nil {
			return err
		}
		assert(rb.String(), Equals, "cd")
		return nil
	}))()

	assert(err, IsNil)
}

func TestPipeWriteMultiThread(t *testing.T) {
	assert := With(t)

	pReader, pWriter := New(WithSizeLimit(0))

	var wg sync.WaitGroup

	for i := 0; i < 10; i++ {
		wg.Add(1)
		go func() {
			b := buf.New()
			b.WriteString("abcd")
			pWriter.WriteMultiBuffer(buf.MultiBuffer{b})
			wg.Done()
		}()
	}

	time.Sleep(time.Millisecond * 100)

	pWriter.Close()
	wg.Wait()

	b, err := pReader.ReadMultiBuffer()
	assert(err, IsNil)
	assert(b[0].Bytes(), Equals, []byte{'a', 'b', 'c', 'd'})
}

func TestInterfaces(t *testing.T) {
	assert := With(t)

	assert((*Reader)(nil), Implements, (*buf.Reader)(nil))
	assert((*Reader)(nil), Implements, (*buf.TimeoutReader)(nil))
}

func BenchmarkPipeReadWrite(b *testing.B) {
	reader, writer := New(WithoutSizeLimit())
	a := buf.New()
	a.Extend(buf.Size)
	c := buf.MultiBuffer{a}

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		common.Must(writer.WriteMultiBuffer(c))
		d, err := reader.ReadMultiBuffer()
		common.Must(err)
		c = d
	}
}
<commit_msg>update pipe test<commit_after>package pipe_test

import (
	"io"
	"sync"
	"testing"
	"time"

	"v2ray.com/core/common"
	"v2ray.com/core/common/buf"
	"v2ray.com/core/common/task"
	. "v2ray.com/core/transport/pipe"
	. "v2ray.com/ext/assert"
)

func TestPipeReadWrite(t *testing.T) {
	assert := With(t)

	pReader, pWriter := New(WithSizeLimit(1024))
	payload := []byte{'a', 'b', 'c', 'd'}
	b := buf.New()
	b.Write(payload)
	assert(pWriter.WriteMultiBuffer(buf.MultiBuffer{b}), IsNil)

	rb, err := pReader.ReadMultiBuffer()
	assert(err, IsNil)
	assert(rb.String(), Equals, b.String())
}

func TestPipeCloseError(t *testing.T) {
	assert := With(t)

	pReader, pWriter := New(WithSizeLimit(1024))
	payload := []byte{'a', 'b', 'c', 'd'}
	b := buf.New()
	b.Write(payload)
	assert(pWriter.WriteMultiBuffer(buf.MultiBuffer{b}), IsNil)
	pWriter.CloseError()

	rb, err := pReader.ReadMultiBuffer()
	assert(err, Equals, io.ErrClosedPipe)
	assert(rb.IsEmpty(), IsTrue)
}

func TestPipeClose(t *testing.T) {
	assert := With(t)

	pReader, pWriter := New(WithSizeLimit(1024))
	payload := []byte{'a', 'b', 'c', 'd'}
	b := buf.New()
	b.Write(payload)
	assert(pWriter.WriteMultiBuffer(buf.MultiBuffer{b}), IsNil)
	assert(pWriter.Close(), IsNil)

	rb, err := pReader.ReadMultiBuffer()
	assert(err, IsNil)
	assert(rb.String(), Equals, b.String())

	rb, err = pReader.ReadMultiBuffer()
	assert(err, Equals, io.EOF)
	assert(rb.IsEmpty(), IsTrue)
}

func TestPipeLimitZero(t *testing.T) {
	assert := With(t)

	pReader, pWriter := New(WithSizeLimit(0))
	bb := buf.New()
	bb.Write([]byte{'a', 'b'})
	assert(pWriter.WriteMultiBuffer(buf.MultiBuffer{bb}), IsNil)

	err := task.Run(task.Parallel(func() error {
		b := buf.New()
		b.Write([]byte{'c', 'd'})
		return pWriter.WriteMultiBuffer(buf.MultiBuffer{b})
	}, func() error {
		time.Sleep(time.Second)

		var container buf.MultiBufferContainer
		if err := buf.Copy(pReader, &container); err != nil {
			return err
		}

		assert(container.String(), Equals, "abcd")
		return nil
	}, func() error {
		time.Sleep(time.Second * 2)
		pWriter.Close()
		return nil
	}))()

	assert(err, IsNil)
}

func TestPipeWriteMultiThread(t *testing.T) {
	assert := With(t)

	pReader, pWriter := New(WithSizeLimit(0))

	var wg sync.WaitGroup

	for i := 0; i < 10; i++ {
		wg.Add(1)
		go func() {
			b := buf.New()
			b.WriteString("abcd")
			pWriter.WriteMultiBuffer(buf.MultiBuffer{b})
			wg.Done()
		}()
	}

	time.Sleep(time.Millisecond * 100)

	pWriter.Close()
	wg.Wait()

	b, err := pReader.ReadMultiBuffer()
	assert(err, IsNil)
	assert(b[0].Bytes(), Equals, []byte{'a', 'b', 'c', 'd'})
}

func TestInterfaces(t *testing.T) {
	assert := With(t)

	assert((*Reader)(nil), Implements, (*buf.Reader)(nil))
	assert((*Reader)(nil), Implements, (*buf.TimeoutReader)(nil))
}

func BenchmarkPipeReadWrite(b *testing.B) {
	reader, writer := New(WithoutSizeLimit())
	a := buf.New()
	a.Extend(buf.Size)
	c := buf.MultiBuffer{a}

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		common.Must(writer.WriteMultiBuffer(c))
		d, err := reader.ReadMultiBuffer()
		common.Must(err)
		c = d
	}
}
<|endoftext|>


<commit_before>package hooks

import (
	"bufio"
	"context"
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"net/url"
	"strings"
	"time"

	"github.com/fsamin/go-dump"
	"golang.org/x/crypto/ssh"

	"github.com/ovh/cds/engine/cache"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/log"
)

// GerritTaskInfo represents gerrit hook task information and filter
type gerritTaskInfo struct {
	UUID   string   `json:"uuid"`
	Events []string `json:"events"`
}

// RegisterGerritRepoHook register hook on gerrit repository
func (d *dao) RegisterGerritRepoHook(vcsServer string, repo string, g gerritTaskInfo) error {
	return d.store.SetAdd(cache.Key(gerritRepoKey, vcsServer, repo), g.UUID, g)
}

func (d *dao) RemoveGerritRepoHook(vcsServer string, repo string, g gerritTaskInfo) {
	d.store.SetRemove(cache.Key(gerritRepoKey, vcsServer, repo), g.UUID, g)
}

// FindGerritTasksByRepo get all gerrit hooks on the given repository
func (d *dao) FindGerritTasksByRepo(ctx context.Context, vcsServer string, repo string) ([]gerritTaskInfo, error) {
	key := cache.Key(gerritRepoKey, vcsServer, repo)
	nbGerritHooks, err := d.store.SetCard(key)
	if err != nil {
		return nil, sdk.WrapError(err, "unable to setCard %v", key)
	}

	hooks := make([]*gerritTaskInfo, nbGerritHooks)
	for i := 0; i < nbGerritHooks; i++ {
		hooks[i] = &gerritTaskInfo{}
	}
	if err := d.store.SetScan(ctx, key, sdk.InterfaceSlice(hooks)...); err != nil {
		return nil, sdk.WrapError(err, "Unable to scan %s", key)
	}

	allHooks := make([]gerritTaskInfo, nbGerritHooks)
	for i := 0; i < nbGerritHooks; i++ {
		allHooks[i] = *hooks[i]
	}

	return allHooks, nil
}

func (s *Service) startGerritHookTask(t *sdk.Task) error {
	g := gerritTaskInfo{
		UUID:   t.UUID,
		Events: strings.Split(t.Config[sdk.HookConfigEventFilter].Value, ";"),
	}
	s.Dao.RegisterGerritRepoHook(t.Config[sdk.HookConfigVCSServer].Value, t.Config[sdk.HookConfigRepoFullName].Value, g)

	// Check that stream is open
	if _, has := gerritRepoHooks[t.Config[sdk.HookConfigVCSServer].Value]; !has {
		// Start listening to gerrit event stream
		vcsConfig, err := s.Client.VCSConfiguration()
		if err != nil {
			return sdk.WrapError(err, "unable to get vcs configuration")
		}
		s.initGerritStreamEvent(context.Background(), t.Config[sdk.HookConfigVCSServer].Value, vcsConfig)
	}
	return nil
}

func (s *Service) stopGerritHookTask(t *sdk.Task) {
	g := gerritTaskInfo{
		UUID:   t.UUID,
		Events: strings.Split(t.Config[sdk.HookConfigEventFilter].Value, ";"),
	}
	s.Dao.RemoveGerritRepoHook(t.Config[sdk.HookConfigVCSServer].Value, t.Config[sdk.HookConfigRepoFullName].Value, g)
}

func (s *Service) doGerritExecution(e *sdk.TaskExecution) (*sdk.WorkflowNodeRunHookEvent, error) {
	log.Debug("Hooks> Processing gerrit event %s %s", e.UUID, e.Type)

	// Prepare a struct to send to CDS API
	h := sdk.WorkflowNodeRunHookEvent{
		WorkflowNodeHookUUID: e.UUID,
	}

	var gerritEvent GerritEvent
	if err := json.Unmarshal(e.GerritEvent.Message, &gerritEvent); err != nil {
		return nil, sdk.WrapError(err, "unable to unmarshal gerrit event %s", string(e.GerritEvent.Message))
	}

	payload := make(map[string]interface{})
	payload[GIT_EVENT] = gerritEvent.Type

	// assignee-* / change-* / comment-* / draft-* / hashtags-* / patchset-* / reviewer-* / topic-* / vote-*
	if gerritEvent.Change != nil {
		payload[GIT_AUTHOR] = gerritEvent.Change.Owner.Username
		payload[GIT_AUTHOR_EMAIL] = gerritEvent.Change.Owner.Email
		payload[GIT_REPOSITORY] = gerritEvent.Change.Project
		payload[CDS_TRIGGERED_BY_USERNAME] = gerritEvent.Change.Owner.Username
		payload[CDS_TRIGGERED_BY_FULLNAME] = gerritEvent.Change.Owner.Name
		payload[CDS_TRIGGERED_BY_EMAIL] = gerritEvent.Change.Owner.Email

		payload[GIT_MESSAGE] = gerritEvent.Change.CommitMessage
		payload["gerrit.change.id"] = fmt.Sprintf("%s~%s~%s", url.QueryEscape(gerritEvent.Change.Project), url.QueryEscape(gerritEvent.Change.Branch), gerritEvent.Change.ID)
		payload["gerrit.change.url"] = gerritEvent.Change.URL
		payload["gerrit.change.status"] = gerritEvent.Change.Status
		payload["gerrit.change.branch"] = gerritEvent.Change.Branch
	}

	// ref-updated
	if gerritEvent.RefUpdate != nil {
		payload[GIT_HASH_BEFORE] = gerritEvent.RefUpdate.OldRev
		payload[GIT_HASH] = gerritEvent.RefUpdate.NewRev
		payload["gerrit.ref.name"] = gerritEvent.RefUpdate.RefName
	}
	// change-merged / ref-updated
	if gerritEvent.Submitter != nil {
		if gerritEvent.Submitter.Username != "" {
			payload[GIT_AUTHOR] = gerritEvent.Submitter.Username
		}
		if gerritEvent.Submitter.Email != "" {
			payload[GIT_AUTHOR_EMAIL] = gerritEvent.Submitter.Email
		}
	}
	// change-* / comment-* / draft-* / patchset-* / reviewer-* / vote-*
	if gerritEvent.PatchSet != nil {
		payload[GIT_HASH] = gerritEvent.PatchSet.Revision
		if len(gerritEvent.PatchSet.Parents) == 1 {
			payload[GIT_HASH_BEFORE] = gerritEvent.PatchSet.Parents[0]
		}
		payload["gerrit.change.ref"] = gerritEvent.PatchSet.Ref
		if gerritEvent.PatchSet.Author != nil {
			if gerritEvent.PatchSet.Author.Username != "" {
				payload[GIT_AUTHOR] = gerritEvent.PatchSet.Author.Username
			}
			if gerritEvent.PatchSet.Author.Email != "" {
				payload[GIT_AUTHOR_EMAIL] = gerritEvent.PatchSet.Author.Email
			}
		}
	}
	// change-merged
	if gerritEvent.NewRev != "" {
		payload[GIT_HASH] = gerritEvent.NewRev
	}

	// Comment
	if gerritEvent.Type == GerritEventTypeCommentAdded {
		payload["gerrit.comment"] = gerritEvent.Comment
		if gerritEvent.Author != nil {
			payload["gerrit.comment.author.username"] = gerritEvent.Author.Username
			payload["gerrit.comment.author.name"] = gerritEvent.Author.Name
			payload["gerrit.comment.author.email"] = gerritEvent.Author.Email
		}
	}

	payload["gerrit.type"] = gerritEvent.Type

	payload[PAYLOAD] = string(e.GerritEvent.Message)

	d := dump.NewDefaultEncoder()
	d.ExtraFields.Type = false
	d.ExtraFields.Len = false
	d.ExtraFields.DetailedMap = false
	d.ExtraFields.DetailedStruct = false
	d.Formatters = []dump.KeyFormatterFunc{dump.WithDefaultLowerCaseFormatter()}
	payloadValues, errDump := d.ToStringMap(payload)
	if errDump != nil {
		return nil, sdk.WrapError(errDump, "Cannot dump payload %+v ", payload)
	}
	h.Payload = payloadValues

	return &h, nil
}

func (s *Service) ComputeGerritStreamEvent(ctx context.Context, vcsServer string, gerritEventChan <-chan GerritEvent) {
	for {
		select {
		case <-ctx.Done():
			return
		case e := <-gerritEventChan:
			var repo string
			switch {
			case e.Change != nil:
				repo = e.Change.Project
			case e.RefUpdate != nil:
				repo = e.RefUpdate.Project
			}

			hooks, err := s.Dao.FindGerritTasksByRepo(ctx, vcsServer, repo)
			if err != nil {
				log.Error(ctx, "ComputeGerritStreamEvent > Unable to list task for repo %s/%s", vcsServer, repo)
				continue
			}

			msg, err := json.Marshal(e)
			if err != nil {
				log.Error(ctx, "unable to marshal gerrit event: %v", err)
			}

			for _, h := range hooks {
				if !sdk.IsInArray(e.Type, h.Events) {
					continue
				}

				//Load the task
				gerritHook := s.Dao.FindTask(ctx, h.UUID)
				if gerritHook == nil {
					log.Error(ctx, "Unknown uuid %s", h.UUID)
					continue
				}

				//Prepare a web hook execution
				exec := &sdk.TaskExecution{
					Timestamp: time.Now().UnixNano(),
					UUID:      h.UUID,
					Status:    TaskExecutionScheduled,
					GerritEvent: &sdk.GerritEventExecution{
						Message: msg,
					},
					Type: gerritHook.Type,
				}

				//Save the web hook execution
				s.Dao.SaveTaskExecution(exec)

				//Push the webhook execution in the queue, so it will be executed
				if err := s.Dao.EnqueueTaskExecution(ctx, exec); err != nil {
					log.Error(ctx, "ComputeGerritStreamEvent > error on EnqueueTaskExecution %v", err)
				}
			}
		}
	}
}

// ListenGerritStreamEvent listen the gerrit event stream
func ListenGerritStreamEvent(ctx context.Context, store cache.Store, v sdk.VCSConfiguration, gerritEventChan chan<- GerritEvent) {
	signer, err := ssh.ParsePrivateKey([]byte(v.Password))
	if err != nil {
		log.Error(ctx, "unable to read ssh key: %v", err)
		return
	}

	// Create config
	config := &ssh.ClientConfig{
		User: v.Username,
		Auth: []ssh.AuthMethod{
			ssh.PublicKeys(signer),
		},
		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
	}

	URL, _ := url.Parse(v.URL)

	// Dial TCP
	conn, err := ssh.Dial("tcp", fmt.Sprintf("%s:%d", URL.Hostname(), v.SSHPort), config)
	if err != nil {
		log.Error(ctx, "ListenGerritStreamEvent> unable to open ssh connection to gerrit: %v", err)
		return
	}
	defer conn.Close()

	session, err := conn.NewSession()
	if err != nil {
		log.Error(ctx, "ListenGerritStreamEvent> unable to create new session: %v", err)
		return
	}

	r, w := io.Pipe()
	session.Stdout = w

	stdoutreader := bufio.NewReader(r)

	go func() {
		// Run command
		log.Debug("Listening to gerrit event stream %s", v.URL)
		if err := session.Run("gerrit stream-events"); err != nil {
			log.Error(ctx, "ListenGerritStreamEvent> unable to run gerrit stream-events command: %v", err)
		}
	}()

	lockKey := cache.Key("gerrit", "event", "lock")
	tick := time.NewTicker(50 * time.Millisecond)
	for {
		select {
		case <-ctx.Done():
			session.Close()
			conn.Close()
		case <-tick.C:
			line, errs := stdoutreader.ReadString('\n')
			if errs == io.EOF {
				continue
			}
			if errs != nil {
				log.Warning(ctx, "ListenGerritStreamEvent> unable to read string")
				continue
			}
			if line == "" {
				continue
			}
			var event GerritEvent
			lineBytes := []byte(line)
			if err := json.Unmarshal(lineBytes, &event); err != nil {
				log.Error(ctx, "unable to read gerrit event %v: %s", err, line)
				continue
			}

			// Avoid that 2 hook uservice dispatch the same event
			// Take the lock to dispatch an event
			_, _ = store.Lock(lockKey, time.Minute, 100, 10)

			// compute md5
			hasher := md5.New()
			hasher.Write(lineBytes) // nolint
			md5 := hex.EncodeToString(hasher.Sum(nil))

			// check if this event has already been dispatched
			k := cache.Key("gerrit", "event", "id", md5)
			var existString string
			b, _ := store.Get(k, &existString)
			if !b {
				_ = store.SetWithTTL(k, md5, 300)
			}

			// release lock
			if err := store.Unlock(lockKey); err == nil {
				log.Error(ctx, "unable to unlock %s. Waiting lock timeout", lockKey)
			}

			if !b {
				gerritEventChan <- event
			}
		}
	}

}
<commit_msg>fix(hooks): display ListenGerritStreamEvent  error logs (#5399)<commit_after>package hooks

import (
	"bufio"
	"context"
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"net/url"
	"strings"
	"time"

	"github.com/fsamin/go-dump"
	"golang.org/x/crypto/ssh"

	"github.com/ovh/cds/engine/cache"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/log"
)

// GerritTaskInfo represents gerrit hook task information and filter
type gerritTaskInfo struct {
	UUID   string   `json:"uuid"`
	Events []string `json:"events"`
}

// RegisterGerritRepoHook register hook on gerrit repository
func (d *dao) RegisterGerritRepoHook(vcsServer string, repo string, g gerritTaskInfo) error {
	return d.store.SetAdd(cache.Key(gerritRepoKey, vcsServer, repo), g.UUID, g)
}

func (d *dao) RemoveGerritRepoHook(vcsServer string, repo string, g gerritTaskInfo) {
	d.store.SetRemove(cache.Key(gerritRepoKey, vcsServer, repo), g.UUID, g)
}

// FindGerritTasksByRepo get all gerrit hooks on the given repository
func (d *dao) FindGerritTasksByRepo(ctx context.Context, vcsServer string, repo string) ([]gerritTaskInfo, error) {
	key := cache.Key(gerritRepoKey, vcsServer, repo)
	nbGerritHooks, err := d.store.SetCard(key)
	if err != nil {
		return nil, sdk.WrapError(err, "unable to setCard %v", key)
	}

	hooks := make([]*gerritTaskInfo, nbGerritHooks)
	for i := 0; i < nbGerritHooks; i++ {
		hooks[i] = &gerritTaskInfo{}
	}
	if err := d.store.SetScan(ctx, key, sdk.InterfaceSlice(hooks)...); err != nil {
		return nil, sdk.WrapError(err, "Unable to scan %s", key)
	}

	allHooks := make([]gerritTaskInfo, nbGerritHooks)
	for i := 0; i < nbGerritHooks; i++ {
		allHooks[i] = *hooks[i]
	}

	return allHooks, nil
}

func (s *Service) startGerritHookTask(t *sdk.Task) error {
	g := gerritTaskInfo{
		UUID:   t.UUID,
		Events: strings.Split(t.Config[sdk.HookConfigEventFilter].Value, ";"),
	}
	s.Dao.RegisterGerritRepoHook(t.Config[sdk.HookConfigVCSServer].Value, t.Config[sdk.HookConfigRepoFullName].Value, g)

	// Check that stream is open
	if _, has := gerritRepoHooks[t.Config[sdk.HookConfigVCSServer].Value]; !has {
		// Start listening to gerrit event stream
		vcsConfig, err := s.Client.VCSConfiguration()
		if err != nil {
			return sdk.WrapError(err, "unable to get vcs configuration")
		}
		s.initGerritStreamEvent(context.Background(), t.Config[sdk.HookConfigVCSServer].Value, vcsConfig)
	}
	return nil
}

func (s *Service) stopGerritHookTask(t *sdk.Task) {
	g := gerritTaskInfo{
		UUID:   t.UUID,
		Events: strings.Split(t.Config[sdk.HookConfigEventFilter].Value, ";"),
	}
	s.Dao.RemoveGerritRepoHook(t.Config[sdk.HookConfigVCSServer].Value, t.Config[sdk.HookConfigRepoFullName].Value, g)
}

func (s *Service) doGerritExecution(e *sdk.TaskExecution) (*sdk.WorkflowNodeRunHookEvent, error) {
	log.Debug("Hooks> Processing gerrit event %s %s", e.UUID, e.Type)

	// Prepare a struct to send to CDS API
	h := sdk.WorkflowNodeRunHookEvent{
		WorkflowNodeHookUUID: e.UUID,
	}

	var gerritEvent GerritEvent
	if err := json.Unmarshal(e.GerritEvent.Message, &gerritEvent); err != nil {
		return nil, sdk.WrapError(err, "unable to unmarshal gerrit event %s", string(e.GerritEvent.Message))
	}

	payload := make(map[string]interface{})
	payload[GIT_EVENT] = gerritEvent.Type

	// assignee-* / change-* / comment-* / draft-* / hashtags-* / patchset-* / reviewer-* / topic-* / vote-*
	if gerritEvent.Change != nil {
		payload[GIT_AUTHOR] = gerritEvent.Change.Owner.Username
		payload[GIT_AUTHOR_EMAIL] = gerritEvent.Change.Owner.Email
		payload[GIT_REPOSITORY] = gerritEvent.Change.Project
		payload[CDS_TRIGGERED_BY_USERNAME] = gerritEvent.Change.Owner.Username
		payload[CDS_TRIGGERED_BY_FULLNAME] = gerritEvent.Change.Owner.Name
		payload[CDS_TRIGGERED_BY_EMAIL] = gerritEvent.Change.Owner.Email

		payload[GIT_MESSAGE] = gerritEvent.Change.CommitMessage
		payload["gerrit.change.id"] = fmt.Sprintf("%s~%s~%s", url.QueryEscape(gerritEvent.Change.Project), url.QueryEscape(gerritEvent.Change.Branch), gerritEvent.Change.ID)
		payload["gerrit.change.url"] = gerritEvent.Change.URL
		payload["gerrit.change.status"] = gerritEvent.Change.Status
		payload["gerrit.change.branch"] = gerritEvent.Change.Branch
	}

	// ref-updated
	if gerritEvent.RefUpdate != nil {
		payload[GIT_HASH_BEFORE] = gerritEvent.RefUpdate.OldRev
		payload[GIT_HASH] = gerritEvent.RefUpdate.NewRev
		payload["gerrit.ref.name"] = gerritEvent.RefUpdate.RefName
	}
	// change-merged / ref-updated
	if gerritEvent.Submitter != nil {
		if gerritEvent.Submitter.Username != "" {
			payload[GIT_AUTHOR] = gerritEvent.Submitter.Username
		}
		if gerritEvent.Submitter.Email != "" {
			payload[GIT_AUTHOR_EMAIL] = gerritEvent.Submitter.Email
		}
	}
	// change-* / comment-* / draft-* / patchset-* / reviewer-* / vote-*
	if gerritEvent.PatchSet != nil {
		payload[GIT_HASH] = gerritEvent.PatchSet.Revision
		if len(gerritEvent.PatchSet.Parents) == 1 {
			payload[GIT_HASH_BEFORE] = gerritEvent.PatchSet.Parents[0]
		}
		payload["gerrit.change.ref"] = gerritEvent.PatchSet.Ref
		if gerritEvent.PatchSet.Author != nil {
			if gerritEvent.PatchSet.Author.Username != "" {
				payload[GIT_AUTHOR] = gerritEvent.PatchSet.Author.Username
			}
			if gerritEvent.PatchSet.Author.Email != "" {
				payload[GIT_AUTHOR_EMAIL] = gerritEvent.PatchSet.Author.Email
			}
		}
	}
	// change-merged
	if gerritEvent.NewRev != "" {
		payload[GIT_HASH] = gerritEvent.NewRev
	}

	// Comment
	if gerritEvent.Type == GerritEventTypeCommentAdded {
		payload["gerrit.comment"] = gerritEvent.Comment
		if gerritEvent.Author != nil {
			payload["gerrit.comment.author.username"] = gerritEvent.Author.Username
			payload["gerrit.comment.author.name"] = gerritEvent.Author.Name
			payload["gerrit.comment.author.email"] = gerritEvent.Author.Email
		}
	}

	payload["gerrit.type"] = gerritEvent.Type

	payload[PAYLOAD] = string(e.GerritEvent.Message)

	d := dump.NewDefaultEncoder()
	d.ExtraFields.Type = false
	d.ExtraFields.Len = false
	d.ExtraFields.DetailedMap = false
	d.ExtraFields.DetailedStruct = false
	d.Formatters = []dump.KeyFormatterFunc{dump.WithDefaultLowerCaseFormatter()}
	payloadValues, errDump := d.ToStringMap(payload)
	if errDump != nil {
		return nil, sdk.WrapError(errDump, "Cannot dump payload %+v ", payload)
	}
	h.Payload = payloadValues

	return &h, nil
}

func (s *Service) ComputeGerritStreamEvent(ctx context.Context, vcsServer string, gerritEventChan <-chan GerritEvent) {
	for {
		select {
		case <-ctx.Done():
			return
		case e := <-gerritEventChan:
			var repo string
			switch {
			case e.Change != nil:
				repo = e.Change.Project
			case e.RefUpdate != nil:
				repo = e.RefUpdate.Project
			}

			hooks, err := s.Dao.FindGerritTasksByRepo(ctx, vcsServer, repo)
			if err != nil {
				log.Error(ctx, "ComputeGerritStreamEvent > Unable to list task for repo %s/%s", vcsServer, repo)
				continue
			}

			msg, err := json.Marshal(e)
			if err != nil {
				log.Error(ctx, "unable to marshal gerrit event: %v", err)
			}

			for _, h := range hooks {
				if !sdk.IsInArray(e.Type, h.Events) {
					continue
				}

				//Load the task
				gerritHook := s.Dao.FindTask(ctx, h.UUID)
				if gerritHook == nil {
					log.Error(ctx, "Unknown uuid %s", h.UUID)
					continue
				}

				//Prepare a web hook execution
				exec := &sdk.TaskExecution{
					Timestamp: time.Now().UnixNano(),
					UUID:      h.UUID,
					Status:    TaskExecutionScheduled,
					GerritEvent: &sdk.GerritEventExecution{
						Message: msg,
					},
					Type: gerritHook.Type,
				}

				//Save the web hook execution
				s.Dao.SaveTaskExecution(exec)

				//Push the webhook execution in the queue, so it will be executed
				if err := s.Dao.EnqueueTaskExecution(ctx, exec); err != nil {
					log.Error(ctx, "ComputeGerritStreamEvent > error on EnqueueTaskExecution %v", err)
				}
			}
		}
	}
}

// ListenGerritStreamEvent listen the gerrit event stream
func ListenGerritStreamEvent(ctx context.Context, store cache.Store, v sdk.VCSConfiguration, gerritEventChan chan<- GerritEvent) {
	signer, err := ssh.ParsePrivateKey([]byte(v.Password))
	if err != nil {
		log.Error(ctx, "unable to read ssh key: %v", err)
		return
	}

	// Create config
	config := &ssh.ClientConfig{
		User: v.Username,
		Auth: []ssh.AuthMethod{
			ssh.PublicKeys(signer),
		},
		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
	}

	URL, _ := url.Parse(v.URL)

	// Dial TCP
	conn, err := ssh.Dial("tcp", fmt.Sprintf("%s:%d", URL.Hostname(), v.SSHPort), config)
	if err != nil {
		log.Error(ctx, "ListenGerritStreamEvent> unable to open ssh connection to gerrit: %v", err)
		return
	}
	defer conn.Close()

	session, err := conn.NewSession()
	if err != nil {
		log.Error(ctx, "ListenGerritStreamEvent> unable to create new session: %v", err)
		return
	}

	r, w := io.Pipe()
	session.Stdout = w

	stdoutreader := bufio.NewReader(r)

	go func() {
		// Run command
		log.Debug("Listening to gerrit event stream %s", v.URL)
		if err := session.Run("gerrit stream-events"); err != nil {
			log.Error(ctx, "ListenGerritStreamEvent> unable to run gerrit stream-events command: %v", err)
		}
	}()

	lockKey := cache.Key("gerrit", "event", "lock")
	tick := time.NewTicker(50 * time.Millisecond)
	for {
		select {
		case <-ctx.Done():
			session.Close()
			conn.Close()
		case <-tick.C:
			line, errs := stdoutreader.ReadString('\n')
			if errs == io.EOF {
				continue
			}
			if errs != nil {
				log.Warning(ctx, "ListenGerritStreamEvent> unable to read string")
				continue
			}
			if line == "" {
				continue
			}
			var event GerritEvent
			lineBytes := []byte(line)
			if err := json.Unmarshal(lineBytes, &event); err != nil {
				log.Error(ctx, "unable to read gerrit event %v: %s", err, line)
				continue
			}

			// Avoid that 2 hook uservice dispatch the same event
			// Take the lock to dispatch an event
			locked, err := store.Lock(lockKey, time.Minute, 100, 15)
			if err != nil {
				log.Error(ctx, "unable to lock %s: %v", lockKey, err)
			}

			// compute md5
			hasher := md5.New()
			hasher.Write(lineBytes) // nolint
			md5 := hex.EncodeToString(hasher.Sum(nil))

			// check if this event has already been dispatched
			k := cache.Key("gerrit", "event", "id", md5)
			var existString string
			b, _ := store.Get(k, &existString)
			if !b {
				_ = store.SetWithTTL(k, md5, 300)
			}

			// release lock
			if locked {
				if err := store.Unlock(lockKey); err == nil {
					log.Error(ctx, "unable to unlock %s: %v", lockKey, err)
				}
			}

			if !b {
				gerritEventChan <- event
			}
		}
	}

}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"sync"

	"github.com/lucas-clemente/quic-go"
	"github.com/lucas-clemente/quic-go/http3"
	"github.com/lucas-clemente/quic-go/internal/testdata"
	"github.com/lucas-clemente/quic-go/internal/utils"
)

func main() {
	verbose := flag.Bool("v", false, "verbose")
	quiet := flag.Bool("q", false, "don't print the data")
	keyLogFile := flag.String("keylog", "", "key log file")
	insecure := flag.Bool("insecure", false, "skip certificate verification")
	qlog := flag.Bool("qlog", false, "output a qlog (in the same directory)")
	flag.Parse()
	urls := flag.Args()

	logger := utils.DefaultLogger

	if *verbose {
		logger.SetLogLevel(utils.LogLevelDebug)
	} else {
		logger.SetLogLevel(utils.LogLevelInfo)
	}
	logger.SetLogTimeFormat("")

	var keyLog *os.File
	if len(*keyLogFile) > 0 {
		var err error
		keyLog, err = os.Create(*keyLogFile)
		if err != nil {
			log.Fatal(err)
		}
		defer keyLog.Close()
	}

	pool, err := x509.SystemCertPool()
	if err != nil {
		log.Fatal(err)
	}
	testdata.AddRootCA(pool)

	var qconf quic.Config
	if *qlog {
		qconf.GetLogWriter = func(connID []byte) io.WriteCloser {
			filename := fmt.Sprintf("client_%x.qlog", connID)
			f, err := os.Create(filename)
			if err != nil {
				log.Fatal(err)
			}
			log.Printf("Creating qlog file %s.\n", filename)
			return f
		}
	}
	roundTripper := &http3.RoundTripper{
		TLSClientConfig: &tls.Config{
			RootCAs:            pool,
			InsecureSkipVerify: *insecure,
			KeyLogWriter:       keyLog,
		},
		QuicConfig: &qconf,
	}
	defer roundTripper.Close()
	hclient := &http.Client{
		Transport: roundTripper,
	}

	var wg sync.WaitGroup
	wg.Add(len(urls))
	for _, addr := range urls {
		logger.Infof("GET %s", addr)
		go func(addr string) {
			rsp, err := hclient.Get(addr)
			if err != nil {
				log.Fatal(err)
			}
			logger.Infof("Got response for %s: %#v", addr, rsp)

			body := &bytes.Buffer{}
			_, err = io.Copy(body, rsp.Body)
			if err != nil {
				log.Fatal(err)
			}
			if *quiet {
				logger.Infof("Request Body: %d bytes", body.Len())
			} else {
				logger.Infof("Request Body:")
				logger.Infof("%s", body.Bytes())
			}
			wg.Done()
		}(addr)
	}
	wg.Wait()
}
<commit_msg>fix keylog creation in the example client<commit_after>package main

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"sync"

	"github.com/lucas-clemente/quic-go"
	"github.com/lucas-clemente/quic-go/http3"
	"github.com/lucas-clemente/quic-go/internal/testdata"
	"github.com/lucas-clemente/quic-go/internal/utils"
)

func main() {
	verbose := flag.Bool("v", false, "verbose")
	quiet := flag.Bool("q", false, "don't print the data")
	keyLogFile := flag.String("keylog", "", "key log file")
	insecure := flag.Bool("insecure", false, "skip certificate verification")
	qlog := flag.Bool("qlog", false, "output a qlog (in the same directory)")
	flag.Parse()
	urls := flag.Args()

	logger := utils.DefaultLogger

	if *verbose {
		logger.SetLogLevel(utils.LogLevelDebug)
	} else {
		logger.SetLogLevel(utils.LogLevelInfo)
	}
	logger.SetLogTimeFormat("")

	var keyLog io.Writer
	if len(*keyLogFile) > 0 {
		f, err := os.Create(*keyLogFile)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		keyLog = f
	}

	pool, err := x509.SystemCertPool()
	if err != nil {
		log.Fatal(err)
	}
	testdata.AddRootCA(pool)

	var qconf quic.Config
	if *qlog {
		qconf.GetLogWriter = func(connID []byte) io.WriteCloser {
			filename := fmt.Sprintf("client_%x.qlog", connID)
			f, err := os.Create(filename)
			if err != nil {
				log.Fatal(err)
			}
			log.Printf("Creating qlog file %s.\n", filename)
			return f
		}
	}
	roundTripper := &http3.RoundTripper{
		TLSClientConfig: &tls.Config{
			RootCAs:            pool,
			InsecureSkipVerify: *insecure,
			KeyLogWriter:       keyLog,
		},
		QuicConfig: &qconf,
	}
	defer roundTripper.Close()
	hclient := &http.Client{
		Transport: roundTripper,
	}

	var wg sync.WaitGroup
	wg.Add(len(urls))
	for _, addr := range urls {
		logger.Infof("GET %s", addr)
		go func(addr string) {
			rsp, err := hclient.Get(addr)
			if err != nil {
				log.Fatal(err)
			}
			logger.Infof("Got response for %s: %#v", addr, rsp)

			body := &bytes.Buffer{}
			_, err = io.Copy(body, rsp.Body)
			if err != nil {
				log.Fatal(err)
			}
			if *quiet {
				logger.Infof("Request Body: %d bytes", body.Len())
			} else {
				logger.Infof("Request Body:")
				logger.Infof("%s", body.Bytes())
			}
			wg.Done()
		}(addr)
	}
	wg.Wait()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/devfeel/dotweb"
	"github.com/devfeel/dotweb/logger"
	"os"
	"strconv"
	"strings"
	"syscall"
	"time"
)

func main() {
	//初始化DotServer
	app := dotweb.New()

	//设置dotserver日志目录
	//如果不设置，默认启用，且默认为当前目录
	//app.SetLogger(NewYLog())
	app.SetEnabledLog(true)
	app.SetLogPath("d:/gotmp/xlog/xlog1/xlog2/")

	fmt.Println(logger.Logger())

	//开启development模式
	app.SetDevelopmentMode()

	//设置路由
	InitRoute(app.HttpServer)

	//启动 监控服务
	app.SetPProfConfig(true, 8081)

	// 开始服务
	port := 8080
	fmt.Println("dotweb.StartServer => " + strconv.Itoa(port))
	err := app.StartServer(port)
	fmt.Println("dotweb.StartServer error => ", err)
}

func Index(ctx dotweb.Context) error {
	ctx.Response().Header().Set("Content-Type", "text/html; charset=utf-8")
	logger.Logger().Debug("debug", "x")
	logger.Logger().Info("info", "x")
	logger.Logger().Warn("warn", "x")
	logger.Logger().Error("error", "x")
	_, err := ctx.WriteStringC(201, "index => ", ctx.RouterParams())
	return err
}

func InitRoute(server *dotweb.HttpServer) {
	server.Router().GET("/", Index)
}

type chanLog struct {
	Content   string
	LogTarget string
}

type yLog struct {
	logRootPath    string
	logChan_Custom chan chanLog
	enabledLog     bool
}

//create new yLog
func NewYLog() *yLog {
	l := &yLog{logChan_Custom: make(chan chanLog, 10000)}
	go l.handleCustom()
	return l
}

const (
	defaultDateFormatForFileName = "2006_01_02"
	defaultDateLayout            = "2006-01-02"
	defaultFullTimeLayout        = "2006-01-02 15:04:05.999999"
	defaultTimeLayout            = "2006-01-02 15:04:05"
)

func (l *yLog) Debug(log string, logTarget string) {
	l.Log(log, logTarget, "debug")
}

func (l *yLog) Info(log string, logTarget string) {
	l.Log(log, logTarget, "info")
}

func (l *yLog) Warn(log string, logTarget string) {
	l.Log(log, logTarget, "warn")
}

func (l *yLog) Error(log string, logTarget string) {
	l.Log(log, logTarget, "error")
}

func (l *yLog) Log(log string, logTarget string, logLevel string) {
	if l.enabledLog {
		chanLog := chanLog{
			LogTarget: "yLog_" + logTarget + "_" + logLevel,
			Content:   log,
		}
		l.logChan_Custom <- chanLog
	}
}

//set log path
func (l *yLog) SetLogPath(rootPath string) {
	//设置日志根目录
	l.logRootPath = rootPath
	if !strings.HasSuffix(l.logRootPath, "/") {
		l.logRootPath = l.logRootPath + "/"
	}
}

//set enabled log
func (l *yLog) SetEnabledLog(enabledLog bool) {
	l.enabledLog = enabledLog
}

//处理日志内部函数
func (l *yLog) handleCustom() {
	for {
		log := <-l.logChan_Custom
		l.writeLog(log, "custom")
	}
}

func (l *yLog) writeLog(chanLog chanLog, level string) {
	filePath := l.logRootPath + chanLog.LogTarget
	switch level {
	case "custom":
		filePath = filePath + "_" + time.Now().Format(defaultDateFormatForFileName) + ".log"
		break
	}
	log := time.Now().Format(defaultFullTimeLayout) + " " + chanLog.Content
	writeFile(filePath, log)
}

func writeFile(logFile string, log string) {
	var mode os.FileMode
	flag := syscall.O_RDWR | syscall.O_APPEND | syscall.O_CREAT
	mode = 0666
	logstr := log + "\r\n"
	file, err := os.OpenFile(logFile, flag, mode)
	defer file.Close()
	if err != nil {
		fmt.Println(logFile, err)
		return
	}
	//fmt.Print(logstr)
	file.WriteString(logstr)
}
<commit_msg>remove test case for example/logger, you can view logger example at https://github.com/devfeel/dotweb-example/tree/master/logger<commit_after><|endoftext|>


<commit_before>package bloom

import (
	"fmt"
	"sync"

	"github.com/spaolacci/murmur3"

	streakKnife "github.com/holiman/bloomfilter/v2"
)

var (
	ErrMaxBytes = fmt.Errorf("too large")
)

type Filter interface {
	// Add adds to filter, assumed thread safe
	Add(...[]byte)

	// Check checks filter, assumed thread safe
	Check([]byte) bool
}

func New(maxN uint64, p float64, maxBytes uint64) (Filter, error) {
	neededBytes := bytesSteakKnifeFilter(maxN, p)
	if neededBytes > maxBytes {
		return nil, ErrMaxBytes
	}
	return newSteakKnifeFilter(maxN, p)
}

type steakKnifeFilter struct {
	lock    sync.RWMutex
	bfilter *streakKnife.Filter
}

func bytesSteakKnifeFilter(maxN uint64, p float64) uint64 {
	m := streakKnife.OptimalM(maxN, p)
	k := streakKnife.OptimalK(m, maxN)

	// this is pulled from bloomFilter.newBits and bloomfilter.newRandKeys
	// the calculation is the size of the bitset which would be created from this filter.
	// to ensure we don't crash memory, we would ensure the size
	msize := (m + 63) / 64
	msize += k

	// 8 == sizeof(uint64))
	return msize * 8
}

func newSteakKnifeFilter(maxN uint64, p float64) (Filter, error) {
	m := streakKnife.OptimalM(maxN, p)
	k := streakKnife.OptimalK(m, maxN)

	bfilter, err := streakKnife.New(m, k)
	return &steakKnifeFilter{bfilter: bfilter}, err
}

func (f *steakKnifeFilter) Add(bl ...[]byte) {
	f.lock.Lock()
	defer f.lock.Unlock()
	for _, b := range bl {
		h := murmur3.New64()
		_, errf := h.Write(b)

		// this shouldnt happen
		if errf != nil {
			continue
		}
		f.bfilter.Add(h)
	}
}

func (f *steakKnifeFilter) Check(b []byte) bool {
	f.lock.RLock()
	defer f.lock.RUnlock()
	h := murmur3.New64()
	_, err := h.Write(b)
	if err != nil {
		return false
	}
	return f.bfilter.Contains(h)
}
<commit_msg>finished bloom cleanup<commit_after>package bloom

import (
	"fmt"
	"sync"

	"github.com/spaolacci/murmur3"

	streakKnife "github.com/holiman/bloomfilter/v2"
)

var (
	ErrMaxBytes = fmt.Errorf("too large")
)

type Filter interface {
	// Add adds to filter, assumed thread safe
	Add(...[]byte)

	// Check checks filter, assumed thread safe
	Check([]byte) bool
}

func New(maxN uint64, p float64, maxBytes uint64) (Filter, error) {
	neededBytes := bytesSteakKnifeFilter(maxN, p)
	if neededBytes > maxBytes {
		return nil, ErrMaxBytes
	}
	return newSteakKnifeFilter(maxN, p)
}

type steakKnifeFilter struct {
	lock   sync.RWMutex
	filter *streakKnife.Filter
}

func bytesSteakKnifeFilter(maxN uint64, p float64) uint64 {
	m := streakKnife.OptimalM(maxN, p)
	k := streakKnife.OptimalK(m, maxN)

	// This is pulled from bloomFilter.newBits and bloomfilter.newRandKeys. The
	// calculation is the size of the bitset which would be created from this
	// filter.
	mSize := (m + 63) / 64
	totalSize := mSize + k

	return totalSize * 8 // 8 == sizeof(uint64))
}

func newSteakKnifeFilter(maxN uint64, p float64) (Filter, error) {
	m := streakKnife.OptimalM(maxN, p)
	k := streakKnife.OptimalK(m, maxN)

	filter, err := streakKnife.New(m, k)
	return &steakKnifeFilter{filter: filter}, err
}

func (f *steakKnifeFilter) Add(bl ...[]byte) {
	f.lock.Lock()
	defer f.lock.Unlock()

	for _, b := range bl {
		h := murmur3.New64()
		_, _ = h.Write(b)
		f.filter.Add(h)
	}
}

func (f *steakKnifeFilter) Check(b []byte) bool {
	f.lock.RLock()
	defer f.lock.RUnlock()

	h := murmur3.New64()
	_, _ = h.Write(b)
	return f.filter.Contains(h)
}
<|endoftext|>


<commit_before>/*
  Copyright 2012 Sergey Cherepanov (https://github.com/cheggaaa)

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
*/

package anteater

import (
	"net/http"
	"fmt"
	"time"
	"sync"
	"log"
	"mime"
)

const (
	VERSION   = "0.06.2 Narr8"
	SERVER_SIGN = "Anteater " + VERSION
)

/**
 * Path to index file
 **/
var IndexPath string = "file.index"

/**
 * Path to data files
 **/
var DataPath  string = "file.data"

/**
 * Config object
 */
var Conf *Config

/**
 * For Container.Id creation
 */
var ContainerLastId int32

/**
 * Map with container objects
 */
var FileContainers map[int32]*Container = make(map[int32]*Container)

var Containers map[int32]*Container = make(map[int32]*Container)

/**
 *	Mutex for allocate new files
 */
var GetFileLock *sync.Mutex = &sync.Mutex{}

/**
 * File info index
 */
var Index map[string]*FileInfo

/**
 * Lock for Index
 */
var IndexLock *sync.Mutex = &sync.Mutex{}

/**
 * Logger object
 */
var Log *AntLog

/**
 * Server start time
 */
var StartTime time.Time = time.Now()

/**
 * Time of last dump
 */
var LastDump time.Time = time.Now()

/**
 * Making dump time
 */
var LastDumpTime time.Duration

/**
 * Size of index file
 */
var IndexFileSize int64

/**
 * Metrics
 */
var HttpCn  *StateHttpCounters           = &StateHttpCounters{}
var AllocCn *StateAllocateCounters       = &StateAllocateCounters{}

func MainInit(config string) {
	// Init config
	var err error
	Conf, err = LoadConfig(config)
	if err != nil {
		log.Fatal(err)
	}

	// Init logger
	Log, err = LogInit()
	if err != nil {
		log.Fatal(err)
	}

	// Set paths
	IndexPath = Conf.DataPath + "/" + IndexPath
	DataPath = Conf.DataPath + "/" + DataPath

	// Load data from index
	err = LoadData(IndexPath)
	if err != nil {
		// or create new
		Log.Debugln("Error while reading index file:", err)
		Log.Debugln("Try create conainer")
		_, err := NewContainer(DataPath)
		if err != nil {
			Log.Warnln("Can't create new container")
			Log.Fatal(err)
		}
		Cleanup()
	}

	go func() {
		ch := time.Tick(60 * time.Second)
		for _ = range ch {
			func () {
				Cleanup()
			}()
		}
	}()

	RegisterMime()

	Log.Infoln("Start server with config", config)
}

func Start() {
	StartRpcServer()
	if Conf.HttpReadAddr != Conf.HttpWriteAddr {
		go RunServer(http.HandlerFunc(HttpRead), Conf.HttpReadAddr)
	}
	RunServer(http.HandlerFunc(HttpReadWrite), Conf.HttpWriteAddr)
}

func Stop() {
	Log.Infoln("Server stopping..")
	fmt.Println("Server stopping now")
	Cleanup()
	for _, c := range(FileContainers) {
		c.Disable()
		c.F.Close()
	}
	fmt.Println("Bye")
}

func Cleanup() {
	var maxSpace int64
	var hasChanges bool

	for _, c := range(FileContainers) {
		if c.HasChanges() {
			hasChanges = true
		}
		c.Clean()
		if c.MaxSpace() > maxSpace {
			maxSpace = c.MaxSpace()
		}
	}

	if maxSpace <= Conf.MinEmptySpace {
		_, err := NewContainer(DataPath)
		if err != nil {
			Log.Warnln(err)
		}
	}

	if hasChanges {
		err := DumpData(IndexPath)
		if err != nil {
			Log.Infoln("Dump error:", err)
		}
	}
}

/**
 * Register Mime types from config
 */
func RegisterMime() {
	types := Conf.MimeTypes
	for k, v := range types {
		mime.AddExtensionType(k, v)
	}
}
<commit_msg>New version<commit_after>/*
  Copyright 2012 Sergey Cherepanov (https://github.com/cheggaaa)

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
*/

package anteater

import (
	"net/http"
	"fmt"
	"time"
	"sync"
	"log"
	"mime"
)

const (
	VERSION   = "0.06.4 Narr8"
	SERVER_SIGN = "Anteater " + VERSION
)

/**
 * Path to index file
 **/
var IndexPath string = "file.index"

/**
 * Path to data files
 **/
var DataPath  string = "file.data"

/**
 * Config object
 */
var Conf *Config

/**
 * For Container.Id creation
 */
var ContainerLastId int32

/**
 * Map with container objects
 */
var FileContainers map[int32]*Container = make(map[int32]*Container)

var Containers map[int32]*Container = make(map[int32]*Container)

/**
 *	Mutex for allocate new files
 */
var GetFileLock *sync.Mutex = &sync.Mutex{}

/**
 * File info index
 */
var Index map[string]*FileInfo

/**
 * Lock for Index
 */
var IndexLock *sync.Mutex = &sync.Mutex{}

/**
 * Logger object
 */
var Log *AntLog

/**
 * Server start time
 */
var StartTime time.Time = time.Now()

/**
 * Time of last dump
 */
var LastDump time.Time = time.Now()

/**
 * Making dump time
 */
var LastDumpTime time.Duration

/**
 * Size of index file
 */
var IndexFileSize int64

/**
 * Metrics
 */
var HttpCn  *StateHttpCounters           = &StateHttpCounters{}
var AllocCn *StateAllocateCounters       = &StateAllocateCounters{}

func MainInit(config string) {
	// Init config
	var err error
	Conf, err = LoadConfig(config)
	if err != nil {
		log.Fatal(err)
	}

	// Init logger
	Log, err = LogInit()
	if err != nil {
		log.Fatal(err)
	}

	// Set paths
	IndexPath = Conf.DataPath + "/" + IndexPath
	DataPath = Conf.DataPath + "/" + DataPath

	// Load data from index
	err = LoadData(IndexPath)
	if err != nil {
		// or create new
		Log.Debugln("Error while reading index file:", err)
		Log.Debugln("Try create conainer")
		_, err := NewContainer(DataPath)
		if err != nil {
			Log.Warnln("Can't create new container")
			Log.Fatal(err)
		}
		Cleanup()
	}

	go func() {
		ch := time.Tick(60 * time.Second)
		for _ = range ch {
			func () {
				Cleanup()
			}()
		}
	}()

	RegisterMime()

	Log.Infoln("Start server with config", config)
}

func Start() {
	StartRpcServer()
	if Conf.HttpReadAddr != Conf.HttpWriteAddr {
		go RunServer(http.HandlerFunc(HttpRead), Conf.HttpReadAddr)
	}
	RunServer(http.HandlerFunc(HttpReadWrite), Conf.HttpWriteAddr)
}

func Stop() {
	Log.Infoln("Server stopping..")
	fmt.Println("Server stopping now")
	Cleanup()
	for _, c := range(FileContainers) {
		c.Disable()
		c.F.Close()
	}
	fmt.Println("Bye")
}

func Cleanup() {
	var maxSpace int64
	var hasChanges bool

	for _, c := range(FileContainers) {
		if c.HasChanges() {
			hasChanges = true
		}
		c.Clean()
		if c.MaxSpace() > maxSpace {
			maxSpace = c.MaxSpace()
		}
	}

	if maxSpace <= Conf.MinEmptySpace {
		_, err := NewContainer(DataPath)
		if err != nil {
			Log.Warnln(err)
		}
	}

	if hasChanges {
		err := DumpData(IndexPath)
		if err != nil {
			Log.Infoln("Dump error:", err)
		}
	}
}

/**
 * Register Mime types from config
 */
func RegisterMime() {
	types := Conf.MimeTypes
	for k, v := range types {
		mime.AddExtensionType(k, v)
	}
}
<|endoftext|>


<commit_before>/*
 * Copyright 2022 The Knative Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"context"
	"fmt"
	"time"

	"k8s.io/apimachinery/pkg/util/wait"
	"knative.dev/pkg/logging"

	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"knative.dev/pkg/system"
)

const (
	ControlPlaneReadinessCheckInterval = 10 * time.Second
	ControlPlaneReadinessCheckTimeout  = 10 * time.Minute

	NewControllerDeploymentName = "kafka-controller"
)

type kafkaChannelPostMigrationDeleter struct {
	k8s kubernetes.Interface
}

func (d *kafkaChannelPostMigrationDeleter) Delete(ctx context.Context) error {
	// at this stage,
	// - we deleted the old controler
	// - we made the existing channel services to point to new data plane pods
	// - we can now delete the old dispatcher resources and all the other leftovers
	// HOWEVER, we need to check if the new controller is up and running.

	logger := logging.FromContext(ctx)
	logger.Infof("Waiting %s for the new control plane to become ready before the migration.", ControlPlaneReadinessCheckTimeout)

	// wait until the new control plane is ready
	err := d.waitForNewControlPlaneReady(ctx)
	if err != nil {
		return fmt.Errorf("error while waiting the new control plane to become ready %w", err)
	}

	logger.Infof("New control plane is ready, progressing with the migration")

	///////////////////////////////////////////////////////////////////////////////
	/////////// START DELETING DISPATCHER RESOURCES
	///////////////////////////////////////////////////////////////////////////////

	// Delete service knative-eventing/kafka-ch-dispatcher
	const dispatcherServiceName = "kafka-ch-dispatcher"
	err = d.k8s.
		CoreV1().
		Services(system.Namespace()).
		Delete(ctx, dispatcherServiceName, metav1.DeleteOptions{})
	if err != nil && !apierrors.IsNotFound(err) {
		return fmt.Errorf("failed to delete service %s/%s: %w", system.Namespace(), dispatcherServiceName, err)
	}

	// Delete deployment.apps/kafka-ch-dispatcher
	const dispatcherDeploymentName = "kafka-ch-dispatcher"
	err = d.k8s.
		AppsV1().
		Deployments(system.Namespace()).
		Delete(ctx, dispatcherDeploymentName, metav1.DeleteOptions{})
	if err != nil && !apierrors.IsNotFound(err) {
		return fmt.Errorf("failed to delete deployment %s/%s: %w", system.Namespace(), dispatcherDeploymentName, err)
	}

	// Delete clusterrolebinding/kafka-ch-dispatcher
	const dispatcherClusterRoleBinding = "kafka-ch-dispatcher"
	err = d.k8s.
		RbacV1().
		ClusterRoleBindings().
		Delete(ctx, dispatcherClusterRoleBinding, metav1.DeleteOptions{})
	if err != nil && !apierrors.IsNotFound(err) {
		return fmt.Errorf("failed to delete ClusterRoleBinding %s: %w", dispatcherClusterRoleBinding, err)
	}

	// Delete clusterrole/kafka-ch-dispatcher
	const dispatcherClusterRole = "kafka-ch-dispatcher"
	err = d.k8s.
		RbacV1().
		ClusterRoles().
		Delete(ctx, dispatcherClusterRole, metav1.DeleteOptions{})
	if err != nil && !apierrors.IsNotFound(err) {
		return fmt.Errorf("failed to delete ClusterRole %s: %w", dispatcherClusterRole, err)
	}

	// Delete serviceaccount/kafka-ch-dispatcher
	const dispatcherServiceAccount = "kafka-ch-dispatcher"
	err = d.k8s.
		CoreV1().
		ServiceAccounts(system.Namespace()).
		Delete(ctx, dispatcherServiceAccount, metav1.DeleteOptions{})
	if err != nil && !apierrors.IsNotFound(err) {
		return fmt.Errorf("failed to delete service account %s/%s: %w", system.Namespace(), dispatcherServiceAccount, err)
	}

	// Delete configmap/config-leader-election-kafkachannel
	const kafkaChannelLeaderElectionConfigmap = "config-leader-election-kafkachannel"
	err = d.k8s.
		CoreV1().
		ConfigMaps(system.Namespace()).
		Delete(ctx, kafkaChannelLeaderElectionConfigmap, metav1.DeleteOptions{})
	if err != nil && !apierrors.IsNotFound(err) {
		return fmt.Errorf("failed to delete Configmap %s: %w", kafkaChannelLeaderElectionConfigmap, err)
	}

	// Delete lease/kafkachannel-dispatcher.knative.dev.eventing-kafka.pkg.channel.consolidated.reconciler.dispatcher.reconciler.00-of-01
	const kafkaChannelDispatcherLease = "kafkachannel-dispatcher.knative.dev.eventing-kafka.pkg.channel.consolidated.reconciler.dispatcher.reconciler.00-of-01"
	err = d.k8s.
		CoordinationV1().
		Leases(system.Namespace()).
		Delete(ctx, kafkaChannelDispatcherLease, metav1.DeleteOptions{})
	if err != nil && !apierrors.IsNotFound(err) {
		return fmt.Errorf("failed to delete Lease %s: %w", kafkaChannelDispatcherLease, err)
	}

	return nil
}

func (d *kafkaChannelPostMigrationDeleter) waitForNewControlPlaneReady(ctx context.Context) error {
	return wait.PollImmediate(ControlPlaneReadinessCheckInterval, ControlPlaneReadinessCheckTimeout, func() (bool, error) {
		return isDeploymentReady(ctx, d.k8s, system.Namespace(), NewControllerDeploymentName)
	})
}
<commit_msg>Wait 10 mins before deleting the consolidated channel dataplane (#2133)<commit_after>/*
 * Copyright 2022 The Knative Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"context"
	"fmt"
	"time"

	"k8s.io/apimachinery/pkg/util/wait"
	"knative.dev/pkg/logging"

	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"knative.dev/pkg/system"
)

const (
	ControlPlaneReadinessCheckInterval = 10 * time.Second
	ControlPlaneReadinessCheckTimeout  = 10 * time.Minute

	WaitDurationBeforePostMigration = 10 * time.Minute

	NewControllerDeploymentName = "kafka-controller"
)

type kafkaChannelPostMigrationDeleter struct {
	k8s kubernetes.Interface
}

func (d *kafkaChannelPostMigrationDeleter) Delete(ctx context.Context) error {
	// at this stage,
	// - we deleted the old controler
	// - we made the existing channel services to point to new data plane pods
	// - we can now delete the old dispatcher resources and all the other leftovers
	// HOWEVER, we need to check if the new controller is up and running.

	logger := logging.FromContext(ctx)
	logger.Infof("Waiting %s for the new control plane to become ready before the migration.", ControlPlaneReadinessCheckTimeout)

	// wait until the new control plane is ready
	err := d.waitForNewControlPlaneReady(ctx)
	if err != nil {
		return fmt.Errorf("error while waiting the new control plane to become ready %w", err)
	}

	logger.Infof("New control plane is ready, waiting %s before deleting old data plane", WaitDurationBeforePostMigration)
	time.Sleep(WaitDurationBeforePostMigration)
	logger.Infof("Done waiting %s. Deleting old data plane...", WaitDurationBeforePostMigration)

	///////////////////////////////////////////////////////////////////////////////
	/////////// START DELETING DISPATCHER RESOURCES
	///////////////////////////////////////////////////////////////////////////////

	// Delete service knative-eventing/kafka-ch-dispatcher
	const dispatcherServiceName = "kafka-ch-dispatcher"
	err = d.k8s.
		CoreV1().
		Services(system.Namespace()).
		Delete(ctx, dispatcherServiceName, metav1.DeleteOptions{})
	if err != nil && !apierrors.IsNotFound(err) {
		return fmt.Errorf("failed to delete service %s/%s: %w", system.Namespace(), dispatcherServiceName, err)
	}

	// Delete deployment.apps/kafka-ch-dispatcher
	const dispatcherDeploymentName = "kafka-ch-dispatcher"
	err = d.k8s.
		AppsV1().
		Deployments(system.Namespace()).
		Delete(ctx, dispatcherDeploymentName, metav1.DeleteOptions{})
	if err != nil && !apierrors.IsNotFound(err) {
		return fmt.Errorf("failed to delete deployment %s/%s: %w", system.Namespace(), dispatcherDeploymentName, err)
	}

	// Delete clusterrolebinding/kafka-ch-dispatcher
	const dispatcherClusterRoleBinding = "kafka-ch-dispatcher"
	err = d.k8s.
		RbacV1().
		ClusterRoleBindings().
		Delete(ctx, dispatcherClusterRoleBinding, metav1.DeleteOptions{})
	if err != nil && !apierrors.IsNotFound(err) {
		return fmt.Errorf("failed to delete ClusterRoleBinding %s: %w", dispatcherClusterRoleBinding, err)
	}

	// Delete clusterrole/kafka-ch-dispatcher
	const dispatcherClusterRole = "kafka-ch-dispatcher"
	err = d.k8s.
		RbacV1().
		ClusterRoles().
		Delete(ctx, dispatcherClusterRole, metav1.DeleteOptions{})
	if err != nil && !apierrors.IsNotFound(err) {
		return fmt.Errorf("failed to delete ClusterRole %s: %w", dispatcherClusterRole, err)
	}

	// Delete serviceaccount/kafka-ch-dispatcher
	const dispatcherServiceAccount = "kafka-ch-dispatcher"
	err = d.k8s.
		CoreV1().
		ServiceAccounts(system.Namespace()).
		Delete(ctx, dispatcherServiceAccount, metav1.DeleteOptions{})
	if err != nil && !apierrors.IsNotFound(err) {
		return fmt.Errorf("failed to delete service account %s/%s: %w", system.Namespace(), dispatcherServiceAccount, err)
	}

	// Delete configmap/config-leader-election-kafkachannel
	const kafkaChannelLeaderElectionConfigmap = "config-leader-election-kafkachannel"
	err = d.k8s.
		CoreV1().
		ConfigMaps(system.Namespace()).
		Delete(ctx, kafkaChannelLeaderElectionConfigmap, metav1.DeleteOptions{})
	if err != nil && !apierrors.IsNotFound(err) {
		return fmt.Errorf("failed to delete Configmap %s: %w", kafkaChannelLeaderElectionConfigmap, err)
	}

	// Delete lease/kafkachannel-dispatcher.knative.dev.eventing-kafka.pkg.channel.consolidated.reconciler.dispatcher.reconciler.00-of-01
	const kafkaChannelDispatcherLease = "kafkachannel-dispatcher.knative.dev.eventing-kafka.pkg.channel.consolidated.reconciler.dispatcher.reconciler.00-of-01"
	err = d.k8s.
		CoordinationV1().
		Leases(system.Namespace()).
		Delete(ctx, kafkaChannelDispatcherLease, metav1.DeleteOptions{})
	if err != nil && !apierrors.IsNotFound(err) {
		return fmt.Errorf("failed to delete Lease %s: %w", kafkaChannelDispatcherLease, err)
	}

	return nil
}

func (d *kafkaChannelPostMigrationDeleter) waitForNewControlPlaneReady(ctx context.Context) error {
	return wait.PollImmediate(ControlPlaneReadinessCheckInterval, ControlPlaneReadinessCheckTimeout, func() (bool, error) {
		return isDeploymentReady(ctx, d.k8s, system.Namespace(), NewControllerDeploymentName)
	})
}
<|endoftext|>