<commit_before>// Package hyperloglog implements the HyperLogLog algorithm for
// cardinality estimation. In English: it counts things. It counts
// things using very small amounts of memory compared to the number of
// objects it is counting.
//
// For a full description of the algorithm, see the paper HyperLogLog:
// the analysis of a near-optimal cardinality estimation algorithm by
// Flajolet, et. al.
package hyperloglog

import (
	"fmt"
	"math"
)

const (
	pow_2_32     = 4294967296.0
	neg_pow_2_32 = -4294967296.0
)

type HyperLogLog struct {
	m         uint    // Number of registers
	b         uint32  // Number of bits used to determine register index
	alpha     float64 // Bias correction constant
	registers []uint8
}

// Compute bias correction alpha_m.
func get_alpha(m uint) (result float64) {
	switch m {
	case 16:
		result = 0.673
	case 32:
		result = 0.697
	case 64:
		result = 0.709
	default:
		result = 0.7213 / (1.0 + 1.079/float64(m))
	}
	return result
}

// Return a new HyperLogLog with the given number of registers. More
// registers leads to lower error in your estimated count, at the
// expense of memory.
//
// Choose a power of two number of registers, depending on the amount
// of memory you're willing to use and the error you're willing to
// tolerate. Each register uses one byte of memory.
//
// Approximate error will be:
//     1.04 / sqrt(registers)
//
func New(registers uint) (*HyperLogLog, error) {
	if registers%2 != 0 {
		return nil, fmt.Errorf("number of registers %d not a power of two", registers)
	}
	h := &HyperLogLog{}
	h.m = registers
	h.b = uint32(math.Ceil(math.Log2(float64(registers))))
	h.alpha = get_alpha(registers)
	h.Reset()
	return h, nil
}

// Reset all internal variables and set the count to zero.
func (h *HyperLogLog) Reset() {
	h.registers = make([]uint8, h.m)
}

// Calculate the position of the leftmost 1-bit.
func rho(val uint32, max uint32) uint8 {
	r := uint32(1)
	for val&0x80000000 == 0 && r <= max {
		r++
		val <<= 1
	}
	return uint8(r)
}

// Add to the count. val should be a 32 bit unsigned integer from a
// good hash function.
func (h *HyperLogLog) Add(val uint32) {
	k := 32 - h.b
	r := rho(val<<h.b, k)
	j := val >> uint(k)
	if r > h.registers[j] {
		h.registers[j] = r
	}
}

// Get the estimated count.
func (h *HyperLogLog) Count() uint64 {
	sum := 0.0
	for _, val := range h.registers {
		sum += 1.0 / math.Pow(2.0, float64(val))
	}
	estimate := h.alpha * float64(h.m*h.m) / sum
	if estimate <= 5.0/2.0*float64(h.m) {
		// Small range correction
		v := 0
		for _, r := range h.registers {
			if r == 0 {
				v++
			}
		}
		if v > 0 {
			estimate = float64(h.m) * math.Log(float64(h.m)/float64(v))
		}
	} else if estimate > 1.0/30.0*pow_2_32 {
		// Large range correction
		estimate = neg_pow_2_32 * math.Log(1-estimate/pow_2_32)
	}
	return uint64(estimate)
}
<commit_msg>Don't take ceiling of log2 of registers<commit_after>// Package hyperloglog implements the HyperLogLog algorithm for
// cardinality estimation. In English: it counts things. It counts
// things using very small amounts of memory compared to the number of
// objects it is counting.
//
// For a full description of the algorithm, see the paper HyperLogLog:
// the analysis of a near-optimal cardinality estimation algorithm by
// Flajolet, et. al.
package hyperloglog

import (
	"fmt"
	"math"
)

const (
	pow_2_32     = 4294967296.0
	neg_pow_2_32 = -4294967296.0
)

type HyperLogLog struct {
	m         uint    // Number of registers
	b         uint32  // Number of bits used to determine register index
	alpha     float64 // Bias correction constant
	registers []uint8
}

// Compute bias correction alpha_m.
func get_alpha(m uint) (result float64) {
	switch m {
	case 16:
		result = 0.673
	case 32:
		result = 0.697
	case 64:
		result = 0.709
	default:
		result = 0.7213 / (1.0 + 1.079/float64(m))
	}
	return result
}

// Return a new HyperLogLog with the given number of registers. More
// registers leads to lower error in your estimated count, at the
// expense of memory.
//
// Choose a power of two number of registers, depending on the amount
// of memory you're willing to use and the error you're willing to
// tolerate. Each register uses one byte of memory.
//
// Approximate error will be:
//     1.04 / sqrt(registers)
//
func New(registers uint) (*HyperLogLog, error) {
	if registers%2 != 0 {
		return nil, fmt.Errorf("number of registers %d not a power of two", registers)
	}
	h := &HyperLogLog{}
	h.m = registers
	h.b = uint32(math.Log2(float64(registers)))
	h.alpha = get_alpha(registers)
	h.Reset()
	return h, nil
}

// Reset all internal variables and set the count to zero.
func (h *HyperLogLog) Reset() {
	h.registers = make([]uint8, h.m)
}

// Calculate the position of the leftmost 1-bit.
func rho(val uint32, max uint32) uint8 {
	r := uint32(1)
	for val&0x80000000 == 0 && r <= max {
		r++
		val <<= 1
	}
	return uint8(r)
}

// Add to the count. val should be a 32 bit unsigned integer from a
// good hash function.
func (h *HyperLogLog) Add(val uint32) {
	k := 32 - h.b
	r := rho(val<<h.b, k)
	j := val >> uint(k)
	if r > h.registers[j] {
		h.registers[j] = r
	}
}

// Get the estimated count.
func (h *HyperLogLog) Count() uint64 {
	sum := 0.0
	for _, val := range h.registers {
		sum += 1.0 / math.Pow(2.0, float64(val))
	}
	estimate := h.alpha * float64(h.m*h.m) / sum
	if estimate <= 5.0/2.0*float64(h.m) {
		// Small range correction
		v := 0
		for _, r := range h.registers {
			if r == 0 {
				v++
			}
		}
		if v > 0 {
			estimate = float64(h.m) * math.Log(float64(h.m)/float64(v))
		}
	} else if estimate > 1.0/30.0*pow_2_32 {
		// Large range correction
		estimate = neg_pow_2_32 * math.Log(1-estimate/pow_2_32)
	}
	return uint64(estimate)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
)

var outputDir string

func main() {
	const (
		defaultOutputDir = "code"
		outputDirUsage   = "Directory where to put the generated source"
	)

	log.Printf("Starting ms-gen...\n")

	flag.StringVar(&outputDir, "output", defaultOutputDir, outputDirUsage)
	flag.StringVar(&outputDir, "o", defaultOutputDir, outputDirUsage+" (shorthand)")

	flag.Parse()

	log.Printf("Configuration: output=%s", outputDir)

	api := NewApi("../generator_tests/api_sample_1.yaml", outputDir)
	if err := api.LoadFromSwagger(); err != nil {
		log.Fatalf("When loading swagger file %s: %s", api.Filename, err)
	}

	projectSource := NewSource(api)
	if err := projectSource.Build(); err != nil {
		log.Fatalf("When build source files: %s", err)
	}

	log.Printf("Finishing ms-gen...\n")
}
<commit_msg>Config -a or -api to specify swagger definition file.<commit_after>package main

import (
	"flag"
	"log"
)

var (
	outputDir string
	apiFile   string
)

func main() {
	const (
		// Output dir flag.
		defaultOutputDir = "code"
		outputDirUsage   = "Directory where to put the generated source"

		// API file flag.
		apiFileUsage = "File that defines the API"
	)

	log.Printf("Starting ms-gen...\n")

	flag.StringVar(&outputDir, "output", defaultOutputDir, outputDirUsage)
	flag.StringVar(&outputDir, "o", defaultOutputDir, outputDirUsage+" (shorthand)")

	flag.StringVar(&apiFile, "api", "", apiFileUsage)
	flag.StringVar(&apiFile, "a", "", apiFileUsage+" (shorthand)")

	flag.Parse()

	log.Printf("Configuration: output='%s' api='%s'", outputDir, apiFile)

	if apiFile == "" {
		log.Fatalf("Parameter 'api' must be informed")
	}

	api := NewApi(apiFile, outputDir)
	if err := api.LoadFromSwagger(); err != nil {
		log.Fatalf("When loading swagger file %s: %s", api.Filename, err)
	}

	projectSource := NewSource(api)
	if err := projectSource.Build(); err != nil {
		log.Fatalf("When build source files: %s", err)
	}

	log.Printf("Finishing ms-gen...\n")
}
<|endoftext|>


<commit_before>package api

import (
	"net/http"

	"github.com/gin-gonic/gin"

	"strings"

	"encoding/json"
	"os"

	"io/ioutil"

	"github.com/mundipagg/boleto-api/bank"
	"github.com/mundipagg/boleto-api/boleto"
	"github.com/mundipagg/boleto-api/config"
	"github.com/mundipagg/boleto-api/db"
	"github.com/mundipagg/boleto-api/log"
	"github.com/mundipagg/boleto-api/models"
)

//Regista um boleto em um determinado banco
func registerBoleto(c *gin.Context) {
	if _, hasErr := c.Get("error"); hasErr {
		return
	}
	_boleto, _ := c.Get("boleto")
	_bank, _ := c.Get("bank")
	boleto := _boleto.(models.BoletoRequest)
	bank := _bank.(bank.Bank)
	lg := bank.Log()
	lg.Operation = "RegisterBoleto"
	lg.NossoNumero = boleto.Title.OurNumber
	lg.Recipient = boleto.Recipient.Name
	lg.RequestKey = boleto.RequestKey
	lg.BankName = bank.GetBankNameIntegration()

	repo, errDb := db.GetDB()
	if checkError(c, errDb, lg) {
		return
	}
	resp, errR := bank.ProcessBoleto(&boleto)
	if checkError(c, errR, lg) {
		return
	}
	st := http.StatusOK
	if len(resp.Errors) > 0 {
		if resp.StatusCode > 0 {
			st = resp.StatusCode
		} else {
			st = http.StatusBadRequest
		}
	} else {
		boView := models.NewBoletoView(boleto, resp, bank.GetBankNameIntegration())
		idBson, _ := boView.ID.MarshalText()
		resp.ID = string(idBson)

		resp.Links = boView.Links
		errMongo := repo.SaveBoleto(boView)
		if errMongo != nil {
			saveBoletoJSONFile(boView, lg, errMongo)
		}
	}
	c.JSON(st, resp)
	c.Set("boletoResponse", resp)
}

func saveBoletoJSONFile(boView models.BoletoView, lg *log.Log, err error) {
	lg.Warn(err.Error(), "Boleto cannot be saved at Database")
	fd, errOpen := os.Create(config.Get().BoletoJSONFileStore + "/boleto_" + boView.UID + ".json")
	if errOpen != nil {
		lg.Fatal(boView, "[BOLETO_ONLINE_CONTINGENCIA]"+errOpen.Error())
	}
	data, _ := json.Marshal(boView)
	_, errW := fd.Write(data)
	if errW != nil {
		lg.Fatal(boView, "[BOLETO_ONLINE_CONTINGENCIA]"+errW.Error())
	}
	fd.Close()
}

func getBoleto(c *gin.Context) {
	c.Status(200)

	id := c.Query("id")
	format := c.Query("fmt")
	repo, errCon := db.GetDB()
	if checkError(c, errCon, log.CreateLog()) {
		return
	}
	_boleto, err := repo.GetBoletoByID(id)
	if err != nil {
		uid := id
		fd, err := os.Open(config.Get().BoletoJSONFileStore + "/boleto_" + uid + ".json")
		if err != nil {
			checkError(c, models.NewHTTPNotFound("Boleto não encontrado na base de dados", "MP404"), log.CreateLog())
			return
		}
		data, errR := ioutil.ReadAll(fd)
		if errR != nil {
			checkError(c, models.NewHTTPNotFound("Boleto não encontrado na base de dados", "MP404"), log.CreateLog())
			return
		}
		json.Unmarshal(data, &_boleto)
		fd.Close()
	}

	s, err := boleto.HTML(_boleto, format)
	if checkError(c, err, log.CreateLog()) {
		return
	}
	if format == "html" {
		c.Header("Content-Type", "text/html; charset=utf-8")
		c.Writer.WriteString(s)
	} else {
		c.Header("Content-Type", "application/pdf")

		buf, err := toPdf(s)

		if err != nil {
			c.Header("Content-Type", "application/json")
			checkError(c, models.NewInternalServerError(err.Error(), "internal error"), log.CreateLog())
			c.Abort()
		} else {
			c.Writer.Write(buf)
		}

	}

}

func toPdf(page string) ([]byte, error) {
	url := config.Get().PdfAPIURL
	payload := strings.NewReader(page)
	if req, err := http.NewRequest("POST", url, payload); err != nil {
		return nil, err
	} else if res, err := http.DefaultClient.Do(req); err != nil {
		return nil, err
	} else {
		defer res.Body.Close()
		return ioutil.ReadAll(res.Body)
	}
}

func getBoletoByID(c *gin.Context) {
	id := c.Param("id")
	db, errDb := db.GetDB()
	if errDb != nil {
		checkError(c, models.NewInternalServerError("MP500", "Erro interno"), log.CreateLog())
	}
	boleto, err := db.GetBoletoByID(id)
	if err != nil {
		checkError(c, models.NewHTTPNotFound("MP404", "Boleto não encontrado"), nil)
		return
	}
	c.JSON(http.StatusOK, boleto)
}
<commit_msg>:art: Mapeia TO.DO<commit_after>package api

import (
	"net/http"

	"github.com/gin-gonic/gin"

	"strings"

	"encoding/json"
	"os"

	"io/ioutil"

	"github.com/mundipagg/boleto-api/bank"
	"github.com/mundipagg/boleto-api/boleto"
	"github.com/mundipagg/boleto-api/config"
	"github.com/mundipagg/boleto-api/db"
	"github.com/mundipagg/boleto-api/log"
	"github.com/mundipagg/boleto-api/models"
)

//Regista um boleto em um determinado banco
func registerBoleto(c *gin.Context) {
	if _, hasErr := c.Get("error"); hasErr {
		return
	}
	_boleto, _ := c.Get("boleto")
	_bank, _ := c.Get("bank")
	boleto := _boleto.(models.BoletoRequest)
	bank := _bank.(bank.Bank)
	lg := bank.Log()
	lg.Operation = "RegisterBoleto"
	lg.NossoNumero = boleto.Title.OurNumber
	lg.Recipient = boleto.Recipient.Name
	lg.RequestKey = boleto.RequestKey
	lg.BankName = bank.GetBankNameIntegration()

	repo, errDb := db.GetDB()
	if checkError(c, errDb, lg) {
		return
	}

	resp, errR := bank.ProcessBoleto(&boleto)
	if checkError(c, errR, lg) {
		return
	}
	st := http.StatusOK
	if len(resp.Errors) > 0 {
		if resp.StatusCode > 0 {
			st = resp.StatusCode
		} else {
			st = http.StatusBadRequest
		}
	} else {
		// MONTAR HTML
		// SALVAR REDIS

		boView := models.NewBoletoView(boleto, resp, bank.GetBankNameIntegration())
		idBson, _ := boView.ID.MarshalText()
		resp.ID = string(idBson)

		resp.Links = boView.Links
		errMongo := repo.SaveBoleto(boView)
		if errMongo != nil {
			saveBoletoJSONFile(boView, lg, errMongo)
		}
	}
	c.JSON(st, resp)
	c.Set("boletoResponse", resp)
}

func saveBoletoJSONFile(boView models.BoletoView, lg *log.Log, err error) {
	lg.Warn(err.Error(), "Boleto cannot be saved at Database")
	fd, errOpen := os.Create(config.Get().BoletoJSONFileStore + "/boleto_" + boView.UID + ".json")
	if errOpen != nil {
		lg.Fatal(boView, "[BOLETO_ONLINE_CONTINGENCIA]"+errOpen.Error())
	}
	data, _ := json.Marshal(boView)
	_, errW := fd.Write(data)
	if errW != nil {
		lg.Fatal(boView, "[BOLETO_ONLINE_CONTINGENCIA]"+errW.Error())
	}
	fd.Close()
}

func getBoleto(c *gin.Context) {
	c.Status(200)

	id := c.Query("id")
	format := c.Query("fmt")
	repo, errCon := db.GetDB()
	if checkError(c, errCon, log.CreateLog()) {
		return
	}
	_boleto, err := repo.GetBoletoByID(id)
	if err != nil {
		uid := id
		fd, err := os.Open(config.Get().BoletoJSONFileStore + "/boleto_" + uid + ".json")
		if err != nil {
			checkError(c, models.NewHTTPNotFound("Boleto não encontrado na base de dados", "MP404"), log.CreateLog())
			return
		}
		data, errR := ioutil.ReadAll(fd)
		if errR != nil {
			checkError(c, models.NewHTTPNotFound("Boleto não encontrado na base de dados", "MP404"), log.CreateLog())
			return
		}
		json.Unmarshal(data, &_boleto)
		fd.Close()
	}

	s, err := boleto.HTML(_boleto, format)
	if checkError(c, err, log.CreateLog()) {
		return
	}
	if format == "html" {
		c.Header("Content-Type", "text/html; charset=utf-8")
		c.Writer.WriteString(s)
	} else {
		c.Header("Content-Type", "application/pdf")

		buf, err := toPdf(s)

		if err != nil {
			c.Header("Content-Type", "application/json")
			checkError(c, models.NewInternalServerError(err.Error(), "internal error"), log.CreateLog())
			c.Abort()
		} else {
			c.Writer.Write(buf)
		}

	}

}

func toPdf(page string) ([]byte, error) {
	url := config.Get().PdfAPIURL
	payload := strings.NewReader(page)
	if req, err := http.NewRequest("POST", url, payload); err != nil {
		return nil, err
	} else if res, err := http.DefaultClient.Do(req); err != nil {
		return nil, err
	} else {
		defer res.Body.Close()
		return ioutil.ReadAll(res.Body)
	}
}

func getBoletoByID(c *gin.Context) {
	id := c.Param("id")
	db, errDb := db.GetDB()
	if errDb != nil {
		checkError(c, models.NewInternalServerError("MP500", "Erro interno"), log.CreateLog())
	}
	boleto, err := db.GetBoletoByID(id)
	if err != nil {
		checkError(c, models.NewHTTPNotFound("MP404", "Boleto não encontrado"), nil)
		return
	}
	c.JSON(http.StatusOK, boleto)
}
<|endoftext|>


<commit_before>package api

import (
	"crypto/tls"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/hashicorp/go-cleanhttp"
	"github.com/hashicorp/go-rootcerts"
	"github.com/sethgrid/pester"
)

const EnvVaultAddress = "VAULT_ADDR"
const EnvVaultCACert = "VAULT_CACERT"
const EnvVaultCAPath = "VAULT_CAPATH"
const EnvVaultClientCert = "VAULT_CLIENT_CERT"
const EnvVaultClientKey = "VAULT_CLIENT_KEY"
const EnvVaultInsecure = "VAULT_SKIP_VERIFY"
const EnvVaultTLSServerName = "VAULT_TLS_SERVER_NAME"
const EnvVaultWrapTTL = "VAULT_WRAP_TTL"
const EnvVaultMaxRetries = "VAULT_MAX_RETRIES"

// WrappingLookupFunc is a function that, given an HTTP verb and a path,
// returns an optional string duration to be used for response wrapping (e.g.
// "15s", or simply "15"). The path will not begin with "/v1/" or "v1/" or "/",
// however, end-of-path forward slashes are not trimmed, so must match your
// called path precisely.
type WrappingLookupFunc func(operation, path string) string

// Config is used to configure the creation of the client.
type Config struct {
	// Address is the address of the Vault server. This should be a complete
	// URL such as "http://vault.example.com". If you need a custom SSL
	// cert or want to enable insecure mode, you need to specify a custom
	// HttpClient.
	Address string

	// HttpClient is the HTTP client to use, which will currently always have the
	// same values as http.DefaultClient. This is used to control redirect behavior.
	HttpClient *http.Client

	redirectSetup sync.Once

	// MaxRetries controls the maximum number of times to retry when a 5xx error
	// occurs. Set to 0 or less to disable retrying. Defaults to 0.
	MaxRetries int
}

// TLSConfig contains the parameters needed to configure TLS on the HTTP client
// used to communicate with Vault.
type TLSConfig struct {
	// CACert is the path to a PEM-encoded CA cert file to use to verify the
	// Vault server SSL certificate.
	CACert string

	// CAPath is the path to a directory of PEM-encoded CA cert files to verify
	// the Vault server SSL certificate.
	CAPath string

	// ClientCert is the path to the certificate for Vault communication
	ClientCert string

	// ClientKey is the path to the private key for Vault communication
	ClientKey string

	// TLSServerName, if set, is used to set the SNI host when connecting via
	// TLS.
	TLSServerName string

	// Insecure enables or disables SSL verification
	Insecure bool
}

// DefaultConfig returns a default configuration for the client. It is
// safe to modify the return value of this function.
//
// The default Address is https://127.0.0.1:8200, but this can be overridden by
// setting the `VAULT_ADDR` environment variable.
func DefaultConfig() *Config {
	config := &Config{
		Address: "https://127.0.0.1:8200",

		HttpClient: cleanhttp.DefaultClient(),
	}
	config.HttpClient.Timeout = time.Second * 60
	transport := config.HttpClient.Transport.(*http.Transport)
	transport.TLSHandshakeTimeout = 10 * time.Second
	transport.TLSClientConfig = &tls.Config{
		MinVersion: tls.VersionTLS12,
	}

	if v := os.Getenv(EnvVaultAddress); v != "" {
		config.Address = v
	}

	return config
}

// ConfigureTLS takes a set of TLS configurations and applies those to the the HTTP client.
func (c *Config) ConfigureTLS(t *TLSConfig) error {

	if c.HttpClient == nil {
		return fmt.Errorf("config HTTP Client must be set")
	}

	var clientCert tls.Certificate
	foundClientCert := false
	if t.CACert != "" || t.CAPath != "" || t.ClientCert != "" || t.ClientKey != "" || t.Insecure {
		if t.ClientCert != "" && t.ClientKey != "" {
			var err error
			clientCert, err = tls.LoadX509KeyPair(t.ClientCert, t.ClientKey)
			if err != nil {
				return err
			}
			foundClientCert = true
		} else if t.ClientCert != "" || t.ClientKey != "" {
			return fmt.Errorf("Both client cert and client key must be provided")
		}
	}

	clientTLSConfig := c.HttpClient.Transport.(*http.Transport).TLSClientConfig
	rootConfig := &rootcerts.Config{
		CAFile: t.CACert,
		CAPath: t.CAPath,
	}
	if err := rootcerts.ConfigureTLS(clientTLSConfig, rootConfig); err != nil {
		return err
	}

	clientTLSConfig.InsecureSkipVerify = t.Insecure

	if foundClientCert {
		clientTLSConfig.Certificates = []tls.Certificate{clientCert}
	}
	if t.TLSServerName != "" {
		clientTLSConfig.ServerName = t.TLSServerName
	}

	return nil
}

// ReadEnvironment reads configuration information from the
// environment. If there is an error, no configuration value
// is updated.
func (c *Config) ReadEnvironment() error {
	var envAddress string
	var envCACert string
	var envCAPath string
	var envClientCert string
	var envClientKey string
	var envInsecure bool
	var envTLSServerName string
	var envMaxRetries *uint64

	// Parse the environment variables
	if v := os.Getenv(EnvVaultAddress); v != "" {
		envAddress = v
	}
	if v := os.Getenv(EnvVaultMaxRetries); v != "" {
		maxRetries, err := strconv.ParseUint(v, 10, 32)
		if err != nil {
			return err
		}
		envMaxRetries = &maxRetries
	}
	if v := os.Getenv(EnvVaultCACert); v != "" {
		envCACert = v
	}
	if v := os.Getenv(EnvVaultCAPath); v != "" {
		envCAPath = v
	}
	if v := os.Getenv(EnvVaultClientCert); v != "" {
		envClientCert = v
	}
	if v := os.Getenv(EnvVaultClientKey); v != "" {
		envClientKey = v
	}
	if v := os.Getenv(EnvVaultInsecure); v != "" {
		var err error
		envInsecure, err = strconv.ParseBool(v)
		if err != nil {
			return fmt.Errorf("Could not parse VAULT_SKIP_VERIFY")
		}
	}
	if v := os.Getenv(EnvVaultTLSServerName); v != "" {
		envTLSServerName = v
	}

	// Configure the HTTP clients TLS configuration.
	t := &TLSConfig{
		CACert:        envCACert,
		CAPath:        envCAPath,
		ClientCert:    envClientCert,
		ClientKey:     envClientKey,
		TLSServerName: envTLSServerName,
		Insecure:      envInsecure,
	}
	if err := c.ConfigureTLS(t); err != nil {
		return err
	}

	if envAddress != "" {
		c.Address = envAddress
	}

	if envMaxRetries != nil {
		c.MaxRetries = int(*envMaxRetries) + 1
	}

	return nil
}

// Client is the client to the Vault API. Create a client with
// NewClient.
type Client struct {
	addr               *url.URL
	config             *Config
	token              string
	wrappingLookupFunc WrappingLookupFunc
}

// NewClient returns a new client for the given configuration.
//
// If the environment variable `VAULT_TOKEN` is present, the token will be
// automatically added to the client. Otherwise, you must manually call
// `SetToken()`.
func NewClient(c *Config) (*Client, error) {
	if c == nil {
		c = DefaultConfig()
		if err := c.ReadEnvironment(); err != nil {
			return nil, fmt.Errorf("error reading environment: %v", err)
		}
	}

	u, err := url.Parse(c.Address)
	if err != nil {
		return nil, err
	}

	if c.HttpClient == nil {
		c.HttpClient = DefaultConfig().HttpClient
	}

	redirFunc := func() {
		// Ensure redirects are not automatically followed
		// Note that this is sane for the API client as it has its own
		// redirect handling logic (and thus also for command/meta),
		// but in e.g. http_test actual redirect handling is necessary
		c.HttpClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {
			// Returning this value causes the Go net library to not close the
			// response body and nil out the error. Otherwise pester tries
			// three times on every redirect because it sees an error from this
			// function being passed through.
			return http.ErrUseLastResponse
		}
	}

	c.redirectSetup.Do(redirFunc)

	client := &Client{
		addr:   u,
		config: c,
	}

	if token := os.Getenv("VAULT_TOKEN"); token != "" {
		client.SetToken(token)
	}

	return client, nil
}

// Sets the address of Vault in the client. The format of address should be
// "<Scheme>://<Host>:<Port>". Setting this on a client will override the
// value of VAULT_ADDR environment variable.
func (c *Client) SetAddress(addr string) error {
	var err error
	if c.addr, err = url.Parse(addr); err != nil {
		return fmt.Errorf("failed to set address: %v", err)
	}

	return nil
}

// Address returns the Vault URL the client is configured to connect to
func (c *Client) Address() string {
	return c.addr.String()
}

// SetWrappingLookupFunc sets a lookup function that returns desired wrap TTLs
// for a given operation and path
func (c *Client) SetWrappingLookupFunc(lookupFunc WrappingLookupFunc) {
	c.wrappingLookupFunc = lookupFunc
}

// Token returns the access token being used by this client. It will
// return the empty string if there is no token set.
func (c *Client) Token() string {
	return c.token
}

// SetToken sets the token directly. This won't perform any auth
// verification, it simply sets the token properly for future requests.
func (c *Client) SetToken(v string) {
	c.token = v
}

// ClearToken deletes the token if it is set or does nothing otherwise.
func (c *Client) ClearToken() {
	c.token = ""
}

// NewRequest creates a new raw request object to query the Vault server
// configured for this client. This is an advanced method and generally
// doesn't need to be called externally.
func (c *Client) NewRequest(method, path string) *Request {
	req := &Request{
		Method: method,
		URL: &url.URL{
			Scheme: c.addr.Scheme,
			Host:   c.addr.Host,
			Path:   path,
		},
		ClientToken: c.token,
		Params:      make(map[string][]string),
	}

	var lookupPath string
	switch {
	case strings.HasPrefix(path, "/v1/"):
		lookupPath = strings.TrimPrefix(path, "/v1/")
	case strings.HasPrefix(path, "v1/"):
		lookupPath = strings.TrimPrefix(path, "v1/")
	default:
		lookupPath = path
	}
	if c.wrappingLookupFunc != nil {
		req.WrapTTL = c.wrappingLookupFunc(method, lookupPath)
	} else {
		req.WrapTTL = DefaultWrappingLookupFunc(method, lookupPath)
	}

	return req
}

// RawRequest performs the raw request given. This request may be against
// a Vault server not configured with this client. This is an advanced operation
// that generally won't need to be called externally.
func (c *Client) RawRequest(r *Request) (*Response, error) {
	redirectCount := 0
START:
	req, err := r.ToHTTP()
	if err != nil {
		return nil, err
	}

	client := pester.NewExtendedClient(c.config.HttpClient)
	client.Backoff = pester.LinearJitterBackoff
	client.MaxRetries = c.config.MaxRetries

	var result *Response
	resp, err := client.Do(req)
	if resp != nil {
		result = &Response{Response: resp}
	}
	if err != nil {
		if strings.Contains(err.Error(), "tls: oversized") {
			err = fmt.Errorf(
				"%s\n\n"+
					"This error usually means that the server is running with TLS disabled\n"+
					"but the client is configured to use TLS. Please either enable TLS\n"+
					"on the server or run the client with -address set to an address\n"+
					"that uses the http protocol:\n\n"+
					"    vault <command> -address http://<address>\n\n"+
					"You can also set the VAULT_ADDR environment variable:\n\n\n"+
					"    VAULT_ADDR=http://<address> vault <command>\n\n"+
					"where <address> is replaced by the actual address to the server.",
				err)
		}
		return result, err
	}

	// Check for a redirect, only allowing for a single redirect
	if (resp.StatusCode == 301 || resp.StatusCode == 302 || resp.StatusCode == 307) && redirectCount == 0 {
		// Parse the updated location
		respLoc, err := resp.Location()
		if err != nil {
			return result, err
		}

		// Ensure a protocol downgrade doesn't happen
		if req.URL.Scheme == "https" && respLoc.Scheme != "https" {
			return result, fmt.Errorf("redirect would cause protocol downgrade")
		}

		// Update the request
		r.URL = respLoc

		// Reset the request body if any
		if err := r.ResetJSONBody(); err != nil {
			return result, err
		}

		// Retry the request
		redirectCount++
		goto START
	}

	if err := result.Error(); err != nil {
		return result, err
	}

	return result, nil
}
<commit_msg>ConfigureTLS() sets default HttpClient if nil (#2329)<commit_after>package api

import (
	"crypto/tls"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/hashicorp/go-cleanhttp"
	"github.com/hashicorp/go-rootcerts"
	"github.com/sethgrid/pester"
)

const EnvVaultAddress = "VAULT_ADDR"
const EnvVaultCACert = "VAULT_CACERT"
const EnvVaultCAPath = "VAULT_CAPATH"
const EnvVaultClientCert = "VAULT_CLIENT_CERT"
const EnvVaultClientKey = "VAULT_CLIENT_KEY"
const EnvVaultInsecure = "VAULT_SKIP_VERIFY"
const EnvVaultTLSServerName = "VAULT_TLS_SERVER_NAME"
const EnvVaultWrapTTL = "VAULT_WRAP_TTL"
const EnvVaultMaxRetries = "VAULT_MAX_RETRIES"

// WrappingLookupFunc is a function that, given an HTTP verb and a path,
// returns an optional string duration to be used for response wrapping (e.g.
// "15s", or simply "15"). The path will not begin with "/v1/" or "v1/" or "/",
// however, end-of-path forward slashes are not trimmed, so must match your
// called path precisely.
type WrappingLookupFunc func(operation, path string) string

// Config is used to configure the creation of the client.
type Config struct {
	// Address is the address of the Vault server. This should be a complete
	// URL such as "http://vault.example.com". If you need a custom SSL
	// cert or want to enable insecure mode, you need to specify a custom
	// HttpClient.
	Address string

	// HttpClient is the HTTP client to use, which will currently always have the
	// same values as http.DefaultClient. This is used to control redirect behavior.
	HttpClient *http.Client

	redirectSetup sync.Once

	// MaxRetries controls the maximum number of times to retry when a 5xx error
	// occurs. Set to 0 or less to disable retrying. Defaults to 0.
	MaxRetries int
}

// TLSConfig contains the parameters needed to configure TLS on the HTTP client
// used to communicate with Vault.
type TLSConfig struct {
	// CACert is the path to a PEM-encoded CA cert file to use to verify the
	// Vault server SSL certificate.
	CACert string

	// CAPath is the path to a directory of PEM-encoded CA cert files to verify
	// the Vault server SSL certificate.
	CAPath string

	// ClientCert is the path to the certificate for Vault communication
	ClientCert string

	// ClientKey is the path to the private key for Vault communication
	ClientKey string

	// TLSServerName, if set, is used to set the SNI host when connecting via
	// TLS.
	TLSServerName string

	// Insecure enables or disables SSL verification
	Insecure bool
}

// DefaultConfig returns a default configuration for the client. It is
// safe to modify the return value of this function.
//
// The default Address is https://127.0.0.1:8200, but this can be overridden by
// setting the `VAULT_ADDR` environment variable.
func DefaultConfig() *Config {
	config := &Config{
		Address: "https://127.0.0.1:8200",

		HttpClient: cleanhttp.DefaultClient(),
	}
	config.HttpClient.Timeout = time.Second * 60
	transport := config.HttpClient.Transport.(*http.Transport)
	transport.TLSHandshakeTimeout = 10 * time.Second
	transport.TLSClientConfig = &tls.Config{
		MinVersion: tls.VersionTLS12,
	}

	if v := os.Getenv(EnvVaultAddress); v != "" {
		config.Address = v
	}

	return config
}

// ConfigureTLS takes a set of TLS configurations and applies those to the the HTTP client.
func (c *Config) ConfigureTLS(t *TLSConfig) error {

	if c.HttpClient == nil {
		c.HttpClient = DefaultConfig().HttpClient
	}

	var clientCert tls.Certificate
	foundClientCert := false
	if t.CACert != "" || t.CAPath != "" || t.ClientCert != "" || t.ClientKey != "" || t.Insecure {
		if t.ClientCert != "" && t.ClientKey != "" {
			var err error
			clientCert, err = tls.LoadX509KeyPair(t.ClientCert, t.ClientKey)
			if err != nil {
				return err
			}
			foundClientCert = true
		} else if t.ClientCert != "" || t.ClientKey != "" {
			return fmt.Errorf("Both client cert and client key must be provided")
		}
	}

	clientTLSConfig := c.HttpClient.Transport.(*http.Transport).TLSClientConfig
	rootConfig := &rootcerts.Config{
		CAFile: t.CACert,
		CAPath: t.CAPath,
	}
	if err := rootcerts.ConfigureTLS(clientTLSConfig, rootConfig); err != nil {
		return err
	}

	clientTLSConfig.InsecureSkipVerify = t.Insecure

	if foundClientCert {
		clientTLSConfig.Certificates = []tls.Certificate{clientCert}
	}
	if t.TLSServerName != "" {
		clientTLSConfig.ServerName = t.TLSServerName
	}

	return nil
}

// ReadEnvironment reads configuration information from the
// environment. If there is an error, no configuration value
// is updated.
func (c *Config) ReadEnvironment() error {
	var envAddress string
	var envCACert string
	var envCAPath string
	var envClientCert string
	var envClientKey string
	var envInsecure bool
	var envTLSServerName string
	var envMaxRetries *uint64

	// Parse the environment variables
	if v := os.Getenv(EnvVaultAddress); v != "" {
		envAddress = v
	}
	if v := os.Getenv(EnvVaultMaxRetries); v != "" {
		maxRetries, err := strconv.ParseUint(v, 10, 32)
		if err != nil {
			return err
		}
		envMaxRetries = &maxRetries
	}
	if v := os.Getenv(EnvVaultCACert); v != "" {
		envCACert = v
	}
	if v := os.Getenv(EnvVaultCAPath); v != "" {
		envCAPath = v
	}
	if v := os.Getenv(EnvVaultClientCert); v != "" {
		envClientCert = v
	}
	if v := os.Getenv(EnvVaultClientKey); v != "" {
		envClientKey = v
	}
	if v := os.Getenv(EnvVaultInsecure); v != "" {
		var err error
		envInsecure, err = strconv.ParseBool(v)
		if err != nil {
			return fmt.Errorf("Could not parse VAULT_SKIP_VERIFY")
		}
	}
	if v := os.Getenv(EnvVaultTLSServerName); v != "" {
		envTLSServerName = v
	}

	// Configure the HTTP clients TLS configuration.
	t := &TLSConfig{
		CACert:        envCACert,
		CAPath:        envCAPath,
		ClientCert:    envClientCert,
		ClientKey:     envClientKey,
		TLSServerName: envTLSServerName,
		Insecure:      envInsecure,
	}
	if err := c.ConfigureTLS(t); err != nil {
		return err
	}

	if envAddress != "" {
		c.Address = envAddress
	}

	if envMaxRetries != nil {
		c.MaxRetries = int(*envMaxRetries) + 1
	}

	return nil
}

// Client is the client to the Vault API. Create a client with
// NewClient.
type Client struct {
	addr               *url.URL
	config             *Config
	token              string
	wrappingLookupFunc WrappingLookupFunc
}

// NewClient returns a new client for the given configuration.
//
// If the environment variable `VAULT_TOKEN` is present, the token will be
// automatically added to the client. Otherwise, you must manually call
// `SetToken()`.
func NewClient(c *Config) (*Client, error) {
	if c == nil {
		c = DefaultConfig()
		if err := c.ReadEnvironment(); err != nil {
			return nil, fmt.Errorf("error reading environment: %v", err)
		}
	}

	u, err := url.Parse(c.Address)
	if err != nil {
		return nil, err
	}

	if c.HttpClient == nil {
		c.HttpClient = DefaultConfig().HttpClient
	}

	redirFunc := func() {
		// Ensure redirects are not automatically followed
		// Note that this is sane for the API client as it has its own
		// redirect handling logic (and thus also for command/meta),
		// but in e.g. http_test actual redirect handling is necessary
		c.HttpClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {
			// Returning this value causes the Go net library to not close the
			// response body and nil out the error. Otherwise pester tries
			// three times on every redirect because it sees an error from this
			// function being passed through.
			return http.ErrUseLastResponse
		}
	}

	c.redirectSetup.Do(redirFunc)

	client := &Client{
		addr:   u,
		config: c,
	}

	if token := os.Getenv("VAULT_TOKEN"); token != "" {
		client.SetToken(token)
	}

	return client, nil
}

// Sets the address of Vault in the client. The format of address should be
// "<Scheme>://<Host>:<Port>". Setting this on a client will override the
// value of VAULT_ADDR environment variable.
func (c *Client) SetAddress(addr string) error {
	var err error
	if c.addr, err = url.Parse(addr); err != nil {
		return fmt.Errorf("failed to set address: %v", err)
	}

	return nil
}

// Address returns the Vault URL the client is configured to connect to
func (c *Client) Address() string {
	return c.addr.String()
}

// SetWrappingLookupFunc sets a lookup function that returns desired wrap TTLs
// for a given operation and path
func (c *Client) SetWrappingLookupFunc(lookupFunc WrappingLookupFunc) {
	c.wrappingLookupFunc = lookupFunc
}

// Token returns the access token being used by this client. It will
// return the empty string if there is no token set.
func (c *Client) Token() string {
	return c.token
}

// SetToken sets the token directly. This won't perform any auth
// verification, it simply sets the token properly for future requests.
func (c *Client) SetToken(v string) {
	c.token = v
}

// ClearToken deletes the token if it is set or does nothing otherwise.
func (c *Client) ClearToken() {
	c.token = ""
}

// NewRequest creates a new raw request object to query the Vault server
// configured for this client. This is an advanced method and generally
// doesn't need to be called externally.
func (c *Client) NewRequest(method, path string) *Request {
	req := &Request{
		Method: method,
		URL: &url.URL{
			Scheme: c.addr.Scheme,
			Host:   c.addr.Host,
			Path:   path,
		},
		ClientToken: c.token,
		Params:      make(map[string][]string),
	}

	var lookupPath string
	switch {
	case strings.HasPrefix(path, "/v1/"):
		lookupPath = strings.TrimPrefix(path, "/v1/")
	case strings.HasPrefix(path, "v1/"):
		lookupPath = strings.TrimPrefix(path, "v1/")
	default:
		lookupPath = path
	}
	if c.wrappingLookupFunc != nil {
		req.WrapTTL = c.wrappingLookupFunc(method, lookupPath)
	} else {
		req.WrapTTL = DefaultWrappingLookupFunc(method, lookupPath)
	}

	return req
}

// RawRequest performs the raw request given. This request may be against
// a Vault server not configured with this client. This is an advanced operation
// that generally won't need to be called externally.
func (c *Client) RawRequest(r *Request) (*Response, error) {
	redirectCount := 0
START:
	req, err := r.ToHTTP()
	if err != nil {
		return nil, err
	}

	client := pester.NewExtendedClient(c.config.HttpClient)
	client.Backoff = pester.LinearJitterBackoff
	client.MaxRetries = c.config.MaxRetries

	var result *Response
	resp, err := client.Do(req)
	if resp != nil {
		result = &Response{Response: resp}
	}
	if err != nil {
		if strings.Contains(err.Error(), "tls: oversized") {
			err = fmt.Errorf(
				"%s\n\n"+
					"This error usually means that the server is running with TLS disabled\n"+
					"but the client is configured to use TLS. Please either enable TLS\n"+
					"on the server or run the client with -address set to an address\n"+
					"that uses the http protocol:\n\n"+
					"    vault <command> -address http://<address>\n\n"+
					"You can also set the VAULT_ADDR environment variable:\n\n\n"+
					"    VAULT_ADDR=http://<address> vault <command>\n\n"+
					"where <address> is replaced by the actual address to the server.",
				err)
		}
		return result, err
	}

	// Check for a redirect, only allowing for a single redirect
	if (resp.StatusCode == 301 || resp.StatusCode == 302 || resp.StatusCode == 307) && redirectCount == 0 {
		// Parse the updated location
		respLoc, err := resp.Location()
		if err != nil {
			return result, err
		}

		// Ensure a protocol downgrade doesn't happen
		if req.URL.Scheme == "https" && respLoc.Scheme != "https" {
			return result, fmt.Errorf("redirect would cause protocol downgrade")
		}

		// Update the request
		r.URL = respLoc

		// Reset the request body if any
		if err := r.ResetJSONBody(); err != nil {
			return result, err
		}

		// Retry the request
		redirectCount++
		goto START
	}

	if err := result.Error(); err != nil {
		return result, err
	}

	return result, nil
}
<|endoftext|>


<commit_before>package fetcher

import (
	"fmt"
	"io"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/oinume/lekcije/server/model"
	"github.com/uber-go/zap"
	"gopkg.in/xmlpath.v2"
)

const (
	userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/601.6.17 (KHTML, like Gecko) Version/9.1.1 Safari/601.6.17"
)

var (
	jst            = time.FixedZone("Asia/Tokyo", 9*60*60)
	titleXPath     = xmlpath.MustCompile(`//title`)
	lessonXPath    = xmlpath.MustCompile("//ul[@class='oneday']//li")
	classAttrXPath = xmlpath.MustCompile("@class")
)

type TeacherLessonFetcher struct {
	httpClient *http.Client
	log        zap.Logger
}

func NewTeacherLessonFetcher(httpClient *http.Client, log zap.Logger) *TeacherLessonFetcher {
	client := httpClient
	if client == nil {
		client = http.DefaultClient
		client.Timeout = 5 * time.Second
		// TODO: retry
	}
	if log == nil {
		log = zap.NewJSON()
	}
	return &TeacherLessonFetcher{
		httpClient: client,
		log:        log,
	}
}

func (fetcher *TeacherLessonFetcher) Fetch(teacherId uint32) (*model.Teacher, []*model.Lesson, error) {
	teacher := model.NewTeacher(teacherId)
	req, err := http.NewRequest("GET", teacher.Url(), nil)
	if err != nil {
		return nil, nil, err
	}

	req.Header.Set("User-Agent", userAgent)
	resp, err := fetcher.httpClient.Do(req)
	if err != nil {
		return nil, nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		// TODO: pkg/errors
		return nil, nil, fmt.Errorf("fetch error: url=%v, status=%v", teacher.Url(), resp.StatusCode)
	}
	return fetcher.parseHtml(teacher, resp.Body)
}

func (fetcher *TeacherLessonFetcher) parseHtml(
	teacher *model.Teacher,
	html io.Reader,
) (*model.Teacher, []*model.Lesson, error) {
	root, err := xmlpath.ParseHTML(html)
	if err != nil {
		return nil, nil, err
	}

	// teacher name
	if title, ok := titleXPath.String(root); ok {
		teacher.Name = strings.Trim(strings.Split(title, "-")[0], " ")
	} else {
		return nil, nil, fmt.Errorf("failed to fetch teacher's name: url=%v", teacher.Url)
	}

	dateRegexp := regexp.MustCompile(`([\d]+)月([\d]+)日(.+)`)
	lessons := make([]*model.Lesson, 0, 1000)
	originalDate := time.Now().Truncate(24 * time.Hour)
	date := originalDate
	// lessons
	for iter := lessonXPath.Iter(root); iter.Next(); {
		node := iter.Node()
		timeClass, ok := classAttrXPath.String(node)
		if !ok {
			continue
		}

		text := strings.Trim(node.String(), " ")

		//fmt.Printf("text = '%v', timeClass = '%v'\n", text, timeClass)
		fetcher.log.Debug("Scraping as", zap.String("timeClass", timeClass), zap.String("text", text))

		// blank, reservable, reserved
		if timeClass == "date" {
			group := dateRegexp.FindStringSubmatch(text)
			if len(group) > 0 {
				month, day := MustInt(group[1]), MustInt(group[2])
				originalDate = time.Date(date.Year(), time.Month(month), int(day), 0, 0, 0, 0, jst)
				date = originalDate
			}
		} else if strings.HasPrefix(timeClass, "t-") && text != "" {
			tmp := strings.Split(timeClass, "-")
			hour, minute := MustInt(tmp[1]), MustInt(tmp[2])
			if hour >= 24 {
				// Convert 24:30 -> 00:30
				hour -= 24
				if date.Unix() == originalDate.Unix() {
					// Set date to next day for 24:30
					date = date.Add(24 * time.Hour)
				}
			}
			dt := time.Date(date.Year(), date.Month(), date.Day(), hour, minute, 0, 0, jst)
			status := model.LessonStatuses.MustValueForAlias(text)
			fetcher.log.Info(
				"lesson",
				zap.String("dt", dt.Format("2006-01-02 15:04")),
				zap.String("status", model.LessonStatuses.MustName(status)),
			)
			lessons = append(lessons, &model.Lesson{
				TeacherId: teacher.Id,
				Datetime:  dt,
				Status:    model.LessonStatuses.MustName(status),
			})
		} else {
			// nop
		}
	}

	return teacher, lessons, nil
}

func MustInt(s string) int {
	i, err := strconv.ParseInt(s, 10, 32)
	if err != nil {
		panic(err)
	}
	return int(i)
}

/*
       # schedule, reservation
       original_date = datetime.date.today()
       date = copy.copy(original_date)
       time_items = root.xpath("//ul[@class='oneday']//li")
       schedules = []
       logger.debug("--- teacher id={}, name={} ---".format(teacher_id, name))
       for time_item in time_items:
           time_class = time_item.attrib["class"]
           text = time_item.text_content().strip()
           # logger.debug("web {time_class}:{text}".format(**locals()))
           # blank, reservable, reserved
           if time_class == "date":
               match = re.match(r"([\d]+)月([\d]+)日(.+)", text)
               if match:
                   original_date = date.replace(date.year, int(match.group(1)), int(match.group(2)))
                   date = copy.copy(original_date)
           elif time_class.startswith("t-") and text != "":
               tmp = time_class.split("-")
               hour, minute = int(tmp[1]), int(tmp[2])
               if hour >= 24:
                   # 24:30 -> 00:30
                   hour -= 24
                   if date == original_date:
                       # Set date to next day for 24:30
                       date += datetime.timedelta(days=1)
               dt = datetime.datetime(date.year, date.month, date.day, hour, minute, 0, 0)
               if text == "終了":
                   status = "finished"
               elif text in ("予約済", "休講"):  # TODO: Add this status to enum
                   status = "reserved"
               elif text == "予約可":
                   status = "reservable"
               else:
                   raise(Exception("Unknown schedule text:{}".format(text)))

               logger.debug("dt={dt}, status={status}".format(**locals()))
               schedule = Schedule(teacher.id, dt, ScheduleStatus[status])
               schedules.append(schedule)
           else:
               pass
       return teacher, schedules

   def close(self):
       self._session.close()

*/
<commit_msg>Remove comment<commit_after>package fetcher

import (
	"fmt"
	"io"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/oinume/lekcije/server/model"
	"github.com/uber-go/zap"
	"gopkg.in/xmlpath.v2"
)

const (
	userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/601.6.17 (KHTML, like Gecko) Version/9.1.1 Safari/601.6.17"
)

var (
	jst            = time.FixedZone("Asia/Tokyo", 9*60*60)
	titleXPath     = xmlpath.MustCompile(`//title`)
	lessonXPath    = xmlpath.MustCompile("//ul[@class='oneday']//li")
	classAttrXPath = xmlpath.MustCompile("@class")
)

type TeacherLessonFetcher struct {
	httpClient *http.Client
	log        zap.Logger
}

func NewTeacherLessonFetcher(httpClient *http.Client, log zap.Logger) *TeacherLessonFetcher {
	client := httpClient
	if client == nil {
		client = http.DefaultClient
		client.Timeout = 5 * time.Second
		// TODO: retry
	}
	if log == nil {
		log = zap.NewJSON()
	}
	return &TeacherLessonFetcher{
		httpClient: client,
		log:        log,
	}
}

func (fetcher *TeacherLessonFetcher) Fetch(teacherId uint32) (*model.Teacher, []*model.Lesson, error) {
	teacher := model.NewTeacher(teacherId)
	req, err := http.NewRequest("GET", teacher.Url(), nil)
	if err != nil {
		return nil, nil, err
	}

	req.Header.Set("User-Agent", userAgent)
	resp, err := fetcher.httpClient.Do(req)
	if err != nil {
		return nil, nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		// TODO: pkg/errors
		return nil, nil, fmt.Errorf("fetch error: url=%v, status=%v", teacher.Url(), resp.StatusCode)
	}
	return fetcher.parseHtml(teacher, resp.Body)
}

func (fetcher *TeacherLessonFetcher) parseHtml(
	teacher *model.Teacher,
	html io.Reader,
) (*model.Teacher, []*model.Lesson, error) {
	root, err := xmlpath.ParseHTML(html)
	if err != nil {
		return nil, nil, err
	}

	// teacher name
	if title, ok := titleXPath.String(root); ok {
		teacher.Name = strings.Trim(strings.Split(title, "-")[0], " ")
	} else {
		return nil, nil, fmt.Errorf("failed to fetch teacher's name: url=%v", teacher.Url)
	}

	dateRegexp := regexp.MustCompile(`([\d]+)月([\d]+)日(.+)`)
	lessons := make([]*model.Lesson, 0, 1000)
	originalDate := time.Now().Truncate(24 * time.Hour)
	date := originalDate
	// lessons
	for iter := lessonXPath.Iter(root); iter.Next(); {
		node := iter.Node()
		timeClass, ok := classAttrXPath.String(node)
		if !ok {
			continue
		}

		text := strings.Trim(node.String(), " ")

		//fmt.Printf("text = '%v', timeClass = '%v'\n", text, timeClass)
		fetcher.log.Debug("Scraping as", zap.String("timeClass", timeClass), zap.String("text", text))

		// blank, reservable, reserved
		if timeClass == "date" {
			group := dateRegexp.FindStringSubmatch(text)
			if len(group) > 0 {
				month, day := MustInt(group[1]), MustInt(group[2])
				originalDate = time.Date(date.Year(), time.Month(month), int(day), 0, 0, 0, 0, jst)
				date = originalDate
			}
		} else if strings.HasPrefix(timeClass, "t-") && text != "" {
			tmp := strings.Split(timeClass, "-")
			hour, minute := MustInt(tmp[1]), MustInt(tmp[2])
			if hour >= 24 {
				// Convert 24:30 -> 00:30
				hour -= 24
				if date.Unix() == originalDate.Unix() {
					// Set date to next day for 24:30
					date = date.Add(24 * time.Hour)
				}
			}
			dt := time.Date(date.Year(), date.Month(), date.Day(), hour, minute, 0, 0, jst)
			status := model.LessonStatuses.MustValueForAlias(text)
			fetcher.log.Info(
				"lesson",
				zap.String("dt", dt.Format("2006-01-02 15:04")),
				zap.String("status", model.LessonStatuses.MustName(status)),
			)
			lessons = append(lessons, &model.Lesson{
				TeacherId: teacher.Id,
				Datetime:  dt,
				Status:    model.LessonStatuses.MustName(status),
			})
		} else {
			// nop
		}
	}

	return teacher, lessons, nil
}

func MustInt(s string) int {
	i, err := strconv.ParseInt(s, 10, 32)
	if err != nil {
		panic(err)
	}
	return int(i)
}
<|endoftext|>


<commit_before>package api

import (
	"encoding/json"
	"net/http"

	"github.com/FederationOfFathers/dashboard/slack"
	"github.com/gorilla/mux"
	"github.com/uber-go/zap"
	stow "gopkg.in/djherbis/stow.v2"
)

var visDB *stow.Store

func init() {
	Router.Path("/api/v0/groups").Methods("GET").Handler(jwtHandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set("Content-Type", "application/json")
			id := getSlackUserID(r)
			admin, _ := slackData.IsUserIDAdmin(id)
			var groups = map[string]map[string]interface{}{}
			for _, group := range slackData.GetGroups() {
				var visible string
				visDB.Get(group.ID, &visible)
				if visible != "true" {
					visible = "false"
				}
				if visible == "false" {
					if !admin {
						continue
					}
				}
				members := []string{}
				for _, memberID := range group.Members {
					user, err := slackData.User(memberID)
					if err != nil {
						continue
					}
					members = append(members, user.Name)
				}
				groups[group.ID] = map[string]interface{}{
					"ID":      group.ID,
					"Name":    group.Name,
					"Members": members,
					"Visible": visible,
				}
			}
			enc := json.NewEncoder(w)
			enc.Encode(groups)
		},
	))

	Router.Path("/api/v0/groups/{groupID}/join").Methods("GET").Handler(jwtHandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set("Content-Type", "application/json")
			id := getSlackUserID(r)
			want := mux.Vars(r)["groupID"]
			var visible string
			visDB.Get(want, &visible)
			if visible == "true" {
				for _, group := range slackData.GetGroups() {
					if group.ID != want {
						continue
					}
					if err := bot.GroupInvite(group.ID, id); err != nil {
						w.WriteHeader(http.StatusInternalServerError)
					}
					return
				}
			}
			w.WriteHeader(http.StatusNotFound)
		},
	))

	Router.Path("/api/v0/groups/{groupID}/visibility").Methods("PUT").Handler(jwtHandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set("Content-Type", "application/json")
			err := requireAdmin(w, r)
			if err != nil {
				return
			}
			err = visDB.Put(
				mux.Vars(r)["groupID"],
				r.FormValue("visible"))
			if err != nil {
				logger.Error("error putting a value to visDB", zap.Error(err))
				w.WriteHeader(http.StatusInternalServerError)
			}
		},
	))
}
<commit_msg>add purpose and topic to the group list<commit_after>package api

import (
	"encoding/json"
	"net/http"

	"github.com/FederationOfFathers/dashboard/slack"
	"github.com/gorilla/mux"
	"github.com/uber-go/zap"
	stow "gopkg.in/djherbis/stow.v2"
)

var visDB *stow.Store

func init() {
	Router.Path("/api/v0/groups").Methods("GET").Handler(jwtHandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set("Content-Type", "application/json")
			id := getSlackUserID(r)
			admin, _ := slackData.IsUserIDAdmin(id)
			var groups = map[string]map[string]interface{}{}
			for _, group := range slackData.GetGroups() {
				var visible string
				visDB.Get(group.ID, &visible)
				if visible != "true" {
					visible = "false"
				}
				if visible == "false" {
					if !admin {
						continue
					}
				}
				members := []string{}
				for _, memberID := range group.Members {
					user, err := slackData.User(memberID)
					if err != nil {
						continue
					}
					members = append(members, user.Name)
				}
				groups[group.ID] = map[string]interface{}{
					"id":      group.ID,
					"name":    group.Name,
					"topic":   group.Topic.Value,
					"purpose": group.Purpose.Value,
					"members": members,
					"visible": visible,
				}
			}
			enc := json.NewEncoder(w)
			enc.Encode(groups)
		},
	))

	Router.Path("/api/v0/groups/{groupID}/join").Methods("GET").Handler(jwtHandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set("Content-Type", "application/json")
			id := getSlackUserID(r)
			want := mux.Vars(r)["groupID"]
			var visible string
			visDB.Get(want, &visible)
			if visible == "true" {
				for _, group := range slackData.GetGroups() {
					if group.ID != want {
						continue
					}
					if err := bot.GroupInvite(group.ID, id); err != nil {
						w.WriteHeader(http.StatusInternalServerError)
					}
					return
				}
			}
			w.WriteHeader(http.StatusNotFound)
		},
	))

	Router.Path("/api/v0/groups/{groupID}/visibility").Methods("PUT").Handler(jwtHandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set("Content-Type", "application/json")
			err := requireAdmin(w, r)
			if err != nil {
				return
			}
			err = visDB.Put(
				mux.Vars(r)["groupID"],
				r.FormValue("visible"))
			if err != nil {
				logger.Error("error putting a value to visDB", zap.Error(err))
				w.WriteHeader(http.StatusInternalServerError)
			}
		},
	))
}
<|endoftext|>


<commit_before>package main

import (
	"google.golang.org/protobuf/compiler/protogen"
)

// var (
// 	bytesPackage   = protogen.GoImportPath("bytes")
// 	contextPackage = protogen.GoImportPath("context")
// 	base64Package  = protogen.GoImportPath("encoding/base64")
// 	jsonPackage    = protogen.GoImportPath("encoding/json")
// 	fmtPackage     = protogen.GoImportPath("fmt")
// 	ioPackage      = protogen.GoImportPath("io")
// 	ioutilPackage  = protogen.GoImportPath("io/ioutil")
// 	mimePackage    = protogen.GoImportPath("mime")
// 	httpPackage    = protogen.GoImportPath("net/http")
// 	strconvPackage = protogen.GoImportPath("strconv")
// 	stringsPackage = protogen.GoImportPath("strings")
// )

// var (
// 	jsonpbPackage = protogen.GoImportPath("github.com/golang/protobuf/jsonpb")
// 	protoPackage  = protogen.GoImportPath("github.com/golang/protobuf/proto")
// 	grpcPackage   = protogen.GoImportPath("google.golang.org/grpc")
// 	codesPackage  = protogen.GoImportPath("google.golang.org/grpc/codes")
// 	statusPackage = protogen.GoImportPath("google.golang.org/grpc/status")
// )

func GenerateFile(gen *protogen.Plugin, file *protogen.File) *protogen.GeneratedFile {
	filename := file.GeneratedFilenamePrefix + ".http.go"
	g := gen.NewGeneratedFile(filename, file.GoImportPath)

	g.P("// Code generated by protoc-gen-gohttp. DO NOT EDIT.")
	g.P("// source: ", file.Desc.Path())
	g.P()
	g.P("package ", file.GoPackageName)

	for _, srv := range file.Services {
		genService(g, srv)
	}

	return g
}

func genService(g *protogen.GeneratedFile, srv *protogen.Service) {
	g.P("// ", srv.GoName, " is the server API for Greeter service.")
	g.P("type ", srv.GoName, " interface {")
	// // SayHello says hello.
	// SayHello(context.Context, *HelloRequest) (*HelloReply, error)
	g.P("}")
}
<commit_msg>Generated interface and struct<commit_after>package main

import (
	"google.golang.org/protobuf/compiler/protogen"
)

var (
	// 	bytesPackage   = protogen.GoImportPath("bytes")
	contextPackage = protogen.GoImportPath("context")

// 	base64Package  = protogen.GoImportPath("encoding/base64")
// 	jsonPackage    = protogen.GoImportPath("encoding/json")
// 	fmtPackage     = protogen.GoImportPath("fmt")
// 	ioPackage      = protogen.GoImportPath("io")
// 	ioutilPackage  = protogen.GoImportPath("io/ioutil")
// 	mimePackage    = protogen.GoImportPath("mime")
// 	httpPackage    = protogen.GoImportPath("net/http")
// 	strconvPackage = protogen.GoImportPath("strconv")
// 	stringsPackage = protogen.GoImportPath("strings")
)

// var (
// 	jsonpbPackage = protogen.GoImportPath("github.com/golang/protobuf/jsonpb")
// 	protoPackage  = protogen.GoImportPath("github.com/golang/protobuf/proto")
// 	grpcPackage   = protogen.GoImportPath("google.golang.org/grpc")
// 	codesPackage  = protogen.GoImportPath("google.golang.org/grpc/codes")
// 	statusPackage = protogen.GoImportPath("google.golang.org/grpc/status")
// )

func GenerateFile(gen *protogen.Plugin, file *protogen.File) *protogen.GeneratedFile {
	filename := file.GeneratedFilenamePrefix + ".http.go"
	g := gen.NewGeneratedFile(filename, file.GoImportPath)

	g.P("// Code generated by protoc-gen-gohttp. DO NOT EDIT.")
	g.P("// source: ", file.Desc.Path())
	g.P()
	g.P("package ", file.GoPackageName)

	for _, srv := range file.Services {
		genService(g, srv)
	}

	return g
}

func genService(g *protogen.GeneratedFile, srv *protogen.Service) {
	g.P("// ", srv.GoName, "HTTPService is the server API for ", srv.GoName, " service.")
	g.P("type ", srv.GoName, "HTTPService interface {")
	for _, method := range srv.Methods {
		if method.Desc.IsStreamingClient() || method.Desc.IsStreamingServer() {
			continue
		}
		g.P(method.GoName, "(", contextPackage.Ident("Context"), ", *", method.Input.GoIdent.GoName, ") (*", method.Output.GoIdent.GoName, ", error)")
	}
	g.P("}")

	g.P("// ", srv.GoName, "HTTPConverter has a function to convert ", srv.GoName, "HTTPService interface to http.HandlerFunc.")
	g.P("type ", srv.GoName, "HTTPConverter struct {")
	g.P("srv ", srv.GoName, "HTTPService")
	g.P("}")
}
<|endoftext|>


<commit_before>package services

import (
	"apertoire.net/unbalance/server/dto"
	"apertoire.net/unbalance/server/model"
	"github.com/gorilla/websocket"
	"github.com/jbrodriguez/mlog"
	"github.com/jbrodriguez/pubsub"
	"net/http"
)

const (
	//	pongWait       = 60 * time.Second
	bufferSize = 8192
)

type Socket struct {
	bus      *pubsub.PubSub
	settings *model.Settings

	// registered connections
	connections map[*Connection]bool

	// inbound messages from connections
	broadcast chan *pubsub.Message

	// inbound messages from connections
	emit chan *pubsub.Message

	// register requests from connection
	register chan *Connection

	// unregister request from connection
	unregister chan *Connection
}

func NewSocket(bus *pubsub.PubSub, settings *model.Settings) *Socket {
	return &Socket{
		bus:      bus,
		settings: settings,

		connections: make(map[*Connection]bool),
		register:    make(chan *Connection),
		unregister:  make(chan *Connection),

		broadcast: bus.Sub("socket:broadcast"),
		emit:      bus.Sub("socket:emit"),
	}
}

func (s *Socket) handler(w http.ResponseWriter, r *http.Request) {
	ws, err := websocket.Upgrade(w, r, nil, bufferSize, bufferSize)
	if _, ok := err.(websocket.HandshakeError); ok {
		http.Error(w, "Not a websocket handshake", 400)
		return
	} else if err != nil {
		mlog.Error(err)
		return
	}

	// mlog.Info("opening connection ... ")
	c := &Connection{send: make(chan *dto.MessageOut, 50), ws: ws, hub: s}
	s.register <- c
	// defer func() { s.unregister <- c }()

	go c.writer()
	c.reader()
}

func (s *Socket) Start() {
	mlog.Info("starting service Socket ...")
	go s.react()
}

func (s *Socket) Stop() {
	mlog.Info("stopped service Core ...")
}

func (s *Socket) react() {
	for {
		select {
		case c := <-s.register:
			s.connections[c] = true
		case c := <-s.unregister:
			if _, ok := s.connections[c]; ok {
				delete(s.connections, c)
				close(c.send)
			}
		case m := <-s.broadcast:
			// mlog.Info("broadcasting %v [%v]", m, m.Payload)

			for c := range s.connections {
				c.send <- m.Payload.(*dto.MessageOut)
				// select {
				// case c.send <- m.Payload.(*dto.MessageOut):
				// 	mlog.Info("after c.send")
				// 	default:
				// 		mlog.Info("default.close")
				// 		close(c.send)
				// 		delete(s.connections, c)
				// 	// go c.ws.Close()
				// }
			}
			//		case _ := <-s.emit:
			//go c.calculateBestFit(msg)
		}
	}
}
<commit_msg>- Fix service name in stop routine<commit_after>package services

import (
	"apertoire.net/unbalance/server/dto"
	"apertoire.net/unbalance/server/model"
	"github.com/gorilla/websocket"
	"github.com/jbrodriguez/mlog"
	"github.com/jbrodriguez/pubsub"
	"net/http"
)

const (
	//	pongWait       = 60 * time.Second
	bufferSize = 8192
)

type Socket struct {
	bus      *pubsub.PubSub
	settings *model.Settings

	// registered connections
	connections map[*Connection]bool

	// inbound messages from connections
	broadcast chan *pubsub.Message

	// inbound messages from connections
	emit chan *pubsub.Message

	// register requests from connection
	register chan *Connection

	// unregister request from connection
	unregister chan *Connection
}

func NewSocket(bus *pubsub.PubSub, settings *model.Settings) *Socket {
	return &Socket{
		bus:      bus,
		settings: settings,

		connections: make(map[*Connection]bool),
		register:    make(chan *Connection),
		unregister:  make(chan *Connection),

		broadcast: bus.Sub("socket:broadcast"),
		emit:      bus.Sub("socket:emit"),
	}
}

func (s *Socket) handler(w http.ResponseWriter, r *http.Request) {
	ws, err := websocket.Upgrade(w, r, nil, bufferSize, bufferSize)
	if _, ok := err.(websocket.HandshakeError); ok {
		http.Error(w, "Not a websocket handshake", 400)
		return
	} else if err != nil {
		mlog.Error(err)
		return
	}

	// mlog.Info("opening connection ... ")
	c := &Connection{send: make(chan *dto.MessageOut, 50), ws: ws, hub: s}
	s.register <- c
	// defer func() { s.unregister <- c }()

	go c.writer()
	c.reader()
}

func (s *Socket) Start() {
	mlog.Info("starting service Socket ...")
	go s.react()
}

func (s *Socket) Stop() {
	mlog.Info("stopped service Socket ...")
}

func (s *Socket) react() {
	for {
		select {
		case c := <-s.register:
			s.connections[c] = true
		case c := <-s.unregister:
			if _, ok := s.connections[c]; ok {
				delete(s.connections, c)
				close(c.send)
			}
		case m := <-s.broadcast:
			// mlog.Info("broadcasting %v [%v]", m, m.Payload)

			for c := range s.connections {
				c.send <- m.Payload.(*dto.MessageOut)
				// select {
				// case c.send <- m.Payload.(*dto.MessageOut):
				// 	mlog.Info("after c.send")
				// 	default:
				// 		mlog.Info("default.close")
				// 		close(c.send)
				// 		delete(s.connections, c)
				// 	// go c.ws.Close()
				// }
			}
			//		case _ := <-s.emit:
			//go c.calculateBestFit(msg)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/hmac"
	"crypto/rand"
	"crypto/sha256"
	"database/sql"
	"encoding/hex"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"text/template"

	"github.com/zenazn/goji"
	"github.com/zenazn/goji/web"

	_ "github.com/mattn/go-sqlite3"
)

const charset = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-.@_"

var db *sql.DB

var cookie_secret = make([]byte, 16)

func validate_charset(s string) bool {
	for _, c := range s {
		if strings.Index(charset, string(c)) == -1 {
			return false
		}
	}
	return true
}

func set_cookie(user_id string, w http.ResponseWriter) {
	mac := hmac.New(sha256.New, cookie_secret)
	mac.Write([]byte(user_id))

	cookie := &http.Cookie{
		Name:     "user_id",
		Value:    hex.EncodeToString(mac.Sum(nil)) + user_id,
		MaxAge:   60 * 5,
		Path:     "/",
		HttpOnly: true,
		// Secure:   true,
	}

	http.SetCookie(w, cookie)
}

func read_cookie(r *http.Request) string {
	cookie, err := r.Cookie("user_id")
	if err != nil {
		return ""
	}

	mac := hmac.New(sha256.New, cookie_secret)
	mac.Write([]byte(cookie.Value[sha256.Size*2:]))

	if !hmac.Equal([]byte(hex.EncodeToString(mac.Sum(nil))),
		[]byte(cookie.Value[:sha256.Size*2])) {
		return ""
	}

	return cookie.Value[sha256.Size*2:]
}

func get_form(c web.C, w http.ResponseWriter, r *http.Request) {
	user_id := read_cookie(r)

	if user_id == "" {
		http.Redirect(w, r, "/login", 302)
		return
	}

	var alias string
	var addr string

	err := db.QueryRow(`SELECT alias, addr FROM "ALIASES" WHERE user_id = ?`,
		user_id).Scan(&alias, &addr)
	if err != nil && err != sql.ErrNoRows {
		log.Println(err)
		http.Error(w, http.StatusText(500), 500)
		return
	}

	csrf_token := make([]byte, 16)
	_, err = rand.Read(csrf_token)
	if err != nil {
		log.Println(err)
		http.Error(w, http.StatusText(500), 500)
		return
	}

	cookie := &http.Cookie{
		Name:     "csrf_token",
		Value:    hex.EncodeToString(csrf_token),
		MaxAge:   60,
		Path:     "/",
		HttpOnly: true,
		// Secure:   true,
	}
	http.SetCookie(w, cookie)

	t, _ := template.ParseFiles("form.html")

	type TemplateContext struct {
		Alias      string
		Addr       string
		Csrf_token string
	}
	context := &TemplateContext{
		Alias:      alias,
		Addr:       addr,
		Csrf_token: hex.EncodeToString(csrf_token),
	}

	err = t.Execute(w, context)
	if err != nil {
		log.Println(err)
		http.Error(w, http.StatusText(500), 500)
		return
	}
}

func post_form(c web.C, w http.ResponseWriter, r *http.Request) {
	user_id := read_cookie(r)

	err := r.ParseForm()
	if err != nil {
		log.Println("Could not parse form params")
		http.Error(w, http.StatusText(400), 400)
		return
	}

	cookie, err := r.Cookie("csrf_token")
	if err != nil {
		log.Println(err)
		http.Error(w, http.StatusText(403), 403)
		return
	}
	if cookie.Value != r.PostForm.Get("csrf_token") {
		http.Error(w, http.StatusText(403), 403)
		return
	}

	alias := r.PostForm.Get("alias")
	addr := r.PostForm.Get("addr")

	if !validate_charset(alias) || !validate_charset(addr) {
		http.Error(w, "Unallowed characters", 403)
		return
	}

	_, err = db.Exec(`DELETE FROM "ALIASES" WHERE user_id = ?`, user_id)
	if err != nil {
		log.Println(err)
		http.Error(w, http.StatusText(500), 500)
		return
	}

	_, err = db.Exec(`INSERT INTO "ALIASES" (user_id, alias, addr)
					  VALUES (?, ?, ?)`, user_id, alias, addr)
	if err != nil {
		log.Println(err)
		file, err := ioutil.ReadFile("./error.html")
		if err != nil {
			log.Println(err)
			http.Error(w, http.StatusText(500), 500)
			return
		}
		w.Write(file)
		return
	}

	http.Redirect(w, r, "/", 303)
}

func main() {
	_, err := rand.Read(cookie_secret)
	if err != nil {
		log.Fatal(err)
	}

	load_oauth()

	db, err = sql.Open("sqlite3", "./alum.db")
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	_, err = db.Exec(`
        CREATE TABLE IF NOT EXISTS "ALIASES" (
        "user_id" TEXT PRIMARY KEY NOT NULL,
        "alias" TEXT UNIQUE NOT NULL,
        "addr" TEXT NOT NULL
        );`)
	if err != nil {
		log.Fatal(err)
	}

	goji.Get("/", get_form)
	goji.Post("/", post_form)
	goji.Serve()
}
<commit_msg>Recreate postfix file after insert/update<commit_after>package main

import (
	"crypto/hmac"
	"crypto/rand"
	"crypto/sha256"
	"database/sql"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
	"text/template"

	"github.com/zenazn/goji"
	"github.com/zenazn/goji/web"

	_ "github.com/mattn/go-sqlite3"
)

const charset = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-.@_"

var db *sql.DB

var cookie_secret = make([]byte, 16)

func validate_charset(s string) bool {
	for _, c := range s {
		if strings.Index(charset, string(c)) == -1 {
			return false
		}
	}
	return true
}

func set_cookie(user_id string, w http.ResponseWriter) {
	mac := hmac.New(sha256.New, cookie_secret)
	mac.Write([]byte(user_id))

	cookie := &http.Cookie{
		Name:     "user_id",
		Value:    hex.EncodeToString(mac.Sum(nil)) + user_id,
		MaxAge:   60 * 5,
		Path:     "/",
		HttpOnly: true,
		// Secure:   true,
	}

	http.SetCookie(w, cookie)
}

func read_cookie(r *http.Request) string {
	cookie, err := r.Cookie("user_id")
	if err != nil {
		return ""
	}

	mac := hmac.New(sha256.New, cookie_secret)
	mac.Write([]byte(cookie.Value[sha256.Size*2:]))

	if !hmac.Equal([]byte(hex.EncodeToString(mac.Sum(nil))),
		[]byte(cookie.Value[:sha256.Size*2])) {
		return ""
	}

	return cookie.Value[sha256.Size*2:]
}

func get_form(c web.C, w http.ResponseWriter, r *http.Request) {
	user_id := read_cookie(r)

	if user_id == "" {
		http.Redirect(w, r, "/login", 302)
		return
	}

	var alias string
	var addr string

	err := db.QueryRow(`SELECT alias, addr FROM "ALIASES" WHERE user_id = ?`,
		user_id).Scan(&alias, &addr)
	if err != nil && err != sql.ErrNoRows {
		log.Println(err)
		http.Error(w, http.StatusText(500), 500)
		return
	}

	csrf_token := make([]byte, 16)
	_, err = rand.Read(csrf_token)
	if err != nil {
		log.Println(err)
		http.Error(w, http.StatusText(500), 500)
		return
	}

	cookie := &http.Cookie{
		Name:     "csrf_token",
		Value:    hex.EncodeToString(csrf_token),
		MaxAge:   60,
		Path:     "/",
		HttpOnly: true,
		// Secure:   true,
	}
	http.SetCookie(w, cookie)

	t, _ := template.ParseFiles("form.html")

	type TemplateContext struct {
		Alias      string
		Addr       string
		Csrf_token string
	}
	context := &TemplateContext{
		Alias:      alias,
		Addr:       addr,
		Csrf_token: hex.EncodeToString(csrf_token),
	}

	err = t.Execute(w, context)
	if err != nil {
		log.Println(err)
		http.Error(w, http.StatusText(500), 500)
		return
	}
}

func post_form(c web.C, w http.ResponseWriter, r *http.Request) {
	user_id := read_cookie(r)

	err := r.ParseForm()
	if err != nil {
		log.Println("Could not parse form params")
		http.Error(w, http.StatusText(400), 400)
		return
	}

	cookie, err := r.Cookie("csrf_token")
	if err != nil {
		log.Println(err)
		http.Error(w, http.StatusText(403), 403)
		return
	}
	if cookie.Value != r.PostForm.Get("csrf_token") {
		http.Error(w, http.StatusText(403), 403)
		return
	}

	alias := r.PostForm.Get("alias")
	addr := r.PostForm.Get("addr")

	if !validate_charset(alias) || !validate_charset(addr) {
		http.Error(w, "Unallowed characters", 403)
		return
	}

	_, err = db.Exec(`DELETE FROM "ALIASES" WHERE user_id = ?`, user_id)
	if err != nil {
		log.Println(err)
		http.Error(w, http.StatusText(500), 500)
		return
	}

	_, err = db.Exec(`INSERT INTO "ALIASES" (user_id, alias, addr)
					  VALUES (?, ?, ?)`, user_id, alias, addr)
	if err != nil {
		log.Println(err)
		file, err := ioutil.ReadFile("./error.html")
		if err != nil {
			log.Println(err)
			http.Error(w, http.StatusText(500), 500)
			return
		}
		w.Write(file)
		return
	}

	// Recreate the postfix file.
	rows, err := db.Query(`SELECT alias, addr FROM "ALIASES"`)
	if err != nil && err != sql.ErrNoRows {
		log.Println(err)
		http.Error(w, http.StatusText(500), 500)
		return
	}

	virtual, err := os.Create("virtual")
	if err != nil {
		log.Println(err)
		http.Error(w, http.StatusText(500), 500)
		return
	}
	defer virtual.Close()

	for rows.Next() {
		var alias string
		var addr string
		err = rows.Scan(&alias, &addr)
		fmt.Fprintf(virtual, "%s@alum.hackerschool.com %s\n", alias, addr)
	}

	http.Redirect(w, r, "/", 303)
}

func main() {
	_, err := rand.Read(cookie_secret)
	if err != nil {
		log.Fatal(err)
	}

	load_oauth()

	db, err = sql.Open("sqlite3", "./alum.db")
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	_, err = db.Exec(`
        CREATE TABLE IF NOT EXISTS "ALIASES" (
        "user_id" TEXT PRIMARY KEY NOT NULL,
        "alias" TEXT UNIQUE NOT NULL,
        "addr" TEXT NOT NULL
        );`)
	if err != nil {
		log.Fatal(err)
	}

	goji.Get("/", get_form)
	goji.Post("/", post_form)
	goji.Serve()
}
<|endoftext|>


<commit_before>package fox

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"util"
)

func getFileName(uuid string) string {
	return util.GetConfig().Storage.Filepath + uuid
}

// StoreFox persists the fox instance to somewhere
func StoreFox(fox Fox, uuid string) UUID {

	err := os.MkdirAll(getFileName(""), 0744)

	if err != nil {
		panic(err)
	}

	err = os.Chmod(getFileName(""), 0744)

	if err != nil {
		panic(err)
	}

	f, err := os.Create(getFileName(uuid))

	if err != nil {
		panic(err)
	}

	defer f.Close()

	// Make sure the file name matches the uuid in the structure
	fox.Uuid = uuid
	if err := json.NewEncoder(f).Encode(fox); err != nil {
		panic(err)
	}

	return UUID{Uuid: uuid}
}

func ReadFox(uuid string) (Fox, error) {
	var fox Fox
	data, err := ioutil.ReadFile(getFileName(uuid))

	if err != nil {
		return fox, err
	}

	if err := json.Unmarshal(data, &fox); err != nil {
		return fox, err
	}

	return fox, nil
}

func GetFoxes() ([]Fox, error) {
	var foxes []Fox

	foxes = make([]Fox, 0)
	fname := util.GetConfig().Storage.Filepath
	files, _ := ioutil.ReadDir(fname)
	for _, f := range files {
		fox, err := ReadFox(f.Name())
		if err != nil {
			return foxes, err
		}
		foxes = append(foxes, fox)
	}

	return foxes, nil
}

func FoxExists(uuid string) bool {
	if _, err := os.Stat(getFileName(uuid)); os.IsNotExist(err) {
		return false
	}
	return true
}

// Deletes the fox, if it exists. Does nothing if it does not
func DeleteFoxFromStorage(uuid string) {

	if !FoxExists(uuid) {
		// Exit quietly, if the fox is not there
		return
	}

	// Attempt to remove the file if it is there. Panic if it fails
	if err := os.Remove(getFileName(uuid)); err != nil {
		panic(err)
	}
}
<commit_msg>lessen permissions for and remove chmod from StoreFox()<commit_after>package fox

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"util"
)

func getFileName(uuid string) string {
	return util.GetConfig().Storage.Filepath + uuid
}

// StoreFox persists the fox instance to somewhere
func StoreFox(fox Fox, uuid string) UUID {

	err := os.MkdirAll(getFileName(""), 0300)

	if err != nil {
		panic(err)
	}

	f, err := os.Create(getFileName(uuid))

	if err != nil {
		panic(err)
	}

	defer f.Close()

	// Make sure the file name matches the uuid in the structure
	fox.Uuid = uuid
	if err := json.NewEncoder(f).Encode(fox); err != nil {
		panic(err)
	}

	return UUID{Uuid: uuid}
}

func ReadFox(uuid string) (Fox, error) {
	var fox Fox
	data, err := ioutil.ReadFile(getFileName(uuid))

	if err != nil {
		return fox, err
	}

	if err := json.Unmarshal(data, &fox); err != nil {
		return fox, err
	}

	return fox, nil
}

func GetFoxes() ([]Fox, error) {
	var foxes []Fox

	foxes = make([]Fox, 0)
	fname := util.GetConfig().Storage.Filepath
	files, _ := ioutil.ReadDir(fname)
	for _, f := range files {
		fox, err := ReadFox(f.Name())
		if err != nil {
			return foxes, err
		}
		foxes = append(foxes, fox)
	}

	return foxes, nil
}

func FoxExists(uuid string) bool {
	if _, err := os.Stat(getFileName(uuid)); os.IsNotExist(err) {
		return false
	}
	return true
}

// Deletes the fox, if it exists. Does nothing if it does not
func DeleteFoxFromStorage(uuid string) {

	if !FoxExists(uuid) {
		// Exit quietly, if the fox is not there
		return
	}

	// Attempt to remove the file if it is there. Panic if it fails
	if err := os.Remove(getFileName(uuid)); err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"runtime"
	"syscall"

	"github.com/intelsdilabs/pulse/control"
	"github.com/intelsdilabs/pulse/schedule"
)

// Pulse Flags for command line
var version = flag.Bool("version", false, "print Pulse version")
var maxProcs = flag.Int("max_procs", 0, "max number of CPUs that can be used simultaneously. Default is use all cores.")

var gitversion string

type coreModule interface {
	Start() error
	Stop()
}

func main() {
	flag.Parse()
	if *version {
		fmt.Println("Pulse version:", gitversion)
		os.Exit(0)
	}
	// Set Max Processors for the Pulse agent.
	setMaxProcs()

	c := control.New()
	s := schedule.New()
	s.MetricManager = c

	// Set interrupt handling so we can die gracefully.
	startInterruptHandling(c, s)

	//  Start our modules
	if err := startModule("Plugin Controller", c); err != nil {
		printErrorAndExit("Plugin Controller", err)
	}
	if err := startModule("Scheduler", s); err != nil {
		if c.Started {
			c.Stop()
		}
		printErrorAndExit("Scheduler", err)
	}

	select {} //run forever and ever
}

func setMaxProcs() {
	var _maxProcs int
	numProcs := runtime.NumCPU()
	if *maxProcs <= 0 {
		if *maxProcs < 0 {
			log.Printf("WARNING: max_procs set to less than zero. Setting GOMAXPROCS to %v", numProcs)
		}
		_maxProcs = numProcs
	} else if *maxProcs > numProcs {
		log.Printf("WARNING: Not allowed to set GOMAXPROCS above number of processors in system. Setting GOMAXPROCS to %v", numProcs)
		_maxProcs = numProcs
	} else {
		_maxProcs = *maxProcs
	}
	log.Printf("Setting GOMAXPROCS to %v\n", _maxProcs)
	runtime.GOMAXPROCS(_maxProcs)

	//Verify setting worked
	actualNumProcs := runtime.GOMAXPROCS(0)
	if actualNumProcs != _maxProcs {
		log.Printf("WARNING: Specified max procs of %v but using %v", _maxProcs, actualNumProcs)
	}
}

func startModule(name string, m coreModule) error {
	log.Printf("Starting Pulse Agent %s module", name)
	return m.Start()
}

func printErrorAndExit(name string, err error) {
	log.Println("ERROR:", err)
	log.Println("ERROR: Error starting Pulse Agent %s module. Exiting now.", name)
	os.Exit(1)
}

func startInterruptHandling(modules ...coreModule) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill, syscall.SIGTERM)

	//Let's block until someone tells us to quit
	go func() {
		sig := <-c
		log.Println("Stopping Pulse Agent modules")
		for _, m := range modules {
			m.Stop()
		}
		log.Printf("Exiting given signal: %v", sig)
		os.Exit(0)
	}()
}
<commit_msg>Add fixes and change default for GOMAXPROCS to 1<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"runtime"
	"syscall"

	"github.com/intelsdilabs/pulse/control"
	"github.com/intelsdilabs/pulse/schedule"
)

var (
	// Pulse Flags for command line
	version  = flag.Bool("version", false, "print Pulse version")
	maxProcs = flag.Int("max_procs", 1, "max number of CPUs that can be used simultaneously. Default is use all cores.")

	gitversion string
)

const (
	defaultQueueSize int = 25
	defaultPoolSize  int = 4
)

type coreModule interface {
	Start() error
	Stop()
}

func main() {
	flag.Parse()
	if *version {
		fmt.Println("Pulse version:", gitversion)
		os.Exit(0)
	}
	// Set Max Processors for the Pulse agent.
	setMaxProcs()

	c := control.New()
	s := schedule.New(defaultPoolSize, defaultQueueSize)
	s.SetMetricManager(c)

	// Set interrupt handling so we can die gracefully.
	startInterruptHandling(c, s)

	//  Start our modules
	if err := startModule("Plugin Controller", c); err != nil {
		printErrorAndExit("Plugin Controller", err)
	}
	if err := startModule("Scheduler", s); err != nil {
		if c.Started {
			c.Stop()
		}
		printErrorAndExit("Scheduler", err)
	}

	select {} //run forever and ever
}

func setMaxProcs() {
	var _maxProcs int
	numProcs := runtime.NumCPU()
	if *maxProcs <= 0 {
		if *maxProcs < 0 {
			log.Println("WARNING: max_procs set to less than zero. Setting GOMAXPROCS to default of 1")
		}
		_maxProcs = 1
	} else if *maxProcs > numProcs {
		log.Printf("WARNING: Not allowed to set GOMAXPROCS above number of processors in system. Setting GOMAXPROCS to %v", numProcs)
		_maxProcs = numProcs
	} else {
		_maxProcs = *maxProcs
	}
	log.Printf("Setting GOMAXPROCS to %v\n", _maxProcs)
	runtime.GOMAXPROCS(_maxProcs)

	//Verify setting worked
	actualNumProcs := runtime.GOMAXPROCS(0)
	if actualNumProcs != _maxProcs {
		log.Printf("WARNING: Specified max procs of %v but using %v", _maxProcs, actualNumProcs)
	}
}

func startModule(name string, m coreModule) error {
	log.Printf("Starting Pulse Agent %s module", name)
	return m.Start()
}

func printErrorAndExit(name string, err error) {
	log.Println("ERROR:", err)
	log.Println("ERROR: Error starting Pulse Agent %s module. Exiting now.", name)
	os.Exit(1)
}

func startInterruptHandling(modules ...coreModule) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill, syscall.SIGTERM)

	//Let's block until someone tells us to quit
	go func() {
		sig := <-c
		log.Println("Stopping Pulse Agent modules")
		for _, m := range modules {
			m.Stop()
		}
		log.Printf("Exiting given signal: %v", sig)
		os.Exit(0)
	}()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"runtime"
	"syscall"

	"github.com/intelsdilabs/pulse/control"
	"github.com/intelsdilabs/pulse/schedule"
)

var (
	// Pulse Flags for command line
	version  = flag.Bool("version", false, "Print Pulse version")
	maxProcs = flag.Int("max_procs", 0, "Set max cores to use for Pulse Agent. Default is 1 core.")

	gitversion string
)

const (
	defaultQueueSize int = 25
	defaultPoolSize  int = 4
)

type coreModule interface {
	Start() error
	Stop()
}

func main() {
	flag.Parse()
	if *version {
		fmt.Println("Pulse version:", gitversion)
		os.Exit(0)
	}
	// Set Max Processors for the Pulse agent.
	setMaxProcs()

	c := control.New()
	s := schedule.New(defaultPoolSize, defaultQueueSize)
	s.SetMetricManager(c)

	// Set interrupt handling so we can die gracefully.
	startInterruptHandling(c, s)

	//  Start our modules
	if err := startModule("Plugin Controller", c); err != nil {
		printErrorAndExit("Plugin Controller", err)
	}
	if err := startModule("Scheduler", s); err != nil {
		if c.Started {
			c.Stop()
		}
		printErrorAndExit("Scheduler", err)
	}

	select {} //run forever and ever
}

func setMaxProcs() {
	var _maxProcs int
	numProcs := runtime.NumCPU()
	if *maxProcs <= 0 {
		if *maxProcs < 0 {
			log.Println("WARNING: max_procs set to less than zero. Setting GOMAXPROCS to default of 1")
		}
		_maxProcs = 1
	} else if *maxProcs > numProcs {
		log.Printf("WARNING: Not allowed to set GOMAXPROCS above number of processors in system. Setting GOMAXPROCS to %v", numProcs)
		_maxProcs = numProcs
	} else {
		_maxProcs = *maxProcs
	}

	log.Printf("Setting GOMAXPROCS to %v\n", _maxProcs)
	runtime.GOMAXPROCS(_maxProcs)

	//Verify setting worked
	actualNumProcs := runtime.GOMAXPROCS(0)
	if actualNumProcs != _maxProcs {
		log.Printf("WARNING: Specified max procs of %v but using %v", _maxProcs, actualNumProcs)
	}
}

func startModule(name string, m coreModule) error {
	log.Printf("Starting Pulse Agent %s module", name)
	return m.Start()
}

func printErrorAndExit(name string, err error) {
	log.Println("ERROR:", err)
	log.Printf("ERROR: Error starting Pulse Agent %s module. Exiting now.", name)
	os.Exit(1)
}

func startInterruptHandling(modules ...coreModule) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill, syscall.SIGTERM)

	//Let's block until someone tells us to quit
	go func() {
		sig := <-c
		log.Println("Stopping Pulse Agent modules")
		for _, m := range modules {
			m.Stop()
		}
		log.Printf("Exiting given signal: %v", sig)
		os.Exit(0)
	}()
}
<commit_msg>Update setting of GOMAXPROCS to check ENV variable and only override ENV variable if max_procs is set.<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"runtime"
	"syscall"

	"github.com/intelsdilabs/pulse/control"
	"github.com/intelsdilabs/pulse/schedule"
)

var (
	// Pulse Flags for command line
	version  = flag.Bool("version", false, "Print Pulse version")
	maxProcs = flag.Int("max_procs", 0, "Set max cores to use for Pulse Agent. Default is 1 core.")

	gitversion string
)

const (
	defaultQueueSize int = 25
	defaultPoolSize  int = 4
)

type coreModule interface {
	Start() error
	Stop()
}

func main() {
	flag.Parse()
	if *version {
		fmt.Println("Pulse version:", gitversion)
		os.Exit(0)
	}
	// Set Max Processors for the Pulse agent.
	setMaxProcs()

	c := control.New()
	s := schedule.New(defaultPoolSize, defaultQueueSize)
	s.SetMetricManager(c)

	// Set interrupt handling so we can die gracefully.
	startInterruptHandling(c, s)

	//  Start our modules
	if err := startModule("Plugin Controller", c); err != nil {
		printErrorAndExit("Plugin Controller", err)
	}
	if err := startModule("Scheduler", s); err != nil {
		if c.Started {
			c.Stop()
		}
		printErrorAndExit("Scheduler", err)
	}

	select {} //run forever and ever
}

func setMaxProcs() {
	var _maxProcs int
	envGoMaxProcs := runtime.GOMAXPROCS(-1)
	numProcs := runtime.NumCPU()
	if *maxProcs == 0 && envGoMaxProcs <= numProcs {
		// By default if max_procs is not set, we set _maxProcs to the ENV variable GOMAXPROCS on the system. If this variable is not set by the user or is a negative number, runtime.GOMAXPROCS(-1) returns 1
		_maxProcs = envGoMaxProcs
	} else if envGoMaxProcs > numProcs {
		// We do not allow the user to exceed the number of cores in the system.
		log.Printf("WARNING: ENV variable GOMAXPROCS greater than number of cores in the system. Setting Pulse to use the number of cores in the system.")
		_maxProcs = numProcs
	} else if *maxProcs > 0 && *maxProcs <= numProcs {
		// Our flag override is set. Use this value
		_maxProcs = *maxProcs
	} else if *maxProcs > numProcs {
		// Do not let the user set a value larger than number of cores in the system
		log.Printf("WARNING: Flag max_procs exceeds number of cores in the system. Setting Pulse to use the number of cores in the system")
		_maxProcs = numProcs
	} else if *maxProcs < 0 {
		// Do not let the user set a negative value to get around number of cores limit
		log.Printf("WARNING: Flag max_procs set to negative number. Setting Pulse to use 1 core.")
		_maxProcs = 1
	}

	log.Printf("Setting GOMAXPROCS to %v\n", _maxProcs)
	runtime.GOMAXPROCS(_maxProcs)

	//Verify setting worked
	actualNumProcs := runtime.GOMAXPROCS(0)
	if actualNumProcs != _maxProcs {
		log.Printf("WARNING: Specified max procs of %v but using %v", _maxProcs, actualNumProcs)
	}
}

func startModule(name string, m coreModule) error {
	log.Printf("Starting Pulse Agent %s module", name)
	return m.Start()
}

func printErrorAndExit(name string, err error) {
	log.Println("ERROR:", err)
	log.Printf("ERROR: Error starting Pulse Agent %s module. Exiting now.", name)
	os.Exit(1)
}

func startInterruptHandling(modules ...coreModule) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill, syscall.SIGTERM)

	//Let's block until someone tells us to quit
	go func() {
		sig := <-c
		log.Println("Stopping Pulse Agent modules")
		for _, m := range modules {
			m.Stop()
		}
		log.Printf("Exiting given signal: %v", sig)
		os.Exit(0)
	}()
}
<|endoftext|>


<commit_before>package service

import (
	"context"
	"fmt"
	"time"

	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/log"
)

// CommonMonitoring returns common part of MonitoringStatus
func (c *Common) CommonMonitoring() sdk.MonitoringStatus {
	t := time.Now()
	m := sdk.MonitoringStatus{Now: t}

	m.Lines = append(m.Lines, sdk.MonitoringStatusLine{Component: "Version", Value: sdk.VERSION, Status: sdk.MonitoringStatusOK})
	m.Lines = append(m.Lines, sdk.MonitoringStatusLine{Component: "Uptime", Value: fmt.Sprintf("%s", time.Since(c.StartupTime)), Status: sdk.MonitoringStatusOK})
	m.Lines = append(m.Lines, sdk.MonitoringStatusLine{Component: "Time", Value: fmt.Sprintf("%dh%dm%ds", t.Hour(), t.Minute(), t.Second()), Status: sdk.MonitoringStatusOK})

	return m
}

// Heartbeat have to be launch as a goroutine, call DoHeartBeat each 30s
func (c *Common) Heartbeat(ctx context.Context, status func() sdk.MonitoringStatus) error {
	ticker := time.NewTicker(30 * time.Second)
	var cancel context.CancelFunc
	ctx, cancel = context.WithCancel(ctx)
	defer cancel()

	var heartbeatFailures int
	for {
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-ticker.C:
			if maxHeartbeatFailures, err := c.DoHeartbeat(status); err != nil {
				log.Error("%s> Heartbeat> Heartbeat failed", c.Name)
				heartbeatFailures++
				if heartbeatFailures > maxHeartbeatFailures {
					return fmt.Errorf("%s> Heartbeat> failed excedeed", c.Name)
				}
			}
			heartbeatFailures = 0
		}
	}
}

// DoHeartbeat registers the service for heartbeat
func (c *Common) DoHeartbeat(status func() sdk.MonitoringStatus) (int, error) {
	// no heartbeat for api
	if c.Type == "api" {
		return 0, nil
	}

	srv := sdk.Service{
		Name:             c.Name,
		HTTPURL:          c.HTTPURL,
		LastHeartbeat:    time.Time{},
		Token:            c.Token,
		Type:             c.Type,
		MonitoringStatus: status(),
	}
	log.Debug("%s> DoHeartbeat", c.Name)
	hash, err := c.Client.ServiceRegister(srv)
	if err != nil {
		return 0, sdk.WrapError(err, "DoHeartbeat>")
	}
	c.Hash = hash
	return c.MaxHeartbeatFailures, nil
}
<commit_msg>fix(engine): remove heartbeat log<commit_after>package service

import (
	"context"
	"fmt"
	"time"

	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/log"
)

// CommonMonitoring returns common part of MonitoringStatus
func (c *Common) CommonMonitoring() sdk.MonitoringStatus {
	t := time.Now()
	m := sdk.MonitoringStatus{Now: t}

	m.Lines = append(m.Lines, sdk.MonitoringStatusLine{Component: "Version", Value: sdk.VERSION, Status: sdk.MonitoringStatusOK})
	m.Lines = append(m.Lines, sdk.MonitoringStatusLine{Component: "Uptime", Value: fmt.Sprintf("%s", time.Since(c.StartupTime)), Status: sdk.MonitoringStatusOK})
	m.Lines = append(m.Lines, sdk.MonitoringStatusLine{Component: "Time", Value: fmt.Sprintf("%dh%dm%ds", t.Hour(), t.Minute(), t.Second()), Status: sdk.MonitoringStatusOK})

	return m
}

// Heartbeat have to be launch as a goroutine, call DoHeartBeat each 30s
func (c *Common) Heartbeat(ctx context.Context, status func() sdk.MonitoringStatus) error {
	ticker := time.NewTicker(30 * time.Second)
	var cancel context.CancelFunc
	ctx, cancel = context.WithCancel(ctx)
	defer cancel()

	var heartbeatFailures int
	for {
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-ticker.C:
			if maxHeartbeatFailures, err := c.DoHeartbeat(status); err != nil {
				log.Error("%s> Heartbeat> Heartbeat failed", c.Name)
				heartbeatFailures++
				if heartbeatFailures > maxHeartbeatFailures {
					return fmt.Errorf("%s> Heartbeat> failed excedeed", c.Name)
				}
			}
			heartbeatFailures = 0
		}
	}
}

// DoHeartbeat registers the service for heartbeat
func (c *Common) DoHeartbeat(status func() sdk.MonitoringStatus) (int, error) {
	// no heartbeat for api
	if c.Type == "api" {
		return 0, nil
	}

	srv := sdk.Service{
		Name:             c.Name,
		HTTPURL:          c.HTTPURL,
		LastHeartbeat:    time.Time{},
		Token:            c.Token,
		Type:             c.Type,
		MonitoringStatus: status(),
	}
	hash, err := c.Client.ServiceRegister(srv)
	if err != nil {
		return 0, sdk.WrapError(err, "DoHeartbeat>")
	}
	c.Hash = hash
	return c.MaxHeartbeatFailures, nil
}
<|endoftext|>


<commit_before>package resource

import (
	"fmt"
	"io"

	"github.com/hashicorp/hcl/hcl/ast"
)

// Registry contains all known resources
var Registry = make(map[string]RegistryItem)

// provider is used to create new resources from an HCL AST object item
type provider func(name string, item *ast.ObjectItem) (Resource, error)

// RegistryItem type represents an item from the registry
type RegistryItem struct {
	// Name of the resource type
	Name string

	// Short desription of the resource
	Description string

	// Resource provider
	Provider provider
}

// Register adds a resource type to the registry
func Register(item RegistryItem) error {
	_, ok := Registry[item.Name]
	if ok {
		return fmt.Errorf("Resource type '%s' is already registered", item.Name)
	}

	Registry[item.Name] = item

	return nil
}

// Resource is the interface type for resources
type Resource interface {
	// ID returns the unique identifier of a resource
	ID() string

	// Type returns the type of the resource
	Type() string

	// Returns the name of the resource
	ResourceName() string

	// Validates the resource
	Validate() error

	// Returns the wanted resources/dependencies
	Want() []string

	// Evaluates the resource and returns it's state
	Evaluate() (State, error)

	// Creates the resource
	Create(w io.Writer) error

	// Deletes the resource
	Delete(w io.Writer) error

	// Updates the resource
	Update(w io.Writer) error
}

// BaseResource is the base resource type for all resources
// The purpose of this type is to be embedded into other resources
// Partially implements the Resource interface
type BaseResource struct {
	// Name of the resource
	Name string `hcl:"name" json:"name"`

	// Desired state of the resource
	State string `hcl:"state" json:"state"`

	// Type of the resource
	ResourceType string `json:"-"`

	// Resource dependencies
	WantResource []string `hcl:"want" json:"want,omitempty"`
}

// ID returns the unique resource id
func (b *BaseResource) ID() string {
	return fmt.Sprintf("%s[%s]", b.ResourceType, b.Name)
}

// Type returns the resource type name
func (b *BaseResource) Type() string {
	return b.ResourceType
}

// ResourceName returns the resource name
func (b *BaseResource) ResourceName() string {
	return b.Name
}

// Validate checks if the resource contains valid information
func (b *BaseResource) Validate() error {
	if b.Name == "" {
		return fmt.Errorf("Missing name for resource %s", b.ID())
	}

	return nil
}

// Want returns the wanted resources/dependencies
func (b *BaseResource) Want() []string {
	return b.WantResource
}

// Printf works just like fmt.Printf except that it writes to the
// given resource writer object and prepends the
// resource id to the output
func (b *BaseResource) Printf(w io.Writer, format string, a ...interface{}) (int, error) {
	fmt.Fprintf(w, "%s ", b.ID())

	return fmt.Fprintf(w, format, a...)
}
<commit_msg>Be able to to specify before and after which other resources a resource should be processed<commit_after>package resource

import (
	"fmt"
	"io"

	"github.com/hashicorp/hcl/hcl/ast"
)

// Registry contains all known resources
var Registry = make(map[string]RegistryItem)

// provider is used to create new resources from an HCL AST object item
type provider func(name string, item *ast.ObjectItem) (Resource, error)

// RegistryItem type represents an item from the registry
type RegistryItem struct {
	// Name of the resource type
	Name string

	// Short desription of the resource
	Description string

	// Resource provider
	Provider provider
}

// Register adds a resource type to the registry
func Register(item RegistryItem) error {
	_, ok := Registry[item.Name]
	if ok {
		return fmt.Errorf("Resource type '%s' is already registered", item.Name)
	}

	Registry[item.Name] = item

	return nil
}

// Resource is the interface type for resources
type Resource interface {
	// ID returns the unique identifier of a resource
	ResourceID() string

	// Type returns the type of the resource
	ResourceType() string

	// Returns the name of the resource
	ResourceName() string

	// Validates the resource
	Validate() error

	// Returns the resources before which this resource shoud be processed
	WantBefore() []string

	// Returns the resources after which this resource should be processed
	WantAfter() []string

	// Evaluates the resource
	Evaluate() (State, error)

	// Creates the resource
	Create(w io.Writer) error

	// Deletes the resource
	Delete(w io.Writer) error

	// Updates the resource
	Update(w io.Writer) error
}

// BaseResource is the base resource type for all resources
// The purpose of this type is to be embedded into other resources
// Partially implements the Resource interface
type BaseResource struct {
	// Name of the resource
	Name string `hcl:"name" json:"name"`

	// Desired state of the resource
	State string `hcl:"state" json:"state"`

	// Type of the resource
	Type string `json:"-"`

	// Resources before which this resource should be processed
	Before []string `hcl:"before" json:"before,omitempty"`

	// Resources after which this resource should be processed
	After []string `hcl:"after" json:"after,omitempty"`
}

// ResourceID returns the unique resource id
func (b *BaseResource) ResourceID() string {
	return fmt.Sprintf("%s[%s]", b.Type, b.Name)
}

// ResourceType returns the resource type name
func (b *BaseResource) ResourceType() string {
	return b.Type
}

// ResourceName returns the resource name
func (b *BaseResource) ResourceName() string {
	return b.Name
}

// Validate checks if the resource contains valid information
func (b *BaseResource) Validate() error {
	if b.Name == "" {
		return fmt.Errorf("Missing name for resource %s", b.ResourceID())
	}

	return nil
}

// WantBefore returns the resources before which this resource
// should be processed
func (b *BaseResource) WantBefore() []string {
	return b.Before
}

// WantAfter returns the resources after which this resource
// should be processed
func (b *BaseResource) WantAfter() []string {
	return b.After
}

// Printf works just like fmt.Printf except that it writes to the
// given resource writer object and prepends the
// resource id to the output
func (b *BaseResource) Printf(w io.Writer, format string, a ...interface{}) (int, error) {
	fmt.Fprintf(w, "%s ", b.ResourceID())

	return fmt.Fprintf(w, format, a...)
}
<|endoftext|>