<commit_before>package function

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/square/metrics/api"
	"github.com/square/metrics/inspect"
	"github.com/square/metrics/optimize"
)

// EvaluationContext is the central piece of logic, providing
// helper funcions & varaibles to evaluate a given piece of
// metrics query.
// * Contains a TimeseriesStorageAPI object, which can be used to fetch data
// from the backend systems.
// * Contains current timerange being queried for - this can be
// changed by say, application of time shift function.
type EvaluationContext struct {
	TimeseriesStorageAPI      api.TimeseriesStorageAPI // Backend to fetch data from
	MetricMetadataAPI         api.MetricMetadataAPI    // Api to obtain metadata from
	Timerange                 api.Timerange            // Timerange to fetch data from
	SampleMethod              api.SampleMethod         // SampleMethod to use when up/downsampling to match the requested resolution
	Predicate                 api.Predicate            // Predicate to apply to TagSets prior to fetching
	FetchLimit                FetchCounter             // A limit on the number of fetches which may be performed
	Cancellable               api.Cancellable
	Registry                  Registry
	Profiler                  *inspect.Profiler // A profiler pointer
	OptimizationConfiguration *optimize.OptimizationConfiguration
	EvaluationNotes           *EvaluationNotes //Debug + numerical notes that can be added during evaluation
	invalid                   bool             // Because these can be copied, it's best to mark a no-longer used context as dead
	UserSpecifiableConfig     api.UserSpecifiableConfig
}

type EvaluationNotes struct {
	mutex sync.Mutex
	notes []string
}

func (notes *EvaluationNotes) AddNote(note string) {
	if notes == nil {
		return
	}
	notes.mutex.Lock()
	defer notes.mutex.Unlock()
	notes.notes = append(notes.notes, note)
}
func (notes *EvaluationNotes) Notes() []string {
	if notes == nil {
		return nil
	}
	notes.mutex.Lock()
	defer notes.mutex.Unlock()
	return notes.notes
}

type Registry interface {
	GetFunction(string) (MetricFunction, bool) // returns an instance of MetricFunction
	All() []string                             // all the registered functions
}

// Groups holds grouping information - which tags to group by (if any), and whether to `collapse` (Collapses = true) or `group` (Collapses = false)
type Groups struct {
	List      []string
	Collapses bool
}

// MetricFunction defines a common logic to dispatch a function in MQE.
type MetricFunction struct {
	Name          string
	MinArguments  int
	MaxArguments  int
	AllowsGroupBy bool // Whether the function allows a 'group by' clause.
	Compute       func(EvaluationContext, []Expression, Groups) (Value, error)
}

func (e EvaluationContext) AddNote(note string) {
	e.EvaluationNotes.AddNote(note)
}

func (e EvaluationContext) Notes() []string {
	return e.EvaluationNotes.Notes()
}

func (e EvaluationContext) WithTimerange(t api.Timerange) EvaluationContext {
	e.Timerange = t
	return e
}

// Evaluate the given metric function.
func (f MetricFunction) Evaluate(context EvaluationContext,
	arguments []Expression, groupBy []string, collapses bool) (Value, error) {
	if context.invalid {
		panic("Attempted to evaluate a function on an EvaluationContext that's been explicitly invalidated.")
	}
	// preprocessing
	length := len(arguments)
	if length < f.MinArguments || (f.MaxArguments != -1 && f.MaxArguments < length) {
		return nil, ArgumentLengthError{f.Name, f.MinArguments, f.MaxArguments, length}
	}
	if len(groupBy) > 0 && !f.AllowsGroupBy {
		// TODO(jee) - use typed errors
		return nil, fmt.Errorf("function %s doesn't allow a group-by clause", f.Name)
	}
	return f.Compute(context, arguments, Groups{groupBy, collapses})
}

// FetchCounter is used to count the number of fetches remaining in a thread-safe manner.
type FetchCounter struct {
	count *int32
	limit int
}

func NewFetchCounter(n int) FetchCounter {
	n32 := int32(n)
	return FetchCounter{
		count: &n32,
		limit: n,
	}
}

// Limit returns the max # of fetches allowed by this counter.
func (c FetchCounter) Limit() int {
	return c.limit
}

func (c FetchCounter) Current() int {
	return c.limit - int(atomic.LoadInt32(c.count))
}

// Consume decrements the internal counter and returns whether the result is at least 0.
// It does so in a threadsafe manner.
func (c FetchCounter) Consume(n int) bool {
	return atomic.AddInt32(c.count, -int32(n)) >= 0
}

// Expression is a piece of code, which can be evaluated in a given
// EvaluationContext. EvaluationContext must never be changed in an Evalute().
//
// The contract of Expressions is that leaf nodes must sample a resulting
// timeseries according to the resolution specified in its EvaluationContext's
// Timerange. Internal nodes may assume that results from evaluating child
// Expressions correspond to the timerange in the current EvaluationContext.
type Expression interface {
	// Evaluate the given expression.
	Evaluate(context EvaluationContext) (Value, error)
}

func EvaluateToScalar(e Expression, context EvaluationContext) (float64, error) {
	scalarValue, err := e.Evaluate(context)
	if err != nil {
		return 0, err
	}
	return scalarValue.ToScalar()
}
func EvaluateToDuration(e Expression, context EvaluationContext) (time.Duration, error) {
	scalarValue, err := e.Evaluate(context)
	if err != nil {
		return 0, err
	}
	return scalarValue.ToDuration()
}
func EvaluateToSeriesList(e Expression, context EvaluationContext) (api.SeriesList, error) {
	seriesValue, err := e.Evaluate(context)
	if err != nil {
		return api.SeriesList{}, err
	}
	return seriesValue.ToSeriesList(context.Timerange)
}

// EvaluateMany evaluates a list of expressions using a single EvaluationContext.
// If any evaluation errors, EvaluateMany will propagate that error. The resulting values
// will be in the order corresponding to the provided expressions.
func EvaluateMany(context EvaluationContext, expressions []Expression) ([]Value, error) {
	if context.invalid {
		panic("Attempted to evaluate a function on an EvaluationContext that's been explicitly invalidated.")
	}

	type result struct {
		index int
		err   error
		value Value
	}
	length := len(expressions)
	if length == 0 {
		return []Value{}, nil
	} else if length == 1 {
		result, err := expressions[0].Evaluate(context)
		if err != nil {
			return nil, err
		}
		return []Value{result}, nil
	} else {
		// concurrent evaluations
		results := make(chan result, length)
		for i, expr := range expressions {
			go func(i int, expr Expression) {
				value, err := expr.Evaluate(context)
				results <- result{i, err, value}
			}(i, expr)
		}
		array := make([]Value, length)
		for i := 0; i < length; i++ {
			result := <-results
			if result.err != nil {
				return nil, result.err
			} else {
				array[result.index] = result.value
			}
		}
		return array, nil
	}
}
<commit_msg>remove the invalid field<commit_after>package function

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/square/metrics/api"
	"github.com/square/metrics/inspect"
	"github.com/square/metrics/optimize"
)

// EvaluationContext is the central piece of logic, providing
// helper funcions & varaibles to evaluate a given piece of
// metrics query.
// * Contains a TimeseriesStorageAPI object, which can be used to fetch data
// from the backend systems.
// * Contains current timerange being queried for - this can be
// changed by say, application of time shift function.
type EvaluationContext struct {
	TimeseriesStorageAPI      api.TimeseriesStorageAPI // Backend to fetch data from
	MetricMetadataAPI         api.MetricMetadataAPI    // Api to obtain metadata from
	Timerange                 api.Timerange            // Timerange to fetch data from
	SampleMethod              api.SampleMethod         // SampleMethod to use when up/downsampling to match the requested resolution
	Predicate                 api.Predicate            // Predicate to apply to TagSets prior to fetching
	FetchLimit                FetchCounter             // A limit on the number of fetches which may be performed
	Cancellable               api.Cancellable
	Registry                  Registry
	Profiler                  *inspect.Profiler // A profiler pointer
	OptimizationConfiguration *optimize.OptimizationConfiguration
	EvaluationNotes           *EvaluationNotes //Debug + numerical notes that can be added during evaluation
	UserSpecifiableConfig     api.UserSpecifiableConfig
}

type EvaluationNotes struct {
	mutex sync.Mutex
	notes []string
}

func (notes *EvaluationNotes) AddNote(note string) {
	if notes == nil {
		return
	}
	notes.mutex.Lock()
	defer notes.mutex.Unlock()
	notes.notes = append(notes.notes, note)
}
func (notes *EvaluationNotes) Notes() []string {
	if notes == nil {
		return nil
	}
	notes.mutex.Lock()
	defer notes.mutex.Unlock()
	return notes.notes
}

type Registry interface {
	GetFunction(string) (MetricFunction, bool) // returns an instance of MetricFunction
	All() []string                             // all the registered functions
}

// Groups holds grouping information - which tags to group by (if any), and whether to `collapse` (Collapses = true) or `group` (Collapses = false)
type Groups struct {
	List      []string
	Collapses bool
}

// MetricFunction defines a common logic to dispatch a function in MQE.
type MetricFunction struct {
	Name          string
	MinArguments  int
	MaxArguments  int
	AllowsGroupBy bool // Whether the function allows a 'group by' clause.
	Compute       func(EvaluationContext, []Expression, Groups) (Value, error)
}

func (e EvaluationContext) AddNote(note string) {
	e.EvaluationNotes.AddNote(note)
}

func (e EvaluationContext) Notes() []string {
	return e.EvaluationNotes.Notes()
}

func (e EvaluationContext) WithTimerange(t api.Timerange) EvaluationContext {
	e.Timerange = t
	return e
}

// Evaluate the given metric function.
func (f MetricFunction) Evaluate(context EvaluationContext,
	arguments []Expression, groupBy []string, collapses bool) (Value, error) {
	// preprocessing
	length := len(arguments)
	if length < f.MinArguments || (f.MaxArguments != -1 && f.MaxArguments < length) {
		return nil, ArgumentLengthError{f.Name, f.MinArguments, f.MaxArguments, length}
	}
	if len(groupBy) > 0 && !f.AllowsGroupBy {
		// TODO(jee) - use typed errors
		return nil, fmt.Errorf("function %s doesn't allow a group-by clause", f.Name)
	}
	return f.Compute(context, arguments, Groups{groupBy, collapses})
}

// FetchCounter is used to count the number of fetches remaining in a thread-safe manner.
type FetchCounter struct {
	count *int32
	limit int
}

func NewFetchCounter(n int) FetchCounter {
	n32 := int32(n)
	return FetchCounter{
		count: &n32,
		limit: n,
	}
}

// Limit returns the max # of fetches allowed by this counter.
func (c FetchCounter) Limit() int {
	return c.limit
}

func (c FetchCounter) Current() int {
	return c.limit - int(atomic.LoadInt32(c.count))
}

// Consume decrements the internal counter and returns whether the result is at least 0.
// It does so in a threadsafe manner.
func (c FetchCounter) Consume(n int) bool {
	return atomic.AddInt32(c.count, -int32(n)) >= 0
}

// Expression is a piece of code, which can be evaluated in a given
// EvaluationContext. EvaluationContext must never be changed in an Evalute().
//
// The contract of Expressions is that leaf nodes must sample a resulting
// timeseries according to the resolution specified in its EvaluationContext's
// Timerange. Internal nodes may assume that results from evaluating child
// Expressions correspond to the timerange in the current EvaluationContext.
type Expression interface {
	// Evaluate the given expression.
	Evaluate(context EvaluationContext) (Value, error)
}

func EvaluateToScalar(e Expression, context EvaluationContext) (float64, error) {
	scalarValue, err := e.Evaluate(context)
	if err != nil {
		return 0, err
	}
	return scalarValue.ToScalar()
}
func EvaluateToDuration(e Expression, context EvaluationContext) (time.Duration, error) {
	scalarValue, err := e.Evaluate(context)
	if err != nil {
		return 0, err
	}
	return scalarValue.ToDuration()
}
func EvaluateToSeriesList(e Expression, context EvaluationContext) (api.SeriesList, error) {
	seriesValue, err := e.Evaluate(context)
	if err != nil {
		return api.SeriesList{}, err
	}
	return seriesValue.ToSeriesList(context.Timerange)
}

// EvaluateMany evaluates a list of expressions using a single EvaluationContext.
// If any evaluation errors, EvaluateMany will propagate that error. The resulting values
// will be in the order corresponding to the provided expressions.
func EvaluateMany(context EvaluationContext, expressions []Expression) ([]Value, error) {
	type result struct {
		index int
		err   error
		value Value
	}
	length := len(expressions)
	if length == 0 {
		return []Value{}, nil
	} else if length == 1 {
		result, err := expressions[0].Evaluate(context)
		if err != nil {
			return nil, err
		}
		return []Value{result}, nil
	} else {
		// concurrent evaluations
		results := make(chan result, length)
		for i, expr := range expressions {
			go func(i int, expr Expression) {
				value, err := expr.Evaluate(context)
				results <- result{i, err, value}
			}(i, expr)
		}
		array := make([]Value, length)
		for i := 0; i < length; i++ {
			result := <-results
			if result.err != nil {
				return nil, result.err
			} else {
				array[result.index] = result.value
			}
		}
		return array, nil
	}
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aip0136

import (
	"strings"

	"bitbucket.org/creachadair/stringset"
	"github.com/googleapis/api-linter/lint"
	"github.com/googleapis/api-linter/rules/internal/utils"
	"github.com/jhump/protoreflect/desc"
)

var standardMethodsOnly = &lint.MethodRule{
	Name:   lint.NewRuleName(136, "declarative-standard-methods-only"),
	OnlyIf: utils.IsDeclarativeFriendlyMethod,
	LintMethod: func(m *desc.MethodDescriptor) []lint.Problem {
		// Standard methods are fine.
		standard := stringset.New("Get", "List", "Create", "Update", "Delete", "Undelete", "Batch")
		for s := range standard {
			if strings.HasPrefix(m.GetName(), s) {
				return nil
			}
		}

		// This is likely to have a non-trivial number of exceptions, and a
		// traditional linter disable may not be appropriate.
		//
		// Therefore, we allow "Imperative only." in an internal comment to make
		// this not complain.
		if cmt := m.GetSourceInfo().GetLeadingComments(); strings.Contains(strings.ToLower(cmt), "imperative only") {
			return nil
		}

		// Okay, complain.
		return []lint.Problem{{
			Message: strings.Join([]string{
				"Declarative-friendly resources should generally avoid custom methods.\n",
				"However, if this is an imperative-only mnethod that does *not* need ",
				`declarative tooling support, add the text "Imperative only." to the comment. `,
				"(Using an internal comment is fine.)",
			}, ""),
			Descriptor: m,
		}}
	},
}
<commit_msg>mnethod -> method (#713)<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aip0136

import (
	"strings"

	"bitbucket.org/creachadair/stringset"
	"github.com/googleapis/api-linter/lint"
	"github.com/googleapis/api-linter/rules/internal/utils"
	"github.com/jhump/protoreflect/desc"
)

var standardMethodsOnly = &lint.MethodRule{
	Name:   lint.NewRuleName(136, "declarative-standard-methods-only"),
	OnlyIf: utils.IsDeclarativeFriendlyMethod,
	LintMethod: func(m *desc.MethodDescriptor) []lint.Problem {
		// Standard methods are fine.
		standard := stringset.New("Get", "List", "Create", "Update", "Delete", "Undelete", "Batch")
		for s := range standard {
			if strings.HasPrefix(m.GetName(), s) {
				return nil
			}
		}

		// This is likely to have a non-trivial number of exceptions, and a
		// traditional linter disable may not be appropriate.
		//
		// Therefore, we allow "Imperative only." in an internal comment to make
		// this not complain.
		if cmt := m.GetSourceInfo().GetLeadingComments(); strings.Contains(strings.ToLower(cmt), "imperative only") {
			return nil
		}

		// Okay, complain.
		return []lint.Problem{{
			Message: strings.Join([]string{
				"Declarative-friendly resources should generally avoid custom methods.\n",
				"However, if this is an imperative-only method that does *not* need ",
				`declarative tooling support, add the text "Imperative only." to the comment. `,
				"(Using an internal comment is fine.)",
			}, ""),
			Descriptor: m,
		}}
	},
}
<|endoftext|>


<commit_before>// This source file is part of the Packet Guardian project.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tasks

import (
	"fmt"
	"os"
	"path/filepath"
	"time"

	"github.com/usi-lfkeitel/packet-guardian/src/common"
)

func init() {
	RegisterJob("Purge old web sessions", cleanUpExpiredSessions)
}

func cleanUpExpiredSessions(e *common.Environment) (string, error) {
	w := sessionWalker{n: time.Now().Add(time.Duration(-24) * time.Hour)}
	err := filepath.Walk("sessions", w.sessionDirWalker)
	if err != nil {
		return "", err
	}
	return fmt.Sprintf("Purged %d sessions", w.c), nil
}

type sessionWalker struct {
	n time.Time
	c int
}

func (s sessionWalker) sessionDirWalker(path string, info os.FileInfo, err error) error {
	if info.IsDir() {
		return filepath.SkipDir
	}
	if info.Size() < 100*1024 {
		s.c++
		return os.Remove(path)
	}
	return nil
}
<commit_msg>Fixed session cleanup job<commit_after>// This source file is part of the Packet Guardian project.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tasks

import (
	"fmt"
	"os"
	"path/filepath"
	"time"

	"github.com/usi-lfkeitel/packet-guardian/src/common"
)

func init() {
	RegisterJob("Purge old web sessions", cleanUpExpiredSessions)
}

func cleanUpExpiredSessions(e *common.Environment) (string, error) {
	w := sessionWalker{n: time.Now().Add(time.Duration(-24) * time.Hour)}
	err := filepath.Walk("sessions", w.sessionDirWalker)
	if err != nil {
		return "", err
	}
	return fmt.Sprintf("Purged %d sessions", w.c), nil
}

type sessionWalker struct {
	n time.Time
	c int
}

func (s sessionWalker) sessionDirWalker(path string, info os.FileInfo, err error) error {
	if info.IsDir() {
		return filepath.SkipDir
	}
	if info.ModTime().Before(s.n) {
		s.c++
		return os.Remove(path)
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright ©2016 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
 * Cephes Math Library Release 2.4:  March,1996
 * Copyright 1984, 1996 by Stephen L. Moshier
 */

/*
 * Adapted from scipy's cephes unity.c
 */

package cephes

import "math"

const (
	invSqrt2 = 1 / math.Sqrt2
	pi4      = math.Pi / 4
	euler    = 0.577215664901532860606512090082402431 // Euler constant
)

/* Coefficients for log(1+x) = x - x**2/2 + x**3 P(x)/Q(x)
 * 1/sqrt(2) <= x < sqrt(2)
 * Theoretical peak relative error = 2.32e-20
 */
var lP = []float64{
	4.5270000862445199635215E-5,
	4.9854102823193375972212E-1,
	6.5787325942061044846969E0,
	2.9911919328553073277375E1,
	6.0949667980987787057556E1,
	5.7112963590585538103336E1,
	2.0039553499201281259648E1,
}

var lQ = []float64{
	1.5062909083469192043167E1,
	8.3047565967967209469434E1,
	2.2176239823732856465394E2,
	3.0909872225312059774938E2,
	2.1642788614495947685003E2,
	6.0118660497603843919306E1,
}

// log1p computes log(1 + x)
func log1p(x float64) float64 {
	z := 1 + x
	if z < invSqrt2 || z > math.Sqrt2 {
		return math.Log(z)
	}
	z = x * x
	z = -0.5*z + x*(z*polevl(x, lP, 6)/p1evl(x, lQ, 6))
	return x + z
}

// log1pmx computes log(1 + x) - x
func log1pmx(x float64) float64 {
	if math.Abs(x) < 0.5 {
		xfac := x
		res := 0.0

		var term float64
		for n := 2; n < maxIter; n++ {
			xfac *= -x
			term = xfac / float64(n)
			res += term
			if math.Abs(term) < machEp*math.Abs(res) {
				break
			}
		}
		return res
	}
	return log1p(x) - x
}

/*  e^x =  1 + 2x P(x^2)/( Q(x^2) - P(x^2) )
 * -0.5 <= x <= 0.5
 */

var eP = []float64{
	1.2617719307481059087798E-4,
	3.0299440770744196129956E-2,
	9.9999999999999999991025E-1,
}

var eQ = []float64{
	3.0019850513866445504159E-6,
	2.5244834034968410419224E-3,
	2.2726554820815502876593E-1,
	2.0000000000000000000897E0,
}

// expm1 computes expm1(x) = exp(x) - 1
func expm1(x float64) float64 {
	if math.IsInf(x, 0) {
		if math.IsNaN(x) || x > 0 {
			return x
		}
		return -1
	}
	if x < -0.5 || x > 0.5 {
		return math.Exp(x) - 1
	}
	xx := x * x
	r := x * polevl(xx, eP, 2)
	r = r / (polevl(xx, eQ, 3) - r)
	return r + r
}

var coscof = []float64{
	4.7377507964246204691685E-14,
	-1.1470284843425359765671E-11,
	2.0876754287081521758361E-9,
	-2.7557319214999787979814E-7,
	2.4801587301570552304991E-5,
	-1.3888888888888872993737E-3,
	4.1666666666666666609054E-2,
}

// cosm1 computes cosm1(x) = cos(x) - 1
func cosm1(x float64) float64 {
	if x < -pi4 || x > pi4 {
		return math.Cos(x) - 1
	}
	xx := x * x
	xx = -0.5*xx + xx*xx*polevl(xx, coscof, 6)
	return xx
}

// lgam1pTayler computes lgam(x + 1) around x = 0 using its Taylor series.
func lgam1pTaylor(x float64) float64 {
	if x == 0 {
		return 0
	}
	res := -euler * x
	xfac := -x
	for n := 2; n < 42; n++ {
		nf := float64(n)
		xfac *= -x
		coeff := Zeta(nf, 1) * xfac / nf
		res += coeff
		if math.Abs(coeff) < machEp*math.Abs(res) {
			break
		}
	}

	return res
}

// lgam1p computes lgam(x + 1).
func lgam1p(x float64) float64 {
	if math.Abs(x) <= 0.5 {
		return lgam1pTaylor(x)
	} else if math.Abs(x-1) < 0.5 {
		return math.Log(x) + lgam1pTaylor(x-1)
	}
	return lgam(x + 1)
}
<commit_msg>Updating unity copyright and comments<commit_after>// Derived from SciPy's special/cephes/unity.c
// https://github.com/scipy/scipy/blob/master/scipy/special/cephes/unity.c
// Made freely available by Stephen L. Moshier without support or guarantee.

// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Copyright ©1984, ©1996 by Stephen L. Moshier
// Portions Copyright ©2016 The gonum Authors. All rights reserved.

package cephes

import "math"

const (
	invSqrt2 = 1 / math.Sqrt2
	pi4      = math.Pi / 4
	euler    = 0.577215664901532860606512090082402431 // Euler constant
)

// Coefficients for
//  log(1+x) = x - \frac{x^2}{2} + \frac{x^3 lP(x)}{lQ(x)}
// for
//  \frac{1}{\sqrt{2}} <= x < \sqrt{2}
// Theoretical peak relative error = 2.32e-20
var lP = []float64{
	4.5270000862445199635215E-5,
	4.9854102823193375972212E-1,
	6.5787325942061044846969E0,
	2.9911919328553073277375E1,
	6.0949667980987787057556E1,
	5.7112963590585538103336E1,
	2.0039553499201281259648E1,
}

var lQ = []float64{
	1.5062909083469192043167E1,
	8.3047565967967209469434E1,
	2.2176239823732856465394E2,
	3.0909872225312059774938E2,
	2.1642788614495947685003E2,
	6.0118660497603843919306E1,
}

// log1p computes
//  log(1 + x)
func log1p(x float64) float64 {
	z := 1 + x
	if z < invSqrt2 || z > math.Sqrt2 {
		return math.Log(z)
	}
	z = x * x
	z = -0.5*z + x*(z*polevl(x, lP, 6)/p1evl(x, lQ, 6))
	return x + z
}

// log1pmx computes
//  log(1 + x) - x
func log1pmx(x float64) float64 {
	if math.Abs(x) < 0.5 {
		xfac := x
		res := 0.0

		var term float64
		for n := 2; n < maxIter; n++ {
			xfac *= -x
			term = xfac / float64(n)
			res += term
			if math.Abs(term) < machEp*math.Abs(res) {
				break
			}
		}
		return res
	}
	return log1p(x) - x
}

// Coefficients for
//  e^x = 1 + \frac{2x eP(x^2)}{eQ(x^2) - eP(x^2)}
// for
//  -0.5 <= x <= 0.5
var eP = []float64{
	1.2617719307481059087798E-4,
	3.0299440770744196129956E-2,
	9.9999999999999999991025E-1,
}

var eQ = []float64{
	3.0019850513866445504159E-6,
	2.5244834034968410419224E-3,
	2.2726554820815502876593E-1,
	2.0000000000000000000897E0,
}

// expm1 computes
//  expm1(x) = e^x - 1
func expm1(x float64) float64 {
	if math.IsInf(x, 0) {
		if math.IsNaN(x) || x > 0 {
			return x
		}
		return -1
	}
	if x < -0.5 || x > 0.5 {
		return math.Exp(x) - 1
	}
	xx := x * x
	r := x * polevl(xx, eP, 2)
	r = r / (polevl(xx, eQ, 3) - r)
	return r + r
}

var coscof = []float64{
	4.7377507964246204691685E-14,
	-1.1470284843425359765671E-11,
	2.0876754287081521758361E-9,
	-2.7557319214999787979814E-7,
	2.4801587301570552304991E-5,
	-1.3888888888888872993737E-3,
	4.1666666666666666609054E-2,
}

// cosm1 computes
//  cosm1(x) = cos(x) - 1
func cosm1(x float64) float64 {
	if x < -pi4 || x > pi4 {
		return math.Cos(x) - 1
	}
	xx := x * x
	xx = -0.5*xx + xx*xx*polevl(xx, coscof, 6)
	return xx
}

// lgam1pTayler computes
//  lgam(x + 1)
//around x = 0 using its Taylor series.
func lgam1pTaylor(x float64) float64 {
	if x == 0 {
		return 0
	}
	res := -euler * x
	xfac := -x
	for n := 2; n < 42; n++ {
		nf := float64(n)
		xfac *= -x
		coeff := Zeta(nf, 1) * xfac / nf
		res += coeff
		if math.Abs(coeff) < machEp*math.Abs(res) {
			break
		}
	}

	return res
}

// lgam1p computes
//  lgam(x + 1)
func lgam1p(x float64) float64 {
	if math.Abs(x) <= 0.5 {
		return lgam1pTaylor(x)
	} else if math.Abs(x-1) < 0.5 {
		return math.Log(x) + lgam1pTaylor(x-1)
	}
	return lgam(x + 1)
}
<|endoftext|>


<commit_before>package users

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"log"
	"net/http"

	"github.com/bradleyfalzon/gopherci-web/internal/session"
	"github.com/google/go-github/github"
	"github.com/google/uuid"
	"github.com/pkg/errors"

	"golang.org/x/oauth2"
)

// OAuthLoginHandler starts the initial oauth login flow by redirecting the
// user to GitHub for authentication and authorisation our app.
func (um *UserManager) OAuthLoginHandler(w http.ResponseWriter, r *http.Request) (int, error) {
	session := session.FromContext(r.Context())
	session.GitHubOAuthState = uuid.New()

	url := um.oauthConf.AuthCodeURL(session.GitHubOAuthState.String(), oauth2.AccessTypeOnline)
	http.Redirect(w, r, url, http.StatusTemporaryRedirect)
	return 0, nil
}

// OAuthCallbackHandler handles the callback after GitHub authentication and
// persists the credentials to storage for use later.
func (um *UserManager) OAuthCallbackHandler(w http.ResponseWriter, r *http.Request) (int, error) {
	session := session.FromContext(r.Context())

	if session.GitHubOAuthState == uuid.Nil {
		return http.StatusBadRequest, errors.New("github: invalid oauth state from session, it is nil")
	}

	state := r.FormValue("state")
	if state != session.GitHubOAuthState.String() {
		return http.StatusBadRequest, fmt.Errorf("github: invalid oauth state from session, have %q, want %q", state, session.GitHubOAuthState.String())
	}
	session.GitHubOAuthState = uuid.Nil

	code := r.FormValue("code")
	token, err := um.oauthConf.Exchange(oauth2.NoContext, code)
	if err != nil {
		return http.StatusInternalServerError, errors.Wrap(err, "github: oauthConf exchange() error")
	}

	client := NewClient(um.oauthConf, token)
	ghUser, _, err := client.Users.Get("")
	if err != nil {
		return http.StatusInternalServerError, errors.Wrap(err, "github: could not get user")
	}
	log.Printf("github: logged in as GitHub user: %s\n", *ghUser.Login)

	// Set session to this user
	id, err := um.GitHubLogin(*ghUser.ID, token)
	if err != nil {
		return http.StatusInternalServerError, errors.Wrap(err, "github: could not set github user in db")
	}
	session.UserID = id

	http.Redirect(w, r, "/console", http.StatusTemporaryRedirect)
	return 0, nil
}

// NewClient returns a github.Client using oauthconf and token.
func NewClient(oauthConf *oauth2.Config, token *oauth2.Token) *github.Client {
	oauthClient := oauthConf.Client(oauth2.NoContext, token)
	client := github.NewClient(oauthClient)
	//if um.overwriteBaseURL != "" {
	//client.BaseURL, _ = url.Parse(um.overwriteBaseURL)
	//}
	return client
}

// GitHubLogin assigns the token to an existing user with the given githubID,
// if the user does not exist, the user is created. If an error occurs err is
// non-nil, else the userID of the user is returned.
func (um *UserManager) GitHubLogin(githubID int, token *oauth2.Token) (userID int, err error) {
	jsonToken, err := json.Marshal(token)
	if err != nil {
		return 0, errors.Wrap(err, "could not marshal oauth2.token")
	}

	// Get user's email
	client := NewClient(um.oauthConf, token)
	user, _, err := client.Users.Get("")
	if err != nil {
		return 0, errors.Wrap(err, "error getting email for new user")
	}

	err = um.db.QueryRow("SELECT id FROM users WHERE github_id = ?", githubID).Scan(&userID)
	switch {
	case err == sql.ErrNoRows:
		// Add token to new user
		res, err := um.db.Exec("INSERT INTO users (email, github_id, github_token) VALUES (?, ?, ?)", *user.Email, githubID, jsonToken)
		if err != nil {
			return 0, errors.Wrapf(err, "error inserting new githubID %q", githubID)
		}
		id, err := res.LastInsertId()
		if err != nil {
			return 0, errors.Wrap(err, "error in lastInsertId")
		}

		return int(id), nil
	case err != nil:
		return 0, errors.Wrapf(err, "error getting userID for githubID %q", githubID)
	}

	// Add token to existing user and update email
	_, err = um.db.Exec("UPDATE users SET email = ?, github_token = ? WHERE id = ?", *user.Email, jsonToken, userID)
	if err != nil {
		return 0, errors.Wrapf(err, "could set userID %q github_token", userID)
	}
	return userID, nil
}
<commit_msg>Get user's email from correct api endpoint<commit_after>package users

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"log"
	"net/http"

	"github.com/bradleyfalzon/gopherci-web/internal/session"
	"github.com/google/go-github/github"
	"github.com/google/uuid"
	"github.com/pkg/errors"

	"golang.org/x/oauth2"
)

// OAuthLoginHandler starts the initial oauth login flow by redirecting the
// user to GitHub for authentication and authorisation our app.
func (um *UserManager) OAuthLoginHandler(w http.ResponseWriter, r *http.Request) (int, error) {
	session := session.FromContext(r.Context())
	session.GitHubOAuthState = uuid.New()

	url := um.oauthConf.AuthCodeURL(session.GitHubOAuthState.String(), oauth2.AccessTypeOnline)
	http.Redirect(w, r, url, http.StatusTemporaryRedirect)
	return 0, nil
}

// OAuthCallbackHandler handles the callback after GitHub authentication and
// persists the credentials to storage for use later.
func (um *UserManager) OAuthCallbackHandler(w http.ResponseWriter, r *http.Request) (int, error) {
	session := session.FromContext(r.Context())

	if session.GitHubOAuthState == uuid.Nil {
		return http.StatusBadRequest, errors.New("github: invalid oauth state from session, it is nil")
	}

	state := r.FormValue("state")
	if state != session.GitHubOAuthState.String() {
		return http.StatusBadRequest, fmt.Errorf("github: invalid oauth state from session, have %q, want %q", state, session.GitHubOAuthState.String())
	}
	session.GitHubOAuthState = uuid.Nil

	code := r.FormValue("code")
	token, err := um.oauthConf.Exchange(oauth2.NoContext, code)
	if err != nil {
		return http.StatusInternalServerError, errors.Wrap(err, "github: oauthConf exchange() error")
	}

	client := NewClient(um.oauthConf, token)
	ghUser, _, err := client.Users.Get("")
	if err != nil {
		return http.StatusInternalServerError, errors.Wrap(err, "github: could not get user")
	}
	log.Printf("github: logged in as GitHub user: %s\n", *ghUser.Login)

	// Set session to this user
	id, err := um.GitHubLogin(*ghUser.ID, token)
	if err != nil {
		return http.StatusInternalServerError, errors.Wrap(err, "github: could not set github user in db")
	}
	session.UserID = id

	http.Redirect(w, r, "/console", http.StatusTemporaryRedirect)
	return 0, nil
}

// NewClient returns a github.Client using oauthconf and token.
func NewClient(oauthConf *oauth2.Config, token *oauth2.Token) *github.Client {
	oauthClient := oauthConf.Client(oauth2.NoContext, token)
	client := github.NewClient(oauthClient)
	//if um.overwriteBaseURL != "" {
	//client.BaseURL, _ = url.Parse(um.overwriteBaseURL)
	//}
	return client
}

// GitHubLogin assigns the token to an existing user with the given githubID,
// if the user does not exist, the user is created. If an error occurs err is
// non-nil, else the userID of the user is returned.
func (um *UserManager) GitHubLogin(githubID int, token *oauth2.Token) (userID int, err error) {
	jsonToken, err := json.Marshal(token)
	if err != nil {
		return 0, errors.Wrap(err, "could not marshal oauth2.token")
	}

	// Get user's email
	client := NewClient(um.oauthConf, token)
	emails, _, err := client.Users.ListEmails(nil)
	if err != nil {
		return 0, errors.Wrap(err, "error getting email for new user")
	}
	var email string
	for _, e := range emails {
		if *e.Primary && *e.Verified {
			email = *e.Email
			break
		}
	}
	if email == "" {
		return 0, errors.New("could not get user's primary verified email from GitHub")
	}

	err = um.db.QueryRow("SELECT id FROM users WHERE github_id = ?", githubID).Scan(&userID)
	switch {
	case err == sql.ErrNoRows:
		// Add token to new user
		res, err := um.db.Exec("INSERT INTO users (email, github_id, github_token) VALUES (?, ?, ?)", email, githubID, jsonToken)
		if err != nil {
			return 0, errors.Wrapf(err, "error inserting new githubID %q", githubID)
		}
		id, err := res.LastInsertId()
		if err != nil {
			return 0, errors.Wrap(err, "error in lastInsertId")
		}
		return int(id), nil
	case err != nil:
		return 0, errors.Wrapf(err, "error getting userID for githubID %q", githubID)
	}

	// Add token to existing user and update email
	_, err = um.db.Exec("UPDATE users SET email = ?, github_token = ? WHERE id = ?", email, jsonToken, userID)
	if err != nil {
		return 0, errors.Wrapf(err, "could set userID %q github_token", userID)
	}
	return userID, nil
}
<|endoftext|>


<commit_before>package virt

import (
	"os"

	"github.com/jimmyfrasche/etlite/internal/device"
	"github.com/jimmyfrasche/etlite/internal/driver"
	"github.com/jimmyfrasche/etlite/internal/format"
	"github.com/jimmyfrasche/etlite/internal/format/csvfmt"
	"github.com/jimmyfrasche/etlite/internal/internal/eol"
	"github.com/jimmyfrasche/etlite/internal/internal/errint"
	"github.com/jimmyfrasche/etlite/internal/internal/null"
	"github.com/jimmyfrasche/etlite/internal/virt/internal/sysdb"
)

//Spec specifies the defaults for a Machine.
type Spec struct {
	//Database is the name of the SQLite db file or the empty string
	//for an in-memory database.
	Database string
	//Input is the initial input device, or stdin if nil.
	Input device.Reader
	//Output is the initial output device, or stdout if nil.
	Output device.Writer
	//Encoder is the initial encoder,
	//or a csv encoder with platform specific EOL encoding if nil.
	Encoder format.Encoder
	//Decoder is the intital decoder,
	//or a csv decoder with platform specific EOL encoding if nil.
	Decoder format.Decoder
	//Environ is the environment to use to populate sys.env,
	//or os.Environ if nil.
	Environ []string
	//Args is used to populate sys.arg.
	//There is no default for Args.
	Args []string
}

//Machine represents an execution context for a script.
type Machine struct {
	name    string
	conn    *driver.Conn
	output  device.Writer
	input   device.Reader
	encoder format.Encoder
	decoder format.Decoder

	sys                        *sysdb.Sysdb
	savepointStmt, releaseStmt *driver.Stmt

	stack            []interface{}
	derivedTableName string
}

//New creates and prepares an execution context.
func New(savepoints []string, s Spec) (*Machine, error) {
	db := s.Database
	if db == "" {
		db = ":memory:"
	}
	c, err := driver.Open(db)
	if err != nil {
		return nil, err
	}
	o := s.Output
	if o == nil {
		o = device.Stdout
	}
	in := s.Input
	if in == nil {
		in = device.Stdin
	}
	e := s.Encoder
	if e == nil {
		e = &csvfmt.Encoder{UseCRLF: eol.Default}
	}
	dec := s.Decoder
	if dec == nil {
		dec = &csvfmt.Decoder{}
	}
	m := &Machine{
		name:    db,
		conn:    c,
		output:  o,
		input:   in,
		encoder: e,
		decoder: dec,
		stack:   make([]interface{}, 0, 128),
	}
	if s.Environ == nil {
		s.Environ = os.Environ()
	}

	m.sys, err = sysdb.New(m.conn, s.Args, s.Environ)
	if err != nil {
		return nil, err
	}

	m.savepointStmt, err = m.conn.Prepare(`SAVEPOINT [1]`)
	if err != nil {
		return nil, errint.Wrap(err)
	}
	m.releaseStmt, err = m.conn.Prepare(`RELEASE SAVEPOINT [1]`)
	if err != nil {
		return nil, errint.Wrap(err)
	}

	return m, nil
}

//Close flushes and closes output and cleans up
//all tracked resources associated with the context.
//It does not track resources allocated by Instructions:
//that is the responsibility of an individual Instruction.
func (m *Machine) Close() (errs []error) {
	err := func(err error) {
		if err != nil {
			errs = append(errs, err)
		}
	}
	o := m.output
	err(o.Flush())
	err(o.Close())
	err(m.input.Close())
	err(m.encoder.Close())
	err(m.decoder.Close())
	err(m.sys.Close())
	err(m.savepointStmt.Close())
	err(m.releaseStmt.Close())
	err(m.conn.Close())
	return
}

//Environ dumps sys.env (which the user is free to modify)
//in Go readable format.
func (m *Machine) Environ() ([]string, error) {
	return m.sys.Environ()
}

//Name reports the name of the main database.
func (m *Machine) Name() string {
	return m.name
}

func (m *Machine) push(what interface{}) {
	m.stack = append(m.stack, what)
}

func (m *Machine) pop() (val interface{}, err error) {
	if len(m.stack) == 0 {
		return nil, errint.New("stack underflow")
	}
	end := len(m.stack) - 1
	val, m.stack[end] = m.stack[end], nil
	m.stack = m.stack[:end]
	return val, nil
}

//PopString pops a *string off the stack.
func (m *Machine) PopString() (*string, error) {
	v, err := m.pop()
	if err != nil {
		return nil, err
	}
	s, ok := v.(*string)
	if !ok {
		return nil, errint.Newf("expected *string on stack but found %#v", v)
	}
	return s, nil
}

//PopStrings pops a []string off the stack.
func (m *Machine) PopStrings() ([]string, error) {
	v, err := m.pop()
	if err != nil {
		return nil, err
	}
	s, ok := v.([]string)
	if !ok {
		return nil, errint.Newf("expected []string on stack but found %#v", v)
	}
	return s, nil
}

//PopNullEncoding pops a null.Encoding off the stack.
func (m *Machine) PopNullEncoding() (null.Encoding, error) {
	v, err := m.pop()
	if err != nil {
		return "", err
	}
	n, ok := v.(null.Encoding)
	if !ok {
		return "", errint.Newf("expected null encoding on stack but found %#v", v)
	}
	return n, nil
}

func (m *Machine) PopBool() (bool, error) {
	v, err := m.pop()
	if err != nil {
		return false, err
	}
	b, ok := v.(bool)
	if !ok {
		return false, errint.Newf("expected bool on stack but found %#v", v)
	}
	return b, nil
}

func (m *Machine) PopInt() (int, error) {
	v, err := m.pop()
	if err != nil {
		return 0, err
	}
	i, ok := v.(int)
	if !ok {
		return 0, errint.Newf("expected int on stack but found %#v", v)
	}
	return i, nil
}

func (m *Machine) PopRune() (rune, error) {
	v, err := m.pop()
	if err != nil {
		return 0, err
	}
	r, ok := v.(rune)
	if !ok {
		return 0, errint.Newf("expected rune on stack but found %#v", v)
	}
	return r, nil
}

func (m *Machine) SetOutput(o device.Writer) error {
	if o == nil {
		return errint.New("no output device specified")
	}
	if m.output == nil {
		return errint.New("no previous output device")
	}
	if err := m.output.Close(); err != nil {
		return err
	}

	m.output = o
	return nil
}

func (m *Machine) SetInput(in device.Reader, derivedTableName string) error {
	if in == nil {
		return errint.New("no input device specified")
	}
	if m.input == nil {
		return errint.New("no previous input device")
	}
	if err := m.input.Close(); err != nil {
		return err
	}

	m.input, m.derivedTableName = in, derivedTableName
	return nil
}

//DerivedTableName returns the derivedTableName from the last call to SetInput.
func (m *Machine) DerivedTableName() string { //TODO need to factor into importer
	return m.derivedTableName
}

func (m *Machine) SetDecoder(d format.Decoder) error {
	if d == nil {
		return errint.New("no decoder specified")
	}
	if m.decoder == nil {
		return errint.New("no previous decoder")
	}
	if err := m.decoder.Close(); err != nil {
		return err
	}
	m.decoder = d
	return nil
}

func (m *Machine) SetEncoder(e format.Encoder) error {
	if e == nil {
		return errint.New("no encoder specified")
	}
	if m.encoder == nil {
		return errint.New("no previous encoder")
	}
	if err := m.encoder.Close(); err != nil {
		return err
	}
	m.encoder = e
	return nil
}

//exec q.
func (m *Machine) exec(q string) error {
	s, err := m.conn.Prepare(q)
	if err != nil {
		return err
	}
	defer s.Close()

	return s.Exec()
}

func (m *Machine) subquery(q string) (*string, error) {
	s, err := m.conn.Prepare(q)
	if err != nil {
		return nil, err
	}
	defer s.Close()

	return s.Subquery()
}
<commit_msg>change default IO to raw, with no headers<commit_after>package virt

import (
	"os"

	"github.com/jimmyfrasche/etlite/internal/device"
	"github.com/jimmyfrasche/etlite/internal/driver"
	"github.com/jimmyfrasche/etlite/internal/format"
	"github.com/jimmyfrasche/etlite/internal/format/rawfmt"
	"github.com/jimmyfrasche/etlite/internal/internal/eol"
	"github.com/jimmyfrasche/etlite/internal/internal/errint"
	"github.com/jimmyfrasche/etlite/internal/internal/null"
	"github.com/jimmyfrasche/etlite/internal/virt/internal/sysdb"
)

//Spec specifies the defaults for a Machine.
type Spec struct {
	//Database is the name of the SQLite db file or the empty string
	//for an in-memory database.
	Database string
	//Input is the initial input device, or stdin if nil.
	Input device.Reader
	//Output is the initial output device, or stdout if nil.
	Output device.Writer
	//Encoder is the initial encoder,
	//or a csv encoder with platform specific EOL encoding if nil.
	Encoder format.Encoder
	//Decoder is the intital decoder,
	//or a csv decoder with platform specific EOL encoding if nil.
	Decoder format.Decoder
	//Environ is the environment to use to populate sys.env,
	//or os.Environ if nil.
	Environ []string
	//Args is used to populate sys.arg.
	//There is no default for Args.
	Args []string
}

//Machine represents an execution context for a script.
type Machine struct {
	name    string
	conn    *driver.Conn
	output  device.Writer
	input   device.Reader
	encoder format.Encoder
	decoder format.Decoder

	sys                        *sysdb.Sysdb
	savepointStmt, releaseStmt *driver.Stmt

	stack            []interface{}
	derivedTableName string
}

//New creates and prepares an execution context.
func New(savepoints []string, s Spec) (*Machine, error) {
	db := s.Database
	if db == "" {
		db = ":memory:"
	}
	c, err := driver.Open(db)
	if err != nil {
		return nil, err
	}
	o := s.Output
	if o == nil {
		o = device.Stdout
	}
	in := s.Input
	if in == nil {
		in = device.Stdin
	}
	e := s.Encoder
	if e == nil {
		e = &rawfmt.Encoder{
			UseCRLF:  eol.Default,
			NoHeader: true,
		}
	}
	dec := s.Decoder
	if dec == nil {
		dec = &rawfmt.Decoder{
			UseCRLF:  eol.Default,
			NoHeader: true,
		}
	}
	m := &Machine{
		name:    db,
		conn:    c,
		output:  o,
		input:   in,
		encoder: e,
		decoder: dec,
		stack:   make([]interface{}, 0, 128),
	}
	if s.Environ == nil {
		s.Environ = os.Environ()
	}

	m.sys, err = sysdb.New(m.conn, s.Args, s.Environ)
	if err != nil {
		return nil, err
	}

	m.savepointStmt, err = m.conn.Prepare(`SAVEPOINT [1]`)
	if err != nil {
		return nil, errint.Wrap(err)
	}
	m.releaseStmt, err = m.conn.Prepare(`RELEASE SAVEPOINT [1]`)
	if err != nil {
		return nil, errint.Wrap(err)
	}

	return m, nil
}

//Close flushes and closes output and cleans up
//all tracked resources associated with the context.
//It does not track resources allocated by Instructions:
//that is the responsibility of an individual Instruction.
func (m *Machine) Close() (errs []error) {
	err := func(err error) {
		if err != nil {
			errs = append(errs, err)
		}
	}
	o := m.output
	err(o.Flush())
	err(o.Close())
	err(m.input.Close())
	err(m.encoder.Close())
	err(m.decoder.Close())
	err(m.sys.Close())
	err(m.savepointStmt.Close())
	err(m.releaseStmt.Close())
	err(m.conn.Close())
	return
}

//Environ dumps sys.env (which the user is free to modify)
//in Go readable format.
func (m *Machine) Environ() ([]string, error) {
	return m.sys.Environ()
}

//Name reports the name of the main database.
func (m *Machine) Name() string {
	return m.name
}

func (m *Machine) push(what interface{}) {
	m.stack = append(m.stack, what)
}

func (m *Machine) pop() (val interface{}, err error) {
	if len(m.stack) == 0 {
		return nil, errint.New("stack underflow")
	}
	end := len(m.stack) - 1
	val, m.stack[end] = m.stack[end], nil
	m.stack = m.stack[:end]
	return val, nil
}

//PopString pops a *string off the stack.
func (m *Machine) PopString() (*string, error) {
	v, err := m.pop()
	if err != nil {
		return nil, err
	}
	s, ok := v.(*string)
	if !ok {
		return nil, errint.Newf("expected *string on stack but found %#v", v)
	}
	return s, nil
}

//PopStrings pops a []string off the stack.
func (m *Machine) PopStrings() ([]string, error) {
	v, err := m.pop()
	if err != nil {
		return nil, err
	}
	s, ok := v.([]string)
	if !ok {
		return nil, errint.Newf("expected []string on stack but found %#v", v)
	}
	return s, nil
}

//PopNullEncoding pops a null.Encoding off the stack.
func (m *Machine) PopNullEncoding() (null.Encoding, error) {
	v, err := m.pop()
	if err != nil {
		return "", err
	}
	n, ok := v.(null.Encoding)
	if !ok {
		return "", errint.Newf("expected null encoding on stack but found %#v", v)
	}
	return n, nil
}

func (m *Machine) PopBool() (bool, error) {
	v, err := m.pop()
	if err != nil {
		return false, err
	}
	b, ok := v.(bool)
	if !ok {
		return false, errint.Newf("expected bool on stack but found %#v", v)
	}
	return b, nil
}

func (m *Machine) PopInt() (int, error) {
	v, err := m.pop()
	if err != nil {
		return 0, err
	}
	i, ok := v.(int)
	if !ok {
		return 0, errint.Newf("expected int on stack but found %#v", v)
	}
	return i, nil
}

func (m *Machine) PopRune() (rune, error) {
	v, err := m.pop()
	if err != nil {
		return 0, err
	}
	r, ok := v.(rune)
	if !ok {
		return 0, errint.Newf("expected rune on stack but found %#v", v)
	}
	return r, nil
}

func (m *Machine) SetOutput(o device.Writer) error {
	if o == nil {
		return errint.New("no output device specified")
	}
	if m.output == nil {
		return errint.New("no previous output device")
	}
	if err := m.output.Close(); err != nil {
		return err
	}

	m.output = o
	return nil
}

func (m *Machine) SetInput(in device.Reader, derivedTableName string) error {
	if in == nil {
		return errint.New("no input device specified")
	}
	if m.input == nil {
		return errint.New("no previous input device")
	}
	if err := m.input.Close(); err != nil {
		return err
	}

	m.input, m.derivedTableName = in, derivedTableName
	return nil
}

//DerivedTableName returns the derivedTableName from the last call to SetInput.
func (m *Machine) DerivedTableName() string { //TODO need to factor into importer
	return m.derivedTableName
}

func (m *Machine) SetDecoder(d format.Decoder) error {
	if d == nil {
		return errint.New("no decoder specified")
	}
	if m.decoder == nil {
		return errint.New("no previous decoder")
	}
	if err := m.decoder.Close(); err != nil {
		return err
	}
	m.decoder = d
	return nil
}

func (m *Machine) SetEncoder(e format.Encoder) error {
	if e == nil {
		return errint.New("no encoder specified")
	}
	if m.encoder == nil {
		return errint.New("no previous encoder")
	}
	if err := m.encoder.Close(); err != nil {
		return err
	}
	m.encoder = e
	return nil
}

//exec q.
func (m *Machine) exec(q string) error {
	s, err := m.conn.Prepare(q)
	if err != nil {
		return err
	}
	defer s.Close()

	return s.Exec()
}

func (m *Machine) subquery(q string) (*string, error) {
	s, err := m.conn.Prepare(q)
	if err != nil {
		return nil, err
	}
	defer s.Close()

	return s.Subquery()
}
<|endoftext|>


<commit_before>package gdsl
func init() { GITLASTTAG = "v1.0.1"; GITLASTCOMMIT = "a0e8c3b5d5bb043a22de8428d50a9513a4458fd4" }
<commit_msg>v1.1.0 latest<commit_after>package gdsl
func init() { GITLASTTAG = "v1.1.0"; GITLASTCOMMIT = "33f37342f63655b3688fbe61f286d38b6dd23617" }
<|endoftext|>


<commit_before>// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tag

import (
	"context"
	"fmt"
	"strings"

	"github.com/hashicorp/go-multierror"
	admit_v1 "k8s.io/api/admissionregistration/v1"
	kerrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"

	"istio.io/api/label"
)

func GetTagWebhooks(ctx context.Context, client kubernetes.Interface) ([]admit_v1.MutatingWebhookConfiguration, error) {
	webhooks, err := client.AdmissionregistrationV1().MutatingWebhookConfigurations().List(ctx, metav1.ListOptions{
		LabelSelector: IstioTagLabel,
	})
	if err != nil {
		return nil, err
	}
	return webhooks.Items, nil
}

// GetWebhooksWithTag returns webhooks tagged with istio.io/tag=<tag>.
func GetWebhooksWithTag(ctx context.Context, client kubernetes.Interface, tag string) ([]admit_v1.MutatingWebhookConfiguration, error) {
	webhooks, err := client.AdmissionregistrationV1().MutatingWebhookConfigurations().List(ctx, metav1.ListOptions{
		LabelSelector: fmt.Sprintf("%s=%s", IstioTagLabel, tag),
	})
	if err != nil {
		return nil, err
	}
	return webhooks.Items, nil
}

// GetWebhooksWithRevision returns webhooks tagged with istio.io/rev=<rev> and NOT TAGGED with istio.io/tag.
// this retrieves the webhook created at revision installation rather than tag webhooks
func GetWebhooksWithRevision(ctx context.Context, client kubernetes.Interface, rev string) ([]admit_v1.MutatingWebhookConfiguration, error) {
	webhooks, err := client.AdmissionregistrationV1().MutatingWebhookConfigurations().List(ctx, metav1.ListOptions{
		LabelSelector: fmt.Sprintf("%s=%s,!%s", label.IoIstioRev.Name, rev, IstioTagLabel),
	})
	if err != nil {
		return nil, err
	}
	return webhooks.Items, nil
}

// GetNamespacesWithTag retrieves all namespaces pointed at the given tag.
func GetNamespacesWithTag(ctx context.Context, client kubernetes.Interface, tag string) ([]string, error) {
	namespaces, err := client.CoreV1().Namespaces().List(ctx, metav1.ListOptions{
		LabelSelector: fmt.Sprintf("%s=%s", label.IoIstioRev.Name, tag),
	})
	if err != nil {
		return nil, err
	}

	nsNames := make([]string, len(namespaces.Items))
	for i, ns := range namespaces.Items {
		nsNames[i] = ns.Name
	}
	return nsNames, nil
}

// GetWebhookTagName extracts tag name from webhook object.
func GetWebhookTagName(wh admit_v1.MutatingWebhookConfiguration) (string, error) {
	if tagName, ok := wh.ObjectMeta.Labels[IstioTagLabel]; ok {
		return tagName, nil
	}
	return "", fmt.Errorf("could not extract tag name from webhook")
}

// GetWebhookRevision extracts tag target revision from webhook object.
func GetWebhookRevision(wh admit_v1.MutatingWebhookConfiguration) (string, error) {
	if tagName, ok := wh.ObjectMeta.Labels[label.IoIstioRev.Name]; ok {
		return tagName, nil
	}
	return "", fmt.Errorf("could not extract tag revision from webhook")
}

// DeleteTagWebhooks deletes the given webhooks.
func DeleteTagWebhooks(ctx context.Context, client kubernetes.Interface, tag string) error {
	webhooks, err := GetWebhooksWithTag(ctx, client, tag)
	if err != nil {
		return err
	}
	var result error
	for _, wh := range webhooks {
		result = multierror.Append(client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(ctx, wh.Name, metav1.DeleteOptions{})).ErrorOrNil()
	}
	return result
}

// DeleteDeprecatedValidator deletes the deprecated validating webhook configuration. This is used after a user explicitly
// sets a new default revision.
func DeleteDeprecatedValidator(ctx context.Context, client kubernetes.Interface) error {
	vwhs, err := client.AdmissionregistrationV1().ValidatingWebhookConfigurations().List(ctx, metav1.ListOptions{
		LabelSelector: "app=istiod",
	})
	if err != nil {
		return err
	}
	var errs *multierror.Error
	for _, vwh := range vwhs.Items {
		// hacky but we want to remove the validators that used to be in base, not the per-revision webhooks.
		if !strings.Contains(vwh.Name, "validator") {
			errs = multierror.Append(errs,
				client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(ctx, vwh.Name, metav1.DeleteOptions{}))
		}
	}
	if kerrors.IsNotFound(err) {
		return nil
	}
	return errs.ErrorOrNil()
}

var neverMatch = &metav1.LabelSelector{
	MatchLabels: map[string]string{
		"istio.io/deactivated": "never-match",
	},
}

// PreviousInstallExists checks whether there is an existing Istio installation. Should be used in installer when deciding
// whether to make an installation the default.
func PreviousInstallExists(ctx context.Context, client kubernetes.Interface) bool {
	mwhs, err := client.AdmissionregistrationV1().MutatingWebhookConfigurations().List(ctx, metav1.ListOptions{
		LabelSelector: "app=sidecar-injector",
	})
	if err != nil {
		return false
	}
	return len(mwhs.Items) > 0
}

// DeactivateIstioInjectionWebhook deactivates the istio-injection webhook from the given MutatingWebhookConfiguration if exists.
// used rather than just deleting the webhook since we want to keep it around after changing the default so user can later
// switch back to it. This is a hack but it is meant to cover a corner case where a user wants to migrate from a non-revisioned
// old version and then later decides to switch back to the old revision again.
func DeactivateIstioInjectionWebhook(ctx context.Context, client kubernetes.Interface) error {
	whs, err := GetWebhooksWithRevision(ctx, client, DefaultRevisionName)
	if err != nil {
		return err
	}
	if len(whs) == 0 {
		// no revision with default, no action required.
		return nil
	}
	if len(whs) > 1 {
		return fmt.Errorf("expected a single webhook for default revision")
	}
	webhook := whs[0]
	for i := range webhook.Webhooks {
		wh := webhook.Webhooks[i]
		// this is an abomination, but if this isn't a per-revision webhook, we want to make it ineffectual
		// without deleting it. Add a nonsense match.
		wh.NamespaceSelector = neverMatch
		wh.ObjectSelector = neverMatch
		webhook.Webhooks[i] = wh
	}
	admit := client.AdmissionregistrationV1().MutatingWebhookConfigurations()
	_, err = admit.Update(ctx, &webhook, metav1.UpdateOptions{})
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>fi (#39760)<commit_after>// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tag

import (
	"context"
	"fmt"
	"strings"

	"github.com/hashicorp/go-multierror"
	admit_v1 "k8s.io/api/admissionregistration/v1"
	kerrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"

	"istio.io/api/label"
)

func GetTagWebhooks(ctx context.Context, client kubernetes.Interface) ([]admit_v1.MutatingWebhookConfiguration, error) {
	webhooks, err := client.AdmissionregistrationV1().MutatingWebhookConfigurations().List(ctx, metav1.ListOptions{
		LabelSelector: IstioTagLabel,
	})
	if err != nil {
		return nil, err
	}
	return webhooks.Items, nil
}

// GetWebhooksWithTag returns webhooks tagged with istio.io/tag=<tag>.
func GetWebhooksWithTag(ctx context.Context, client kubernetes.Interface, tag string) ([]admit_v1.MutatingWebhookConfiguration, error) {
	webhooks, err := client.AdmissionregistrationV1().MutatingWebhookConfigurations().List(ctx, metav1.ListOptions{
		LabelSelector: fmt.Sprintf("%s=%s", IstioTagLabel, tag),
	})
	if err != nil {
		return nil, err
	}
	return webhooks.Items, nil
}

// GetWebhooksWithRevision returns webhooks tagged with istio.io/rev=<rev> and NOT TAGGED with istio.io/tag.
// this retrieves the webhook created at revision installation rather than tag webhooks
func GetWebhooksWithRevision(ctx context.Context, client kubernetes.Interface, rev string) ([]admit_v1.MutatingWebhookConfiguration, error) {
	webhooks, err := client.AdmissionregistrationV1().MutatingWebhookConfigurations().List(ctx, metav1.ListOptions{
		LabelSelector: fmt.Sprintf("%s=%s,!%s", label.IoIstioRev.Name, rev, IstioTagLabel),
	})
	if err != nil {
		return nil, err
	}
	return webhooks.Items, nil
}

// GetNamespacesWithTag retrieves all namespaces pointed at the given tag.
func GetNamespacesWithTag(ctx context.Context, client kubernetes.Interface, tag string) ([]string, error) {
	namespaces, err := client.CoreV1().Namespaces().List(ctx, metav1.ListOptions{
		LabelSelector: fmt.Sprintf("%s=%s", label.IoIstioRev.Name, tag),
	})
	if err != nil {
		return nil, err
	}

	nsNames := make([]string, len(namespaces.Items))
	for i, ns := range namespaces.Items {
		nsNames[i] = ns.Name
	}
	return nsNames, nil
}

// GetWebhookTagName extracts tag name from webhook object.
func GetWebhookTagName(wh admit_v1.MutatingWebhookConfiguration) (string, error) {
	if tagName, ok := wh.ObjectMeta.Labels[IstioTagLabel]; ok {
		return tagName, nil
	}
	return "", fmt.Errorf("could not extract tag name from webhook")
}

// GetWebhookRevision extracts tag target revision from webhook object.
func GetWebhookRevision(wh admit_v1.MutatingWebhookConfiguration) (string, error) {
	if tagName, ok := wh.ObjectMeta.Labels[label.IoIstioRev.Name]; ok {
		return tagName, nil
	}
	return "", fmt.Errorf("could not extract tag revision from webhook")
}

// DeleteTagWebhooks deletes the given webhooks.
func DeleteTagWebhooks(ctx context.Context, client kubernetes.Interface, tag string) error {
	webhooks, err := GetWebhooksWithTag(ctx, client, tag)
	if err != nil {
		return err
	}
	var result error
	for _, wh := range webhooks {
		result = multierror.Append(result, client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(ctx, wh.Name, metav1.DeleteOptions{})).ErrorOrNil()
	}
	return result
}

// DeleteDeprecatedValidator deletes the deprecated validating webhook configuration. This is used after a user explicitly
// sets a new default revision.
func DeleteDeprecatedValidator(ctx context.Context, client kubernetes.Interface) error {
	vwhs, err := client.AdmissionregistrationV1().ValidatingWebhookConfigurations().List(ctx, metav1.ListOptions{
		LabelSelector: "app=istiod",
	})
	if err != nil {
		return err
	}
	var errs *multierror.Error
	for _, vwh := range vwhs.Items {
		// hacky but we want to remove the validators that used to be in base, not the per-revision webhooks.
		if !strings.Contains(vwh.Name, "validator") {
			errs = multierror.Append(errs,
				client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(ctx, vwh.Name, metav1.DeleteOptions{}))
		}
	}
	if kerrors.IsNotFound(err) {
		return nil
	}
	return errs.ErrorOrNil()
}

var neverMatch = &metav1.LabelSelector{
	MatchLabels: map[string]string{
		"istio.io/deactivated": "never-match",
	},
}

// PreviousInstallExists checks whether there is an existing Istio installation. Should be used in installer when deciding
// whether to make an installation the default.
func PreviousInstallExists(ctx context.Context, client kubernetes.Interface) bool {
	mwhs, err := client.AdmissionregistrationV1().MutatingWebhookConfigurations().List(ctx, metav1.ListOptions{
		LabelSelector: "app=sidecar-injector",
	})
	if err != nil {
		return false
	}
	return len(mwhs.Items) > 0
}

// DeactivateIstioInjectionWebhook deactivates the istio-injection webhook from the given MutatingWebhookConfiguration if exists.
// used rather than just deleting the webhook since we want to keep it around after changing the default so user can later
// switch back to it. This is a hack but it is meant to cover a corner case where a user wants to migrate from a non-revisioned
// old version and then later decides to switch back to the old revision again.
func DeactivateIstioInjectionWebhook(ctx context.Context, client kubernetes.Interface) error {
	whs, err := GetWebhooksWithRevision(ctx, client, DefaultRevisionName)
	if err != nil {
		return err
	}
	if len(whs) == 0 {
		// no revision with default, no action required.
		return nil
	}
	if len(whs) > 1 {
		return fmt.Errorf("expected a single webhook for default revision")
	}
	webhook := whs[0]
	for i := range webhook.Webhooks {
		wh := webhook.Webhooks[i]
		// this is an abomination, but if this isn't a per-revision webhook, we want to make it ineffectual
		// without deleting it. Add a nonsense match.
		wh.NamespaceSelector = neverMatch
		wh.ObjectSelector = neverMatch
		webhook.Webhooks[i] = wh
	}
	admit := client.AdmissionregistrationV1().MutatingWebhookConfigurations()
	_, err = admit.Update(ctx, &webhook, metav1.UpdateOptions{})
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Project Gonder.
// Author Supme
// Copyright Supme 2016
// License http://opensource.org/licenses/MIT MIT License
//
//  THE SOFTWARE AND DOCUMENTATION ARE PROVIDED "AS IS" WITHOUT WARRANTY OF
//  ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
//  IMPLIED WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR
//  PURPOSE.
//
// Please see the License.txt file for more information.
//
package campaign

import (
	"github.com/supme/gonder/mailer"
	"github.com/supme/gonder/models"
	"errors"
	"time"
	"strconv"
	"math/rand"
)

type (
	campaign struct {
		id, from_email, from_name, subject, body, iface, host string
		send_unsubscribe bool
		stream, resend_delay, resend_count int
		attachments []mailer.Attachment
		recipients  []recipient
	}

	recipient struct {
		id, to_email, to_name	string
	}
)

func (c *campaign) getNullRecipients() {
	var d recipient
	c.recipients = nil

	query, err := models.Db.Prepare("SELECT `id`, `email`, `name` FROM recipient WHERE campaign_id=? AND removed=0 AND status IS NULL")
	checkErr(err)
	defer query.Close()

	//q, err := query.Query(c.id, c.stream)
	q, err := query.Query(c.id)
	checkErr(err)
	defer q.Close()

	for q.Next() {
		err = q.Scan(&d.id, &d.to_email, &d.to_name)
		checkErr(err)
		c.recipients = append(c.recipients, d)
	}
}

func (c *campaign) getSoftBounceRecipients() {
	var	d recipient
	c.recipients = nil

	query, err := models.Db.Prepare("SELECT DISTINCT r.`id`, r.`email`, r.`name` FROM `recipient` as r,`status` as s WHERE r.`campaign_id`=? AND r.`removed`=0 AND s.`bounce_id`=2 AND UPPER(`r`.`status`) LIKE CONCAT(\"%\",s.`pattern`,\"%\")")
	checkErr(err)
	defer query.Close()

	q, err := query.Query(c.id)
	checkErr(err)
	defer q.Close()

	for q.Next() {
		err = q.Scan(&d.id, &d.to_email, &d.to_name)
		checkErr(err)
		c.recipients = append(c.recipients, d)
	}

}

func (r *recipient) unsubscribe(campaignId string) bool {
	var unsubscribeCount int
	models.Db.QueryRow("SELECT COUNT(*) FROM `unsubscribe` t1 INNER JOIN `campaign` t2 ON t1.group_id = t2.group_id WHERE t2.id = ? AND t1.email = ?", campaignId, r.to_email).Scan(&unsubscribeCount)
	if unsubscribeCount == 0 {
		return false
	}
	return true
}

func (r recipient) send(c *campaign) string {
	data := new(mailer.MailData)
	data.Iface = c.iface
	data.From_email = c.from_email
	data.From_name = c.from_name
	data.Host = c.host
	data.Attachments = c.attachments
	data.To_email = r.to_email
	data.To_name = r.to_name

	message := new(models.Message)
	message.CampaignId = c.id
	message.RecipientId = r.id
	message.CampaignSubject = c.subject
	message.CampaignTemplate = c.body
	message.RecipientEmail = r.to_email
	message.RecipientName = r.to_name

	var rs string
	m, e := message.RenderMessage()
	if e == nil {
		data.Subject = message.CampaignSubject
		data.Html = m
		data.Extra_header = "List-Unsubscribe: " + message.UnsubscribeMailLink() + "\r\nPrecedence: bulk\r\n"
		data.Extra_header += "Message-ID: <" + strconv.FormatInt(time.Now().Unix(), 10) + c.id + "." + r.id +"@" + c.host + ">" + "\r\n"

		var res error
		if models.Config.RealSend {
			// Send mail
			res = data.Send()
		} else {
			wait := time.Duration(rand.Int()/10000000000) * time.Nanosecond
			time.Sleep(wait)
			res = errors.New("Test send")
		}

		if res == nil {
			rs = "Ok"
		} else {
			rs = res.Error()
		}
	} else {
		rs = e.Error()
	}

	camplog.Printf("Campaign %s for recipient id %s email %s is %s", c.id, r.id, data.To_email, rs)
	return rs
}

func (c *campaign) get(id string) {
	models.Db.QueryRow("SELECT t1.`id`,t3.`email`,t3.`name`,t1.`subject`,t1.`body`,t2.`iface`,t2.`host`,t2.`stream`,t1.`send_unsubscribe`,t2.`resend_delay`,t2.`resend_count` FROM `campaign` t1 INNER JOIN `profile` t2 ON t2.`id`=t1.`profile_id` INNER JOIN `sender` t3 ON t3.`id`=t1.`from_id` WHERE t1.`id`=?", id).Scan(
		c.id,
		c.from_email,
		c.from_name,
		c.subject,
		c.body,
		c.iface,
		c.host,
		c.stream,
		c.send_unsubscribe,
		c.resend_delay,
		c.resend_count,
	)
}

func (c *campaign) getAttachments() {
	attachment, err := models.Db.Prepare("SELECT `path`, `file` FROM attachment WHERE campaign_id=?")
	checkErr(err)
	defer attachment.Close()

	attach, err := attachment.Query(c.id)
	checkErr(err)
	defer attach.Close()

	c.attachments = nil
	var location string
	var name string
	for attach.Next() {
		err = attach.Scan(&location, &name)
		checkErr(err)
		c.attachments = append(c.attachments, mailer.Attachment{Location: location, Name: name})
	}
}

func (c campaign) send() {
	count := 0
	stream := 0
	next := make(chan bool)

	c.getNullRecipients()
	camplog.Printf("Start campaign %s. Count recipients %d", c.id, len(c.recipients))

	for _, r := range c.recipients {
		count += 1
		if r.unsubscribe(c.id) == false  || c.send_unsubscribe {
			models.Db.Exec("UPDATE recipient SET status='Sending', date=NOW() WHERE id=?", r.id)
			stream += 1
			if stream > c.stream {
				<-next
				stream -= 1
			}
			go func(d recipient) {
				rs := d.send(&c)
				models.Db.Exec("UPDATE recipient SET status=?, date=NOW() WHERE id=?", rs, d.id)
				next <- true
			}(r)
		} else {
			models.Db.Exec("UPDATE recipient SET status='Unsubscribe', date=NOW() WHERE id=?", r.id)
			camplog.Printf("Recipient id %s email %s is unsubscribed", r.id, r.to_email)
		}
	}

	for stream != 0 {
		<-next
		stream -= 1
	}
	close(next)

	camplog.Printf("Done campaign %s. Count %d", c.id, count)
}

func (c campaign) resendSoftBounce() {
	c.getSoftBounceRecipients()
	if c.resend_count != 0 {
		camplog.Printf("Start %d resend by campaign id %s ", len(c.recipients), c.id)
	}
	if len(c.recipients) == 0 {
		return
	}

	for n := 0; n < c.resend_count; n++ {
		time.Sleep(time.Duration(c.resend_delay) * time.Second)
		c.getSoftBounceRecipients()
		for _, r := range c.recipients {
			// если пользователь ни разу не отказался от подписки в этой группе
			var unsubscribeCount int
			models.Db.QueryRow("SELECT COUNT(*) FROM `unsubscribe` t1 INNER JOIN `campaign` t2 ON t1.group_id = t2.group_id WHERE t2.id = ? AND t1.email = ?", c.id, r.to_email).Scan(&unsubscribeCount)
			if unsubscribeCount == 0 || c.send_unsubscribe {
				models.Db.Exec("UPDATE recipient SET status='Sending', date=NOW() WHERE id=?", r.id)
				rs := r.send(&c)
				models.Db.Exec("UPDATE recipient SET status=?, date=NOW() WHERE id=?", rs, r.id)
			} else {
				models.Db.Exec("UPDATE recipient SET status='Unsubscribe', date=NOW() WHERE id=?", r.id)
				camplog.Printf("Recipient id %s email %s is unsubscribed", r.id, r.to_email)
			}
		}
	}
}

func checkErr(err error) {
	if err != nil {
		camplog.Println(err)
	}
}
<commit_msg>small fix<commit_after>// Project Gonder.
// Author Supme
// Copyright Supme 2016
// License http://opensource.org/licenses/MIT MIT License
//
//  THE SOFTWARE AND DOCUMENTATION ARE PROVIDED "AS IS" WITHOUT WARRANTY OF
//  ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
//  IMPLIED WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR
//  PURPOSE.
//
// Please see the License.txt file for more information.
//
package campaign

import (
	"github.com/supme/gonder/mailer"
	"github.com/supme/gonder/models"
	"errors"
	"time"
	"strconv"
	"math/rand"
	"bytes"
)

type (
	campaign struct {
		id, from_email, from_name, subject, body, iface, host string
		send_unsubscribe bool
		stream, resend_delay, resend_count int
		attachments []mailer.Attachment
		recipients  []recipient
	}

	recipient struct {
		id, to_email, to_name	string
	}
)

func (c *campaign) getNullRecipients() {
	var d recipient
	c.recipients = nil

	query, err := models.Db.Prepare("SELECT `id`, `email`, `name` FROM recipient WHERE campaign_id=? AND removed=0 AND status IS NULL")
	checkErr(err)
	defer query.Close()

	//q, err := query.Query(c.id, c.stream)
	q, err := query.Query(c.id)
	checkErr(err)
	defer q.Close()

	for q.Next() {
		err = q.Scan(&d.id, &d.to_email, &d.to_name)
		checkErr(err)
		c.recipients = append(c.recipients, d)
	}
}

func (c *campaign) getSoftBounceRecipients() {
	var	d recipient
	c.recipients = nil

	query, err := models.Db.Prepare("SELECT DISTINCT r.`id`, r.`email`, r.`name` FROM `recipient` as r,`status` as s WHERE r.`campaign_id`=? AND r.`removed`=0 AND s.`bounce_id`=2 AND UPPER(`r`.`status`) LIKE CONCAT(\"%\",s.`pattern`,\"%\")")
	checkErr(err)
	defer query.Close()

	q, err := query.Query(c.id)
	checkErr(err)
	defer q.Close()

	for q.Next() {
		err = q.Scan(&d.id, &d.to_email, &d.to_name)
		checkErr(err)
		c.recipients = append(c.recipients, d)
	}

}

func (r *recipient) unsubscribe(campaignId string) bool {
	var unsubscribeCount int
	models.Db.QueryRow("SELECT COUNT(*) FROM `unsubscribe` t1 INNER JOIN `campaign` t2 ON t1.group_id = t2.group_id WHERE t2.id = ? AND t1.email = ?", campaignId, r.to_email).Scan(&unsubscribeCount)
	if unsubscribeCount == 0 {
		return false
	}
	return true
}

func (r recipient) send(c *campaign) string {
	data := new(mailer.MailData)
	data.Iface = c.iface
	data.From_email = c.from_email
	data.From_name = c.from_name
	data.Host = c.host
	data.Attachments = c.attachments
	data.To_email = r.to_email
	data.To_name = r.to_name

	message := new(models.Message)
	message.CampaignId = c.id
	message.RecipientId = r.id
	message.CampaignSubject = c.subject
	message.CampaignTemplate = c.body
	message.RecipientEmail = r.to_email
	message.RecipientName = r.to_name

	var rs string
	m, e := message.RenderMessage()
	if e == nil {
		data.Subject = message.CampaignSubject
		data.Html = m
		var extraHeader bytes.Buffer
		extraHeader.WriteString("List-Unsubscribe: " + message.UnsubscribeMailLink() + "\r\nPrecedence: bulk\r\n")
		extraHeader.WriteString("Message-ID: <" + strconv.FormatInt(time.Now().Unix(), 10) + c.id + "." + r.id +"@" + c.host + ">" + "\r\n")
		data.Extra_header = extraHeader.String()

		var res error
		if models.Config.RealSend {
			// Send mail
			res = data.Send()
		} else {
			wait := time.Duration(rand.Int()/10000000000) * time.Nanosecond
			time.Sleep(wait)
			res = errors.New("Test send")
		}

		if res == nil {
			rs = "Ok"
		} else {
			rs = res.Error()
		}
	} else {
		rs = e.Error()
	}

	camplog.Printf("Campaign %s for recipient id %s email %s is %s", c.id, r.id, data.To_email, rs)
	return rs
}

func (c *campaign) get(id string) {
	models.Db.QueryRow("SELECT t1.`id`,t3.`email`,t3.`name`,t1.`subject`,t1.`body`,t2.`iface`,t2.`host`,t2.`stream`,t1.`send_unsubscribe`,t2.`resend_delay`,t2.`resend_count` FROM `campaign` t1 INNER JOIN `profile` t2 ON t2.`id`=t1.`profile_id` INNER JOIN `sender` t3 ON t3.`id`=t1.`from_id` WHERE t1.`id`=?", id).Scan(
		c.id,
		c.from_email,
		c.from_name,
		c.subject,
		c.body,
		c.iface,
		c.host,
		c.stream,
		c.send_unsubscribe,
		c.resend_delay,
		c.resend_count,
	)
}

func (c *campaign) getAttachments() {
	attachment, err := models.Db.Prepare("SELECT `path`, `file` FROM attachment WHERE campaign_id=?")
	checkErr(err)
	defer attachment.Close()

	attach, err := attachment.Query(c.id)
	checkErr(err)
	defer attach.Close()

	c.attachments = nil
	var location string
	var name string
	for attach.Next() {
		err = attach.Scan(&location, &name)
		checkErr(err)
		c.attachments = append(c.attachments, mailer.Attachment{Location: location, Name: name})
	}
}

func (c campaign) send() {
	count := 0
	stream := 0
	next := make(chan bool)

	c.getNullRecipients()
	camplog.Printf("Start campaign %s. Count recipients %d", c.id, len(c.recipients))

	for _, r := range c.recipients {
		count += 1
		if r.unsubscribe(c.id) == false  || c.send_unsubscribe {
			models.Db.Exec("UPDATE recipient SET status='Sending', date=NOW() WHERE id=?", r.id)
			stream += 1
			if stream > c.stream {
				<-next
				stream -= 1
			}
			go func(d recipient) {
				rs := d.send(&c)
				models.Db.Exec("UPDATE recipient SET status=?, date=NOW() WHERE id=?", rs, d.id)
				next <- true
			}(r)
		} else {
			models.Db.Exec("UPDATE recipient SET status='Unsubscribe', date=NOW() WHERE id=?", r.id)
			camplog.Printf("Recipient id %s email %s is unsubscribed", r.id, r.to_email)
		}
	}

	for stream != 0 {
		<-next
		stream -= 1
	}
	close(next)

	camplog.Printf("Done campaign %s. Count %d", c.id, count)
}

func (c campaign) resendSoftBounce() {
	c.getSoftBounceRecipients()
	if c.resend_count != 0 {
		camplog.Printf("Start %d resend by campaign id %s ", len(c.recipients), c.id)
	}
	if len(c.recipients) == 0 {
		return
	}

	for n := 0; n < c.resend_count; n++ {
		time.Sleep(time.Duration(c.resend_delay) * time.Second)
		c.getSoftBounceRecipients()
		for _, r := range c.recipients {
			// если пользователь ни разу не отказался от подписки в этой группе
			var unsubscribeCount int
			models.Db.QueryRow("SELECT COUNT(*) FROM `unsubscribe` t1 INNER JOIN `campaign` t2 ON t1.group_id = t2.group_id WHERE t2.id = ? AND t1.email = ?", c.id, r.to_email).Scan(&unsubscribeCount)
			if unsubscribeCount == 0 || c.send_unsubscribe {
				models.Db.Exec("UPDATE recipient SET status='Sending', date=NOW() WHERE id=?", r.id)
				rs := r.send(&c)
				models.Db.Exec("UPDATE recipient SET status=?, date=NOW() WHERE id=?", rs, r.id)
			} else {
				models.Db.Exec("UPDATE recipient SET status='Unsubscribe', date=NOW() WHERE id=?", r.id)
				camplog.Printf("Recipient id %s email %s is unsubscribed", r.id, r.to_email)
			}
		}
	}
}

func checkErr(err error) {
	if err != nil {
		camplog.Println(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"syscall"

	"github.com/lomik/carbon-clickhouse/carbon"
	"github.com/lomik/carbon-clickhouse/helper/RowBinary"
	"github.com/lomik/zapwriter"
	"go.uber.org/zap"

	_ "net/http/pprof"
)

// Version of carbon-clickhouse
const Version = "0.7.0-alpha3"

func httpServe(addr string) (func(), error) {
	tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		return nil, err
	}

	listener, err := net.ListenTCP("tcp", tcpAddr)
	if err != nil {
		return nil, err
	}

	go http.Serve(listener, nil)
	return func() { listener.Close() }, nil
}

func main() {
	var err error

	/* CONFIG start */

	configFile := flag.String("config", "/etc/carbon-clickhouse/carbon-clickhouse.conf", "Filename of config")
	printDefaultConfig := flag.Bool("config-print-default", false, "Print default config")
	checkConfig := flag.Bool("check-config", false, "Check config and exit")
	printVersion := flag.Bool("version", false, "Print version")
	cat := flag.String("cat", "", "Print RowBinary file in TabSeparated format")
	bincat := flag.String("recover", "", "Read all good records from corrupted data file. Write binary data to stdout")

	flag.Parse()

	if *printVersion {
		fmt.Print(Version)
		return
	}

	if *cat != "" {
		reader, err := RowBinary.NewReader(*cat)
		if err != nil {
			log.Fatal(err)
		}

		for {
			metric, err := reader.ReadRecord()
			if err != nil {
				if err == io.EOF {
					return
				}
				log.Fatal(err)
			}

			fmt.Printf("%s\t%#v\t%d\t%s\t%d\n",
				string(metric),
				reader.Value(),
				reader.Timestamp(),
				reader.DaysString(),
				reader.Version(),
			)
		}
	}

	if *bincat != "" {
		reader, err := RowBinary.NewReader(*bincat)
		if err != nil {
			log.Fatal(err)
		}

		io.Copy(os.Stdout, reader)
		return
	}

	if *printDefaultConfig {
		if err = carbon.PrintDefaultConfig(); err != nil {
			log.Fatal(err)
		}
		return
	}

	app := carbon.New(*configFile)

	if err = app.ParseConfig(); err != nil {
		log.Fatal(err)
	}

	// config parsed successfully. Exit in check-only mode
	if *checkConfig {
		return
	}

	cfg := app.Config

	if err = zapwriter.ApplyConfig(cfg.Logging); err != nil {
		log.Fatal(err)
	}

	mainLogger := zapwriter.Logger("main")

	/* CONFIG end */

	// pprof
	if cfg.Pprof.Enabled {
		_, err = httpServe(cfg.Pprof.Listen)
		if err != nil {
			mainLogger.Fatal("pprof listen failed", zap.Error(err))
		}
	}

	if err = app.Start(); err != nil {
		mainLogger.Fatal("app start failed", zap.Error(err))
	} else {
		mainLogger.Info("app started")
	}

	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGHUP)

		for {
			s := <-c
			switch s {
			case syscall.SIGUSR1:
				mainLogger.Info("SIGUSR1 received. Clear tree cache")
				app.Reset()
			case syscall.SIGUSR2:
				mainLogger.Info("SIGUSR2 received. Ignoring")
			case syscall.SIGHUP:
				mainLogger.Info("SIGHUP received. Ignoring")
			}
		}
	}()

	app.Loop()

	mainLogger.Info("app stopped")
}
<commit_msg>v0.7.0<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"syscall"

	"github.com/lomik/carbon-clickhouse/carbon"
	"github.com/lomik/carbon-clickhouse/helper/RowBinary"
	"github.com/lomik/zapwriter"
	"go.uber.org/zap"

	_ "net/http/pprof"
)

// Version of carbon-clickhouse
const Version = "0.7.0"

func httpServe(addr string) (func(), error) {
	tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		return nil, err
	}

	listener, err := net.ListenTCP("tcp", tcpAddr)
	if err != nil {
		return nil, err
	}

	go http.Serve(listener, nil)
	return func() { listener.Close() }, nil
}

func main() {
	var err error

	/* CONFIG start */

	configFile := flag.String("config", "/etc/carbon-clickhouse/carbon-clickhouse.conf", "Filename of config")
	printDefaultConfig := flag.Bool("config-print-default", false, "Print default config")
	checkConfig := flag.Bool("check-config", false, "Check config and exit")
	printVersion := flag.Bool("version", false, "Print version")
	cat := flag.String("cat", "", "Print RowBinary file in TabSeparated format")
	bincat := flag.String("recover", "", "Read all good records from corrupted data file. Write binary data to stdout")

	flag.Parse()

	if *printVersion {
		fmt.Print(Version)
		return
	}

	if *cat != "" {
		reader, err := RowBinary.NewReader(*cat)
		if err != nil {
			log.Fatal(err)
		}

		for {
			metric, err := reader.ReadRecord()
			if err != nil {
				if err == io.EOF {
					return
				}
				log.Fatal(err)
			}

			fmt.Printf("%s\t%#v\t%d\t%s\t%d\n",
				string(metric),
				reader.Value(),
				reader.Timestamp(),
				reader.DaysString(),
				reader.Version(),
			)
		}
	}

	if *bincat != "" {
		reader, err := RowBinary.NewReader(*bincat)
		if err != nil {
			log.Fatal(err)
		}

		io.Copy(os.Stdout, reader)
		return
	}

	if *printDefaultConfig {
		if err = carbon.PrintDefaultConfig(); err != nil {
			log.Fatal(err)
		}
		return
	}

	app := carbon.New(*configFile)

	if err = app.ParseConfig(); err != nil {
		log.Fatal(err)
	}

	// config parsed successfully. Exit in check-only mode
	if *checkConfig {
		return
	}

	cfg := app.Config

	if err = zapwriter.ApplyConfig(cfg.Logging); err != nil {
		log.Fatal(err)
	}

	mainLogger := zapwriter.Logger("main")

	/* CONFIG end */

	// pprof
	if cfg.Pprof.Enabled {
		_, err = httpServe(cfg.Pprof.Listen)
		if err != nil {
			mainLogger.Fatal("pprof listen failed", zap.Error(err))
		}
	}

	if err = app.Start(); err != nil {
		mainLogger.Fatal("app start failed", zap.Error(err))
	} else {
		mainLogger.Info("app started")
	}

	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGHUP)

		for {
			s := <-c
			switch s {
			case syscall.SIGUSR1:
				mainLogger.Info("SIGUSR1 received. Clear tree cache")
				app.Reset()
			case syscall.SIGUSR2:
				mainLogger.Info("SIGUSR2 received. Ignoring")
			case syscall.SIGHUP:
				mainLogger.Info("SIGHUP received. Ignoring")
			}
		}
	}()

	app.Loop()

	mainLogger.Info("app stopped")
}
<|endoftext|>


<commit_before>package generator_test

import (
	"reflect"
	"sync"
	"testing"

	"github.com/julien-noblet/download-geofabrik/config"
	"github.com/julien-noblet/download-geofabrik/element"
	"github.com/julien-noblet/download-geofabrik/formats"
	"github.com/julien-noblet/download-geofabrik/generator"
	yaml "gopkg.in/yaml.v2"
)

var sampleAfricaElementPtr = element.Element{
	ID:   "africa",
	Name: "Africa",
	Formats: []string{
		formats.FormatOsmPbf,
		"osm.pbf.md5",
		formats.FormatOsmBz2,
		"osm.bz2.md5",
		formats.FormatOshPbf,
		"osh.pbf.md5",
		formats.FormatPoly,
		formats.FormatKml,
		formats.FormatState,
	},
}
var sampleGeorgiaUsElementPtr = element.Element{
	ID:   "georgia-us",
	File: "georgia",
	Name: "Georgia (US State)",
	Formats: []string{
		formats.FormatOsmPbf,
		"osm.pbf.md5",
		formats.FormatShpZip,
		formats.FormatOsmBz2,
		"osm.bz2.md5",
		formats.FormatOshPbf,
		"osh.pbf.md5",
		formats.FormatPoly,
		formats.FormatKml,
		formats.FormatState,
	},
	Parent: "us",
}
var sampleUsElementPtr = element.Element{
	ID:     "us",
	Meta:   true,
	Name:   "United States of America",
	Parent: "north-america",
}
var sampleNorthAmericaElementPtr = element.Element{
	ID:   "north-america",
	Name: "North America",
	Formats: []string{
		formats.FormatOsmPbf,
		"osm.pbf.md5",
		formats.FormatOsmBz2,
		"osm.bz2.md5",
		formats.FormatOshPbf,
		"osh.pbf.md5",
		formats.FormatPoly,
		formats.FormatKml,
		formats.FormatState,
	},
}
var sampleElementValidPtr = map[string]element.Element{
	"africa":        sampleAfricaElementPtr,
	"georgia-us":    sampleGeorgiaUsElementPtr,
	"us":            sampleUsElementPtr,
	"north-america": sampleNorthAmericaElementPtr,
}
var sampleFormatValidPtr = map[string]formats.Format{
	// Blank
	"": {
		ID:       "",
		Loc:      "",
		BasePath: "",
	}, formats.FormatOsmPbf: {
		ID:  formats.FormatOsmPbf,
		Loc: ".osm.pbf",
		//BasePath: "/",
	}, formats.FormatState: {
		ID:       formats.FormatState,
		Loc:      "-updates/state.txt",
		BasePath: "../state/",
	}, formats.FormatPoly: {
		ID:      formats.FormatPoly,
		Loc:     ".poly",
		BaseURL: "http://my.new.url/folder",
	}, formats.FormatOsmBz2: {
		ID:       formats.FormatOsmBz2,
		Loc:      ".osm.bz2",
		BasePath: "../osmbz2/",
		BaseURL:  "http://my.new.url/folder",
	}, formats.FormatOsmGz: {
		ID:       formats.FormatOsmGz,
		Loc:      ".osm.gz",
		BasePath: "../osmgz/",
		BaseURL:  "http://my.new.url/folder",
	},
}

var SampleConfigValidPtr = config.Config{
	BaseURL:  "https://my.base.url",
	Formats:  sampleFormatValidPtr,
	Elements: sampleElementValidPtr,
}
var sampleFakeGeorgiaPtr = element.Element{
	ID:   "georgia-usf",
	File: "georgia-fake",
	Name: "Georgia (US State) - fake test",
	Formats: []string{
		formats.FormatOsmPbf,
		"osm.pbf.md5",
		formats.FormatShpZip,
		formats.FormatOsmBz2,
		"osm.bz2.md5",
		formats.FormatOshPbf,
		"osh.pbf.md5",
		formats.FormatPoly,
		formats.FormatKml,
		formats.FormatState,
	},
	Parent: "us", // keep good parent!
}
var sampleFakeGeorgia2Ptr = element.Element{
	ID:   "georgia-us2",
	File: "georgia",
	Name: "Georgia (US State)",
	Formats: []string{
		formats.FormatOsmPbf,
		"osm.pbf.md5",
		formats.FormatShpZip,
		formats.FormatOsmBz2,
		"osm.bz2.md5",
		formats.FormatOshPbf,
		"osh.pbf.md5",
		formats.FormatPoly,
		formats.FormatKml,
		formats.FormatState,
	},
	Parent: "notus", // bad parent not exist!
}

func TestSlice_Generate(t *testing.T) {
	tests := []struct {
		name    string
		e       element.Slice
		want    []byte
		wantErr bool
	}{
		// TODO: Add test cases.
		{
			name:    "Marshaling OK, no error",
			e:       sampleElementValidPtr,
			want:    []byte{},
			wantErr: false,
		},
	}
	for _, tt := range tests {
		tt := tt
		myConfig := &SampleConfigValidPtr
		myConfig.Elements = map[string]element.Element{} // void Elements
		myConfig.Elements = tt.e
		tt.want, _ = yaml.Marshal(*myConfig)
		t.Run(tt.name, func(t *testing.T) {
			got, err := myConfig.Generate()
			if err != nil != tt.wantErr {
				t.Errorf("Slice.Generate() error = %v, wantErr %v", err, tt.wantErr)
				return
			}
			if !reflect.DeepEqual(got, tt.want) {
				t.Errorf("Slice.Generate() = %v, want %v", got, tt.want)
			}
		})
	}
}

func TestGenerate(t *testing.T) {
	type args struct {
		configfile string
	}

	tests := []struct {
		name string
		args args
	}{
		// TODO: Add test cases.
	}

	for _, tt := range tests {
		tt := tt
		t.Run(tt.name, func(t *testing.T) {
			generator.Generate(tt.args.configfile)
		})
	}
}

func TestConfig_mergeElement(t *testing.T) {
	type args struct {
		element *element.Element
	}

	myFakeGeorgia := sampleFakeGeorgia2Ptr
	myFakeGeorgia.ID = "georgia-us"
	tests := []struct {
		name    string
		args    args
		wantErr bool
	}{
		// TODO: Add test cases.
		{
			name: "adding a new element",
			args: args{
				element: &sampleFakeGeorgiaPtr,
			},
			wantErr: false,
		},
		{
			name: "adding a same element",
			args: args{
				element: &sampleAfricaElementPtr,
			},
			wantErr: false,
		},
		{
			name: "adding a meta element",
			args: args{
				element: &sampleUsElementPtr,
			},
			wantErr: false,
		},
		{
			name: "adding a format to a meta element",
			args: args{
				element: &element.Element{ID: "us", Parent: "north-america", Formats: []string{formats.FormatOsmPbf}},
			},
			wantErr: false,
		},
		{
			name: "adding a same element parent differ",
			args: args{
				element: &myFakeGeorgia,
			},
			wantErr: true,
		},
	}

	for _, tt := range tests {
		tt := tt
		t.Run(tt.name, func(t *testing.T) {
			c := config.Config{
				Elements:      sampleElementValidPtr,
				ElementsMutex: &sync.RWMutex{},
			}
			if err := c.MergeElement(tt.args.element); err != nil != tt.wantErr {
				t.Errorf("Ext.mergeElement() error = %v, wantErr %v", err, tt.wantErr)
			}
		})
	}
}
<commit_msg>test: removing duplicate test<commit_after>package generator_test

import (
	"reflect"
	"testing"

	"github.com/julien-noblet/download-geofabrik/config"
	"github.com/julien-noblet/download-geofabrik/element"
	"github.com/julien-noblet/download-geofabrik/formats"
	"github.com/julien-noblet/download-geofabrik/generator"
	yaml "gopkg.in/yaml.v2"
)

var sampleAfricaElementPtr = element.Element{
	ID:   "africa",
	Name: "Africa",
	Formats: []string{
		formats.FormatOsmPbf,
		"osm.pbf.md5",
		formats.FormatOsmBz2,
		"osm.bz2.md5",
		formats.FormatOshPbf,
		"osh.pbf.md5",
		formats.FormatPoly,
		formats.FormatKml,
		formats.FormatState,
	},
}
var sampleGeorgiaUsElementPtr = element.Element{
	ID:   "georgia-us",
	File: "georgia",
	Name: "Georgia (US State)",
	Formats: []string{
		formats.FormatOsmPbf,
		"osm.pbf.md5",
		formats.FormatShpZip,
		formats.FormatOsmBz2,
		"osm.bz2.md5",
		formats.FormatOshPbf,
		"osh.pbf.md5",
		formats.FormatPoly,
		formats.FormatKml,
		formats.FormatState,
	},
	Parent: "us",
}
var sampleUsElementPtr = element.Element{
	ID:     "us",
	Meta:   true,
	Name:   "United States of America",
	Parent: "north-america",
}
var sampleNorthAmericaElementPtr = element.Element{
	ID:   "north-america",
	Name: "North America",
	Formats: []string{
		formats.FormatOsmPbf,
		"osm.pbf.md5",
		formats.FormatOsmBz2,
		"osm.bz2.md5",
		formats.FormatOshPbf,
		"osh.pbf.md5",
		formats.FormatPoly,
		formats.FormatKml,
		formats.FormatState,
	},
}
var sampleElementValidPtr = map[string]element.Element{
	"africa":        sampleAfricaElementPtr,
	"georgia-us":    sampleGeorgiaUsElementPtr,
	"us":            sampleUsElementPtr,
	"north-america": sampleNorthAmericaElementPtr,
}
var sampleFormatValidPtr = map[string]formats.Format{
	// Blank
	"": {
		ID:       "",
		Loc:      "",
		BasePath: "",
	}, formats.FormatOsmPbf: {
		ID:  formats.FormatOsmPbf,
		Loc: ".osm.pbf",
		//BasePath: "/",
	}, formats.FormatState: {
		ID:       formats.FormatState,
		Loc:      "-updates/state.txt",
		BasePath: "../state/",
	}, formats.FormatPoly: {
		ID:      formats.FormatPoly,
		Loc:     ".poly",
		BaseURL: "http://my.new.url/folder",
	}, formats.FormatOsmBz2: {
		ID:       formats.FormatOsmBz2,
		Loc:      ".osm.bz2",
		BasePath: "../osmbz2/",
		BaseURL:  "http://my.new.url/folder",
	}, formats.FormatOsmGz: {
		ID:       formats.FormatOsmGz,
		Loc:      ".osm.gz",
		BasePath: "../osmgz/",
		BaseURL:  "http://my.new.url/folder",
	},
}

var SampleConfigValidPtr = config.Config{
	BaseURL:  "https://my.base.url",
	Formats:  sampleFormatValidPtr,
	Elements: sampleElementValidPtr,
}

func TestSlice_Generate(t *testing.T) {
	tests := []struct {
		name    string
		e       element.Slice
		want    []byte
		wantErr bool
	}{
		// TODO: Add test cases.
		{
			name:    "Marshaling OK, no error",
			e:       sampleElementValidPtr,
			want:    []byte{},
			wantErr: false,
		},
	}
	for _, tt := range tests {
		tt := tt
		myConfig := &SampleConfigValidPtr
		myConfig.Elements = map[string]element.Element{} // void Elements
		myConfig.Elements = tt.e
		tt.want, _ = yaml.Marshal(*myConfig)
		t.Run(tt.name, func(t *testing.T) {
			got, err := myConfig.Generate()
			if err != nil != tt.wantErr {
				t.Errorf("Slice.Generate() error = %v, wantErr %v", err, tt.wantErr)
				return
			}
			if !reflect.DeepEqual(got, tt.want) {
				t.Errorf("Slice.Generate() = %v, want %v", got, tt.want)
			}
		})
	}
}

func TestGenerate(t *testing.T) {
	type args struct {
		configfile string
	}

	tests := []struct {
		name string
		args args
	}{
		// TODO: Add test cases.
	}

	for _, tt := range tests {
		tt := tt
		t.Run(tt.name, func(t *testing.T) {
			generator.Generate(tt.args.configfile)
		})
	}
}
<|endoftext|>