<commit_before>package main

import (
	"fmt"
	"os/exec"
	"strings"
	"testing"
)

// See issue docker/docker#8141
func TestPullImageWithAliases(t *testing.T) {
	defer setupRegistry(t)()

	repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL)
	defer deleteImages(repoName)

	repos := []string{}
	for _, tag := range []string{"recent", "fresh"} {
		repos = append(repos, fmt.Sprintf("%v:%v", repoName, tag))
	}

	// Tag and push the same image multiple times.
	for _, repo := range repos {
		if out, _, err := runCommandWithOutput(exec.Command(dockerBinary, "tag", "busybox", repo)); err != nil {
			t.Fatalf("Failed to tag image %v: error %v, output %q", repos, err, out)
		}
		if out, err := exec.Command(dockerBinary, "push", repo).CombinedOutput(); err != nil {
			t.Fatalf("Failed to push image %v: error %v, output %q", err, string(out))
		}
	}

	// Clear local images store.
	args := append([]string{"rmi"}, repos...)
	if out, err := exec.Command(dockerBinary, args...).CombinedOutput(); err != nil {
		t.Fatalf("Failed to clean images: error %v, output %q", err, string(out))
	}

	// Pull a single tag and verify it doesn't bring down all aliases.
	pullCmd := exec.Command(dockerBinary, "pull", repos[0])
	if out, _, err := runCommandWithOutput(pullCmd); err != nil {
		t.Fatalf("Failed to pull %v: error %v, output %q", repoName, err, out)
	}
	defer deleteImages(repos[0])
	if err := exec.Command(dockerBinary, "inspect", repos[0]).Run(); err != nil {
		t.Fatalf("Image %v was not pulled down", repos[0])
	}
	for _, repo := range repos[1:] {
		if err := exec.Command(dockerBinary, "inspect", repo).Run(); err == nil {
			t.Fatalf("Image %v shouldn't have been pulled down", repo)
		}
	}

	logDone("pull - image with aliases")
}

// pulling an image from the central registry should work
func TestPullImageFromCentralRegistry(t *testing.T) {
	pullCmd := exec.Command(dockerBinary, "pull", "hello-world")
	if out, _, err := runCommandWithOutput(pullCmd); err != nil {
		t.Fatalf("pulling the hello-world image from the registry has failed: %s, %v", out, err)
	}
	logDone("pull - pull hello-world")
}

// pulling a non-existing image from the central registry should return a non-zero exit code
func TestPullNonExistingImage(t *testing.T) {
	pullCmd := exec.Command(dockerBinary, "pull", "fooblahblah1234")
	if out, _, err := runCommandWithOutput(pullCmd); err == nil {
		t.Fatalf("expected non-zero exit status when pulling non-existing image: %s", out)
	}
	logDone("pull - pull fooblahblah1234 (non-existing image)")
}

// pulling an image from the central registry using official names should work
// ensure all pulls result in the same image
func TestPullImageOfficialNames(t *testing.T) {
	names := []string{
		"docker.io/hello-world",
		"index.docker.io/hello-world",
		"library/hello-world",
		"docker.io/library/hello-world",
		"index.docker.io/library/hello-world",
	}
	for _, name := range names {
		pullCmd := exec.Command(dockerBinary, "pull", name)
		out, exitCode, err := runCommandWithOutput(pullCmd)
		if err != nil || exitCode != 0 {
			t.Errorf("pulling the '%s' image from the registry has failed: %s", name, err)
			continue
		}

		// ensure we don't have multiple image names.
		imagesCmd := exec.Command(dockerBinary, "images")
		out, _, err = runCommandWithOutput(imagesCmd)
		if err != nil {
			t.Errorf("listing images failed with errors: %v", err)
		} else if strings.Contains(out, name) {
			t.Errorf("images should not have listed '%s'", name)
		}
	}
	logDone("pull - pull official names")
}
<commit_msg>Add test for pull verified<commit_after>package main

import (
	"fmt"
	"os/exec"
	"strings"
	"testing"
)

// See issue docker/docker#8141
func TestPullImageWithAliases(t *testing.T) {
	defer setupRegistry(t)()

	repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL)
	defer deleteImages(repoName)

	repos := []string{}
	for _, tag := range []string{"recent", "fresh"} {
		repos = append(repos, fmt.Sprintf("%v:%v", repoName, tag))
	}

	// Tag and push the same image multiple times.
	for _, repo := range repos {
		if out, _, err := runCommandWithOutput(exec.Command(dockerBinary, "tag", "busybox", repo)); err != nil {
			t.Fatalf("Failed to tag image %v: error %v, output %q", repos, err, out)
		}
		if out, err := exec.Command(dockerBinary, "push", repo).CombinedOutput(); err != nil {
			t.Fatalf("Failed to push image %v: error %v, output %q", err, string(out))
		}
	}

	// Clear local images store.
	args := append([]string{"rmi"}, repos...)
	if out, err := exec.Command(dockerBinary, args...).CombinedOutput(); err != nil {
		t.Fatalf("Failed to clean images: error %v, output %q", err, string(out))
	}

	// Pull a single tag and verify it doesn't bring down all aliases.
	pullCmd := exec.Command(dockerBinary, "pull", repos[0])
	if out, _, err := runCommandWithOutput(pullCmd); err != nil {
		t.Fatalf("Failed to pull %v: error %v, output %q", repoName, err, out)
	}
	defer deleteImages(repos[0])
	if err := exec.Command(dockerBinary, "inspect", repos[0]).Run(); err != nil {
		t.Fatalf("Image %v was not pulled down", repos[0])
	}
	for _, repo := range repos[1:] {
		if err := exec.Command(dockerBinary, "inspect", repo).Run(); err == nil {
			t.Fatalf("Image %v shouldn't have been pulled down", repo)
		}
	}

	logDone("pull - image with aliases")
}

// pulling busybox should show verified message
func TestPullVerified(t *testing.T) {
	defer setupRegistry(t)()

	repo := fmt.Sprintf("%v/dockercli/busybox:verified", privateRegistryURL)
	defer deleteImages(repo)

	// tag the image
	if out, _, err := runCommandWithOutput(exec.Command(dockerBinary, "tag", "busybox", repo)); err != nil {
		t.Fatalf("Failed to tag image verifiedTest: error %v, output %q", err, out)
	}

	// push it
	if out, err := exec.Command(dockerBinary, "push", repo).CombinedOutput(); err != nil {
		t.Fatalf("Failed to push image %v: error %v, output %q", err, string(out))
	}

	// remove it locally
	if out, err := exec.Command(dockerBinary, "rmi", repo).CombinedOutput(); err != nil {
		t.Fatalf("Failed to clean images: error %v, output %q", err, string(out))
	}

	// pull it
	expected := "The image you are pulling has been verified"
	pullCmd := exec.Command(dockerBinary, "pull", repo)
	if out, _, err := runCommandWithOutput(pullCmd); err != nil || !strings.Contains(out, expected) {
		t.Fatalf("pulling a verified image failed. expected: %s\ngot: %s, %v", expected, out, err)
	}

	// pull it again
	pullCmd = exec.Command(dockerBinary, "pull", repo)
	if out, _, err := runCommandWithOutput(pullCmd); err != nil || !strings.Contains(out, expected) {
		t.Fatalf("pulling a verified image failed. expected: %s\ngot: %s, %v", expected, out, err)
	}

	logDone("pull - pull verified")
}

// pulling an image from the central registry should work
func TestPullImageFromCentralRegistry(t *testing.T) {
	pullCmd := exec.Command(dockerBinary, "pull", "hello-world")
	if out, _, err := runCommandWithOutput(pullCmd); err != nil {
		t.Fatalf("pulling the hello-world image from the registry has failed: %s, %v", out, err)
	}
	logDone("pull - pull hello-world")
}

// pulling a non-existing image from the central registry should return a non-zero exit code
func TestPullNonExistingImage(t *testing.T) {
	pullCmd := exec.Command(dockerBinary, "pull", "fooblahblah1234")
	if out, _, err := runCommandWithOutput(pullCmd); err == nil {
		t.Fatalf("expected non-zero exit status when pulling non-existing image: %s", out)
	}
	logDone("pull - pull fooblahblah1234 (non-existing image)")
}

// pulling an image from the central registry using official names should work
// ensure all pulls result in the same image
func TestPullImageOfficialNames(t *testing.T) {
	names := []string{
		"docker.io/hello-world",
		"index.docker.io/hello-world",
		"library/hello-world",
		"docker.io/library/hello-world",
		"index.docker.io/library/hello-world",
	}
	for _, name := range names {
		pullCmd := exec.Command(dockerBinary, "pull", name)
		out, exitCode, err := runCommandWithOutput(pullCmd)
		if err != nil || exitCode != 0 {
			t.Errorf("pulling the '%s' image from the registry has failed: %s", name, err)
			continue
		}

		// ensure we don't have multiple image names.
		imagesCmd := exec.Command(dockerBinary, "images")
		out, _, err = runCommandWithOutput(imagesCmd)
		if err != nil {
			t.Errorf("listing images failed with errors: %v", err)
		} else if strings.Contains(out, name) {
			t.Errorf("images should not have listed '%s'", name)
		}
	}
	logDone("pull - pull official names")
}
<|endoftext|>


<commit_before>package bark

import (
	"context"
	"net/http"

	"github.com/NYTimes/gizmo/auth"
	"github.com/NYTimes/gizmo/auth/gcp"
	"github.com/NYTimes/gizmo/server/kit"
	"github.com/go-kit/kit/endpoint"
	httptransport "github.com/go-kit/kit/transport/http"
	"github.com/kelseyhightower/envconfig"
	"github.com/pkg/errors"
	"google.golang.org/grpc"
)

type econfig struct {
	SlackKeys []string `envconfig:"SLACK_KEYS"`

	TwitterTokens  []string `envconfig:"TWITTER_TOKENS"`
	TwitterSecrets []string `envconfig:"TWITTER_SECRETS"`

	Auth gcp.IdentityConfig `envconfig:"AUTH"`
}

type service struct {
	alertsOut []AlertBarker

	eventsOut []EventBarker

	verifier *auth.Verifier
}

func NewService() (kit.Service, error) {
	ctx := context.Background()

	var cfg econfig
	envconfig.MustProcess("", &cfg)

	v, err := gcp.NewDefaultIdentityVerifier(ctx, cfg.Auth)
	if err != nil {
		return nil, errors.Wrap(err, "unable to init ID verifier")
	}

	var (
		alerts []AlertBarker
		events []EventBarker
	)

	for _, key := range cfg.SlackKeys {
		alerts = append(alerts, NewSlackAlertBarker(
			SlackConfig{Key: key, BotName: "Newshound Alerts"}))
		events = append(events, NewSlackEventBarker(
			SlackConfig{Key: key, BotName: "Newshound Alerts"}))
	}

	if len(cfg.TwitterSecrets) != len(cfg.TwitterTokens) {
		return nil, errors.Wrap(err, "invalid twitter config. token counts mismatch")
	}

	for i, token := range cfg.TwitterTokens {
		secret := cfg.TwitterSecrets[i]
		alerts = append(alerts, NewTwitterAlertBarker(token, secret))
		events = append(events, NewTwitterEventBarker(token, secret))
	}

	return &service{
		verifier:  v,
		alertsOut: alerts,
		eventsOut: events,
	}, nil
}

func (s *service) Middleware(e endpoint.Endpoint) endpoint.Endpoint {
	return e
}

func (s *service) HTTPMiddleware(h http.Handler) http.Handler {
	if s.verifier == nil {
		return h
	}
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		ok, err := s.verifier.VerifyRequest(r)
		if err != nil || !ok {
			code := http.StatusForbidden
			http.Error(w, http.StatusText(code), code)
			return
		}

		h.ServeHTTP(w, r)
	})
}

func (s *service) HTTPOptions() []httptransport.ServerOption {
	return nil
}

func (s *service) HTTPRouterOptions() []kit.RouterOption {
	return nil
}

func (s *service) HTTPEndpoints() map[string]map[string]kit.HTTPEndpoint {
	return map[string]map[string]kit.HTTPEndpoint{
		"/svc/newshound/v1/bark/alert": {
			"POST": {
				Decoder:  decodeAlert,
				Endpoint: s.postAlert,
			},
		},
		"/svc/newshound/v1/bark/event": {
			"POST": {
				Decoder:  decodeEvent,
				Endpoint: s.postEvent,
			},
		},
	}
}

func (s *service) RPCMiddleware() grpc.UnaryServerInterceptor {
	return nil
}

func (s *service) RPCServiceDesc() *grpc.ServiceDesc {
	return nil
}

func (s *service) RPCOptions() []grpc.ServerOption {
	return nil
}

type psmessage struct {
	Message psdata `json:"message"`
}
type psdata struct {
	Data []byte `json:"data"`
}
<commit_msg>debugging barkd<commit_after>package bark

import (
	"context"
	"log"
	"net/http"

	"github.com/NYTimes/gizmo/auth"
	"github.com/NYTimes/gizmo/auth/gcp"
	"github.com/NYTimes/gizmo/server/kit"
	"github.com/go-kit/kit/endpoint"
	httptransport "github.com/go-kit/kit/transport/http"
	"github.com/kelseyhightower/envconfig"
	"github.com/pkg/errors"
	"google.golang.org/grpc"
)

type econfig struct {
	SlackKeys []string `envconfig:"SLACK_KEYS"`

	TwitterTokens  []string `envconfig:"TWITTER_TOKENS"`
	TwitterSecrets []string `envconfig:"TWITTER_SECRETS"`

	Auth gcp.IdentityConfig `envconfig:"AUTH"`
}

type service struct {
	alertsOut []AlertBarker

	eventsOut []EventBarker

	verifier *auth.Verifier
}

func NewService() (kit.Service, error) {
	ctx := context.Background()

	var cfg econfig
	envconfig.MustProcess("", &cfg)

	v, err := gcp.NewDefaultIdentityVerifier(ctx, cfg.Auth)
	if err != nil {
		return nil, errors.Wrap(err, "unable to init ID verifier")
	}

	var (
		alerts             []AlertBarker
		events             []EventBarker
		slackers, twitters int
	)

	for _, key := range cfg.SlackKeys {
		alerts = append(alerts, NewSlackAlertBarker(
			SlackConfig{Key: key, BotName: "Newshound Alerts"}))
		events = append(events, NewSlackEventBarker(
			SlackConfig{Key: key, BotName: "Newshound Alerts"}))
		slackers++
	}

	if len(cfg.TwitterSecrets) != len(cfg.TwitterTokens) {
		return nil, errors.Wrap(err, "invalid twitter config. token counts mismatch")
	}

	for i, token := range cfg.TwitterTokens {
		secret := cfg.TwitterSecrets[i]
		alerts = append(alerts, NewTwitterAlertBarker(token, secret))
		events = append(events, NewTwitterEventBarker(token, secret))
		twitters++
	}

	log.Printf("starting with %d slack accounts and %d twitter accounts",
		slackers, twitters)

	return &service{
		verifier:  v,
		alertsOut: alerts,
		eventsOut: events,
	}, nil
}

func (s *service) Middleware(e endpoint.Endpoint) endpoint.Endpoint {
	return e
}

func (s *service) HTTPMiddleware(h http.Handler) http.Handler {
	if s.verifier == nil {
		return h
	}
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		ok, err := s.verifier.VerifyRequest(r)
		if err != nil || !ok {
			code := http.StatusForbidden
			http.Error(w, http.StatusText(code), code)
			return
		}

		h.ServeHTTP(w, r)
	})
}

func (s *service) HTTPOptions() []httptransport.ServerOption {
	return nil
}

func (s *service) HTTPRouterOptions() []kit.RouterOption {
	return nil
}

func (s *service) HTTPEndpoints() map[string]map[string]kit.HTTPEndpoint {
	return map[string]map[string]kit.HTTPEndpoint{
		"/svc/newshound/v1/bark/alert": {
			"POST": {
				Decoder:  decodeAlert,
				Endpoint: s.postAlert,
			},
		},
		"/svc/newshound/v1/bark/event": {
			"POST": {
				Decoder:  decodeEvent,
				Endpoint: s.postEvent,
			},
		},
	}
}

func (s *service) RPCMiddleware() grpc.UnaryServerInterceptor {
	return nil
}

func (s *service) RPCServiceDesc() *grpc.ServiceDesc {
	return nil
}

func (s *service) RPCOptions() []grpc.ServerOption {
	return nil
}

type psmessage struct {
	Message psdata `json:"message"`
}
type psdata struct {
	Data []byte `json:"data"`
}
<|endoftext|>


<commit_before>package benchmarks

import (
	"runtime"
	"time"
)

type benchmark struct {
	repeats int
	runs    int
	signal  chan interface{}
	f       func(int)
	result  float64
}

func (b *benchmark) launch() {
	defer func() {
		b.signal <- b
	}()

	first := true
	var minDuration time.Duration
	for i := 0; i < b.runs; i++ {
		runtime.GC()
		start := time.Now()
		b.f(b.repeats)
		duration := time.Now().Sub(start)
		if first {
			minDuration = duration
		} else {
			if minDuration > duration {
				minDuration = duration
			}
		}
	}
	b.result = minDuration.Seconds()
}

func runBenchmark(runs, repeats int, f func(int)) float64 {
	b := benchmark{
		repeats: repeats,
		runs:    runs,
		signal:  make(chan interface{}),
		f:       f,
	}

	go b.launch()
	<-b.signal
	return b.result
}
<commit_msg>fix warnings<commit_after>package benchmarks

import (
	"runtime"
	"time"
)

type benchmark struct {
	signal chan interface{}
	f      func(int)
	result float64
}

func (b *benchmark) launch(runs, repeats int) {
	defer func() {
		b.signal <- b
	}()

	first := true
	var minDuration time.Duration
	for i := 0; i < runs; i++ {
		runtime.GC()
		start := time.Now()
		b.f(repeats)
		duration := time.Now().Sub(start)
		if first {
			minDuration = duration
		} else {
			if minDuration > duration {
				minDuration = duration
			}
		}
	}
	b.result = minDuration.Seconds()
}

func runBenchmark(runs, repeats int, f func(int)) float64 {
	b := benchmark{
		signal: make(chan interface{}),
		f:      f,
	}

	go b.launch(runs, repeats)
	<-b.signal
	return b.result
}
<|endoftext|>


<commit_before>package main

import "fmt"

type Number interface {
	int64 | float64
}
// SumNumbers sums the values of map m. Its supports both integers
// and floats as map values.
func SumNumbers[K comparable, V Number](m map[K]V) V {
	var s V
	for _, v := range m {
		s += v
	}
	return s
}
// SumInts adds together the values of m.
func SumInts(m map[string]int64) int64 {
	var s int64
	for _, v := range m {
		s += v
	}
	return s
}

// SumFloats adds together the values of m.
func SumFloats(m map[string]float64) float64 {
	var s float64
	for _, v := range m {
		s += v
	}
	return s
}

func main() {
	// Initialize a map for the integer values
	ints := map[string]int64{
		"first":  34,
		"second": 12,
	}

	// Initialize a map for the float values
	floats := map[string]float64{
		"first":  35.98,
		"second": 26.99,
	}

	fmt.Printf("Non-Generic Sums: %v and %v\n",
		SumInts(ints),
		SumFloats(floats))

	fmt.Printf("Generic Sums: %v and %v\n",
		SumIntsOrFloats[string, int64](ints),
		SumIntsOrFloats[string, float64](floats))

	fmt.Printf("Generic Sums, type parameters inferred: %v and %v\n",
		SumIntsOrFloats(ints),
		SumIntsOrFloats(floats))

	fmt.Printf("Generic Sums with Constraint: %v and %v\n",
		SumNumbers(ints),
		SumNumbers(floats))

}

// SumIntsOrFloats sums the values of map m. It supports both int64 and float64
// as types for map values.
func SumIntsOrFloats[K comparable, V int64 | float64](m map[K]V) V {
	var s V
	for _, v := range m {
		s += v
	}
	return s
}
<commit_msg>Modified Generics example<commit_after>package main

import (
	"fmt"
	"math"
	"math/rand"
)

type Number interface {
	~int | ~int32 | ~int64 | ~float64
}
// SumNumbers sums the values of map m. Its supports both integers
// and floats as map values.
func SumNumbers[K comparable, V Number](m map[K]V) V {
	var s V
	for _, v := range m {
		s += v
	}
	return s
}
// SumInts adds together the values of m.
func SumInts(m map[string]int64) int64 {
	var s int64
	for _, v := range m {
		s += v
	}
	return s
}

// SumFloats adds together the values of m.
func SumFloats(m map[string]float64) float64 {
	var s float64
	for _, v := range m {
		s += v
	}
	return s
}

func main() {
	// Initialize a map for the integer values
	ints := map[string]int64{
		"first":  34,
		"second": 12,
	}

	// Initialize a map for the float values
	floats := map[string]float64{
		"first":  35.98,
		"second": 26.99,
	}

	fmt.Printf("Non-Generic Sums: %v and %v\n",
		SumInts(ints),
		SumFloats(floats))

	fmt.Printf("Generic Sums: %v and %v\n",
		SumIntsOrFloats[string, int64](ints),
		SumIntsOrFloats[string, float64](floats))

	fmt.Printf("Generic Sums, type parameters inferred: %v and %v\n",
		SumIntsOrFloats(ints),
		SumIntsOrFloats(floats))

	fmt.Printf("Generic Sums with Constraint: %v and %v\n",
		SumNumbers(ints),
		SumNumbers(floats))

	fmt.Printf("Max with integer: %v\n",Max([]num{10, 15, 4, 25, 16, 18, 2}))
	fmt.Printf("Max with float: %v\n",Max([]float64{6.2, 4.1, 6.2, 9.6, 8.2, 1.5, 4.7}))
	fmt.Printf("Contains with string: %v\n", Contains([]string{"one", "two", "three"},"two"))
	fmt.Printf("Contains with int: %v\n", Contains([]int{100, 200, 300},50))

	randomNumbers:=make([]int, 5, 5)
	for i:=0;i<5; i++ {
		randomNumbers[i]=rand.Intn(25)
	}
   fibvalues:= MapSlice(randomNumbers,findFibonacci)
   fmt.Println(fibvalues)
   sqrs:= MapSlice(randomNumbers, func(num int) int {
	   return num * num
   })
	fmt.Println(sqrs)

}

type fibvalue struct {
	input, value int
}
func findFibonacci(num int) fibvalue {
	input := float64(num)
	// Fibonacci using Binet's formula
	Phi := (1 + math.Sqrt(5)) / 2
	phi := (1 - math.Sqrt(5)) / 2
	result := (math.Pow(Phi, input) - math.Pow(phi, input)) / math.Sqrt(5)
	return fibvalue {
		input: num,
		value: int(result),
	}
}
// SumIntsOrFloats sums the values of map m. It supports both int64 and float64
// as types for map values.
func SumIntsOrFloats[K comparable, V int64 | float64](m map[K]V) V {
	var s V
	for _, v := range m {
		s += v
	}
	return s
}

type num int
func Max[T Number](s []T) T {
	if len(s) == 0 {
		var zero T
		return zero
	}
	m := s[0]
	for _, v := range s {
		if m < v {
			m = v
		}
	}
	return m
}
func Contains[T comparable](elems []T, v T) bool {
	for _, s := range elems {
		if v == s {
			return true
		}
	}
	return false
}

func MapSlice[T any, M any](a []T, f func(T) M) []M {
	n := make([]M, len(a))
	for i, e := range a {
		n[i] = f(e)
	}
	return n
}<|endoftext|>


<commit_before>// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package deltas

import (
	"database/sql"
	"fmt"

	"github.com/lib/pq"
	"github.com/matrix-org/dendrite/internal/sqlutil"
	"github.com/matrix-org/dendrite/roomserver/types"
	"github.com/matrix-org/util"
	"github.com/sirupsen/logrus"
)

type stateSnapshotData struct {
	StateSnapshotNID types.StateSnapshotNID
	RoomNID          types.RoomNID
}

type stateBlockData struct {
	stateSnapshotData
	StateBlockNID types.StateBlockNID
	EventNIDs     types.EventNIDs
}

func LoadStateBlocksRefactor(m *sqlutil.Migrations) {
	m.AddMigration(UpStateBlocksRefactor, DownStateBlocksRefactor)
}

// nolint:gocyclo
func UpStateBlocksRefactor(tx *sql.Tx) error {
	logrus.Warn("Performing state storage upgrade. Please wait, this may take some time!")
	defer logrus.Warn("State storage upgrade complete")

	var snapshotcount int
	var maxsnapshotid int
	var maxblockid int
	if err := tx.QueryRow(`SELECT COUNT(DISTINCT state_snapshot_nid) FROM roomserver_state_snapshots;`).Scan(&snapshotcount); err != nil {
		return fmt.Errorf("tx.QueryRow.Scan (count snapshots): %w", err)
	}
	if err := tx.QueryRow(`SELECT COALESCE(MAX(state_snapshot_nid),0) FROM roomserver_state_snapshots;`).Scan(&maxsnapshotid); err != nil {
		return fmt.Errorf("tx.QueryRow.Scan (count snapshots): %w", err)
	}
	if err := tx.QueryRow(`SELECT COALESCE(MAX(state_block_nid),0) FROM roomserver_state_block;`).Scan(&maxblockid); err != nil {
		return fmt.Errorf("tx.QueryRow.Scan (count snapshots): %w", err)
	}
	maxsnapshotid++
	maxblockid++

	if _, err := tx.Exec(`ALTER TABLE roomserver_state_block RENAME TO _roomserver_state_block;`); err != nil {
		return fmt.Errorf("tx.Exec: %w", err)
	}
	if _, err := tx.Exec(`ALTER TABLE roomserver_state_snapshots RENAME TO _roomserver_state_snapshots;`); err != nil {
		return fmt.Errorf("tx.Exec: %w", err)
	}
	// We create new sequences starting with the maximum state snapshot and block NIDs.
	// This means that all newly created snapshots and blocks by the migration will have
	// NIDs higher than these values, so that when we come to update the references to
	// these NIDs using UPDATE statements, we can guarantee we are only ever updating old
	// values and not accidentally overwriting new ones.
	if _, err := tx.Exec(fmt.Sprintf(`CREATE SEQUENCE roomserver_state_block_nid_sequence START WITH %d;`, maxblockid)); err != nil {
		return fmt.Errorf("tx.Exec: %w", err)
	}
	if _, err := tx.Exec(fmt.Sprintf(`CREATE SEQUENCE roomserver_state_snapshot_nid_sequence START WITH %d;`, maxsnapshotid)); err != nil {
		return fmt.Errorf("tx.Exec: %w", err)
	}
	_, err := tx.Exec(`
		CREATE TABLE IF NOT EXISTS roomserver_state_block (
			state_block_nid bigint PRIMARY KEY DEFAULT nextval('roomserver_state_block_nid_sequence'),
			state_block_hash BYTEA UNIQUE,
			event_nids bigint[] NOT NULL
		);
	`)
	if err != nil {
		return fmt.Errorf("tx.Exec (create blocks table): %w", err)
	}
	_, err = tx.Exec(`
		CREATE TABLE IF NOT EXISTS roomserver_state_snapshots (
			state_snapshot_nid bigint PRIMARY KEY DEFAULT nextval('roomserver_state_snapshot_nid_sequence'),
			state_snapshot_hash BYTEA UNIQUE,
			room_nid bigint NOT NULL,
			state_block_nids bigint[] NOT NULL
		);
	`)
	if err != nil {
		return fmt.Errorf("tx.Exec (create snapshots table): %w", err)
	}
	logrus.Warn("New tables created...")

	batchsize := 100
	for batchoffset := 0; batchoffset < snapshotcount; batchoffset += batchsize {
		var snapshotrows *sql.Rows
		snapshotrows, err = tx.Query(`
			SELECT
				state_snapshot_nid,
				room_nid,
				state_block_nid,
				ARRAY_AGG(event_nid) AS event_nids
			FROM (
				SELECT
					_roomserver_state_snapshots.state_snapshot_nid,
					_roomserver_state_snapshots.room_nid,
					_roomserver_state_block.state_block_nid,
					_roomserver_state_block.event_nid
				FROM
					_roomserver_state_snapshots
					JOIN _roomserver_state_block ON _roomserver_state_block.state_block_nid = ANY (_roomserver_state_snapshots.state_block_nids)
				WHERE
					_roomserver_state_snapshots.state_snapshot_nid = ANY (
						SELECT
							_roomserver_state_snapshots.state_snapshot_nid
						FROM
							_roomserver_state_snapshots
						ORDER BY _roomserver_state_snapshots.state_snapshot_nid ASC
						LIMIT $1 OFFSET $2
					)
			) AS _roomserver_state_block
			GROUP BY
				state_snapshot_nid,
				room_nid,
				state_block_nid;
		`, batchsize, batchoffset)
		if err != nil {
			return fmt.Errorf("tx.Query: %w", err)
		}

		logrus.Warnf("Rewriting snapshots %d-%d of %d...", batchoffset, batchoffset+batchsize, snapshotcount)
		var snapshots []stateBlockData

		for snapshotrows.Next() {
			var snapshot stateBlockData
			var eventsarray pq.Int64Array
			if err = snapshotrows.Scan(&snapshot.StateSnapshotNID, &snapshot.RoomNID, &snapshot.StateBlockNID, &eventsarray); err != nil {
				return fmt.Errorf("rows.Scan: %w", err)
			}
			for _, e := range eventsarray {
				snapshot.EventNIDs = append(snapshot.EventNIDs, types.EventNID(e))
			}
			snapshot.EventNIDs = snapshot.EventNIDs[:util.SortAndUnique(snapshot.EventNIDs)]
			snapshots = append(snapshots, snapshot)
		}

		if err = snapshotrows.Close(); err != nil {
			return fmt.Errorf("snapshots.Close: %w", err)
		}

		newsnapshots := map[stateSnapshotData]types.StateBlockNIDs{}

		for _, snapshot := range snapshots {
			var eventsarray pq.Int64Array
			for _, e := range snapshot.EventNIDs {
				eventsarray = append(eventsarray, int64(e))
			}

			var blocknid types.StateBlockNID
			err = tx.QueryRow(`
				INSERT INTO roomserver_state_block (state_block_hash, event_nids)
					VALUES ($1, $2)
					ON CONFLICT (state_block_hash) DO UPDATE SET event_nids=$2
					RETURNING state_block_nid
			`, snapshot.EventNIDs.Hash(), eventsarray).Scan(&blocknid)
			if err != nil {
				return fmt.Errorf("tx.QueryRow.Scan (insert new block with %d events): %w", len(eventsarray), err)
			}
			index := stateSnapshotData{snapshot.StateSnapshotNID, snapshot.RoomNID}
			newsnapshots[index] = append(newsnapshots[index], blocknid)
		}

		for snapshotdata, newblocks := range newsnapshots {
			var newblocksarray pq.Int64Array
			for _, b := range newblocks {
				newblocksarray = append(newblocksarray, int64(b))
			}

			var newNID types.StateSnapshotNID
			err = tx.QueryRow(`
				INSERT INTO roomserver_state_snapshots (state_snapshot_hash, room_nid, state_block_nids)
					VALUES ($1, $2, $3)
					ON CONFLICT (state_snapshot_hash) DO UPDATE SET room_nid=$2
					RETURNING state_snapshot_nid
			`, newblocks.Hash(), snapshotdata.RoomNID, newblocksarray).Scan(&newNID)
			if err != nil {
				return fmt.Errorf("tx.QueryRow.Scan (insert new snapshot): %w", err)
			}

			if _, err = tx.Exec(`UPDATE roomserver_events SET state_snapshot_nid=$1 WHERE state_snapshot_nid=$2 AND state_snapshot_nid<$3`, newNID, snapshotdata.StateSnapshotNID, maxsnapshotid); err != nil {
				return fmt.Errorf("tx.Exec (update events): %w", err)
			}

			if _, err = tx.Exec(`UPDATE roomserver_rooms SET state_snapshot_nid=$1 WHERE state_snapshot_nid=$2 AND state_snapshot_nid<$3`, newNID, snapshotdata.StateSnapshotNID, maxsnapshotid); err != nil {
				return fmt.Errorf("tx.Exec (update rooms): %w", err)
			}
		}
	}

	// By this point we should have no more state_snapshot_nids below maxsnapshotid in either roomserver_rooms or roomserver_events
	// If we do, this is a problem if Dendrite tries to load the snapshot as it will not exist
	// in roomserver_state_snapshots
	var count int64
	if err = tx.QueryRow(`SELECT COUNT(*) FROM roomserver_events WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, maxsnapshotid).Scan(&count); err != nil {
		return fmt.Errorf("assertion query failed: %s", err)
	}
	if count > 0 {
		return fmt.Errorf("%d events exist in roomserver_events which have not been converted to a new state_snapshot_nid; this is a bug, please report", count)
	}
	if err = tx.QueryRow(`SELECT COUNT(*) FROM roomserver_rooms WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, maxsnapshotid).Scan(&count); err != nil {
		return fmt.Errorf("assertion query failed: %s", err)
	}
	if count > 0 {
		return fmt.Errorf("%d rooms exist in roomserver_rooms which have not been converted to a new state_snapshot_nid; this is a bug, please report", count)
	}

	if _, err = tx.Exec(`
		DROP TABLE _roomserver_state_snapshots;
		DROP SEQUENCE roomserver_state_snapshot_nid_seq;
	`); err != nil {
		return fmt.Errorf("tx.Exec (delete old snapshot table): %w", err)
	}
	if _, err = tx.Exec(`
		DROP TABLE _roomserver_state_block;
		DROP SEQUENCE roomserver_state_block_nid_seq;
	`); err != nil {
		return fmt.Errorf("tx.Exec (delete old block table): %w", err)
	}

	return nil
}

func DownStateBlocksRefactor(tx *sql.Tx) error {
	panic("Downgrading state storage is not supported")
}
<commit_msg>db migration: handle create events with no state blocks from v0.1.0 (#1904)<commit_after>// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package deltas

import (
	"database/sql"
	"fmt"

	"github.com/lib/pq"
	"github.com/matrix-org/dendrite/internal/sqlutil"
	"github.com/matrix-org/dendrite/roomserver/types"
	"github.com/matrix-org/util"
	"github.com/sirupsen/logrus"
)

type stateSnapshotData struct {
	StateSnapshotNID types.StateSnapshotNID
	RoomNID          types.RoomNID
}

type stateBlockData struct {
	stateSnapshotData
	StateBlockNID types.StateBlockNID
	EventNIDs     types.EventNIDs
}

func LoadStateBlocksRefactor(m *sqlutil.Migrations) {
	m.AddMigration(UpStateBlocksRefactor, DownStateBlocksRefactor)
}

// nolint:gocyclo
func UpStateBlocksRefactor(tx *sql.Tx) error {
	logrus.Warn("Performing state storage upgrade. Please wait, this may take some time!")
	defer logrus.Warn("State storage upgrade complete")

	var snapshotcount int
	var maxsnapshotid int
	var maxblockid int
	if err := tx.QueryRow(`SELECT COUNT(DISTINCT state_snapshot_nid) FROM roomserver_state_snapshots;`).Scan(&snapshotcount); err != nil {
		return fmt.Errorf("tx.QueryRow.Scan (count snapshots): %w", err)
	}
	if err := tx.QueryRow(`SELECT COALESCE(MAX(state_snapshot_nid),0) FROM roomserver_state_snapshots;`).Scan(&maxsnapshotid); err != nil {
		return fmt.Errorf("tx.QueryRow.Scan (count snapshots): %w", err)
	}
	if err := tx.QueryRow(`SELECT COALESCE(MAX(state_block_nid),0) FROM roomserver_state_block;`).Scan(&maxblockid); err != nil {
		return fmt.Errorf("tx.QueryRow.Scan (count snapshots): %w", err)
	}
	maxsnapshotid++
	maxblockid++

	if _, err := tx.Exec(`ALTER TABLE roomserver_state_block RENAME TO _roomserver_state_block;`); err != nil {
		return fmt.Errorf("tx.Exec: %w", err)
	}
	if _, err := tx.Exec(`ALTER TABLE roomserver_state_snapshots RENAME TO _roomserver_state_snapshots;`); err != nil {
		return fmt.Errorf("tx.Exec: %w", err)
	}
	// We create new sequences starting with the maximum state snapshot and block NIDs.
	// This means that all newly created snapshots and blocks by the migration will have
	// NIDs higher than these values, so that when we come to update the references to
	// these NIDs using UPDATE statements, we can guarantee we are only ever updating old
	// values and not accidentally overwriting new ones.
	if _, err := tx.Exec(fmt.Sprintf(`CREATE SEQUENCE roomserver_state_block_nid_sequence START WITH %d;`, maxblockid)); err != nil {
		return fmt.Errorf("tx.Exec: %w", err)
	}
	if _, err := tx.Exec(fmt.Sprintf(`CREATE SEQUENCE roomserver_state_snapshot_nid_sequence START WITH %d;`, maxsnapshotid)); err != nil {
		return fmt.Errorf("tx.Exec: %w", err)
	}
	_, err := tx.Exec(`
		CREATE TABLE IF NOT EXISTS roomserver_state_block (
			state_block_nid bigint PRIMARY KEY DEFAULT nextval('roomserver_state_block_nid_sequence'),
			state_block_hash BYTEA UNIQUE,
			event_nids bigint[] NOT NULL
		);
	`)
	if err != nil {
		return fmt.Errorf("tx.Exec (create blocks table): %w", err)
	}
	_, err = tx.Exec(`
		CREATE TABLE IF NOT EXISTS roomserver_state_snapshots (
			state_snapshot_nid bigint PRIMARY KEY DEFAULT nextval('roomserver_state_snapshot_nid_sequence'),
			state_snapshot_hash BYTEA UNIQUE,
			room_nid bigint NOT NULL,
			state_block_nids bigint[] NOT NULL
		);
	`)
	if err != nil {
		return fmt.Errorf("tx.Exec (create snapshots table): %w", err)
	}
	logrus.Warn("New tables created...")

	batchsize := 100
	for batchoffset := 0; batchoffset < snapshotcount; batchoffset += batchsize {
		var snapshotrows *sql.Rows
		snapshotrows, err = tx.Query(`
			SELECT
				state_snapshot_nid,
				room_nid,
				state_block_nid,
				ARRAY_AGG(event_nid) AS event_nids
			FROM (
				SELECT
					_roomserver_state_snapshots.state_snapshot_nid,
					_roomserver_state_snapshots.room_nid,
					_roomserver_state_block.state_block_nid,
					_roomserver_state_block.event_nid
				FROM
					_roomserver_state_snapshots
					LEFT JOIN _roomserver_state_block ON _roomserver_state_block.state_block_nid = ANY (_roomserver_state_snapshots.state_block_nids)
				WHERE
					_roomserver_state_snapshots.state_snapshot_nid = ANY (
						SELECT
							_roomserver_state_snapshots.state_snapshot_nid
						FROM
							_roomserver_state_snapshots
						ORDER BY _roomserver_state_snapshots.state_snapshot_nid ASC
						LIMIT $1 OFFSET $2
					)
			) AS _roomserver_state_block
			GROUP BY
				state_snapshot_nid,
				room_nid,
				state_block_nid;
		`, batchsize, batchoffset)
		if err != nil {
			return fmt.Errorf("tx.Query: %w", err)
		}

		logrus.Warnf("Rewriting snapshots %d-%d of %d...", batchoffset, batchoffset+batchsize, snapshotcount)
		var snapshots []stateBlockData

		var badCreateSnapshots []stateBlockData
		for snapshotrows.Next() {
			var snapshot stateBlockData
			var eventsarray []sql.NullInt64
			var nulStateBlockNID sql.NullInt64
			if err = snapshotrows.Scan(&snapshot.StateSnapshotNID, &snapshot.RoomNID, &nulStateBlockNID, pq.Array(&eventsarray)); err != nil {
				return fmt.Errorf("rows.Scan: %w", err)
			}
			if nulStateBlockNID.Valid {
				snapshot.StateBlockNID = types.StateBlockNID(nulStateBlockNID.Int64)
			}
			// Dendrite v0.1.0 would not make a state block for the create event, resulting in [NULL] from the query above.
			// Remember the snapshot and we'll fill it in after we close this cursor as we can't have 2 queries running at the same time
			if len(eventsarray) == 1 && !eventsarray[0].Valid {
				badCreateSnapshots = append(badCreateSnapshots, snapshot)
				continue
			}
			for _, e := range eventsarray {
				if e.Valid {
					snapshot.EventNIDs = append(snapshot.EventNIDs, types.EventNID(e.Int64))
				}
			}
			snapshot.EventNIDs = snapshot.EventNIDs[:util.SortAndUnique(snapshot.EventNIDs)]
			snapshots = append(snapshots, snapshot)
		}
		if err = snapshotrows.Close(); err != nil {
			return fmt.Errorf("snapshots.Close: %w", err)
		}
		// fill in bad create snapshots
		for _, s := range badCreateSnapshots {
			var createEventNID types.EventNID
			err = tx.QueryRow(
				`SELECT event_nid FROM roomserver_events WHERE state_snapshot_nid = $1 AND event_type_nid = 1`, s.StateSnapshotNID,
			).Scan(&createEventNID)
			if err != nil {
				return fmt.Errorf("cannot xref null state block with snapshot %d: %s", s.StateSnapshotNID, err)
			}
			if createEventNID == 0 {
				return fmt.Errorf("cannot xref null state block with snapshot %d, no create event", s.StateSnapshotNID)
			}
			s.EventNIDs = append(s.EventNIDs, createEventNID)
			snapshots = append(snapshots, s)
		}

		newsnapshots := map[stateSnapshotData]types.StateBlockNIDs{}

		for _, snapshot := range snapshots {
			var eventsarray pq.Int64Array
			for _, e := range snapshot.EventNIDs {
				eventsarray = append(eventsarray, int64(e))
			}

			var blocknid types.StateBlockNID
			err = tx.QueryRow(`
				INSERT INTO roomserver_state_block (state_block_hash, event_nids)
					VALUES ($1, $2)
					ON CONFLICT (state_block_hash) DO UPDATE SET event_nids=$2
					RETURNING state_block_nid
			`, snapshot.EventNIDs.Hash(), eventsarray).Scan(&blocknid)
			if err != nil {
				return fmt.Errorf("tx.QueryRow.Scan (insert new block with %d events): %w", len(eventsarray), err)
			}
			index := stateSnapshotData{snapshot.StateSnapshotNID, snapshot.RoomNID}
			newsnapshots[index] = append(newsnapshots[index], blocknid)
		}

		for snapshotdata, newblocks := range newsnapshots {
			var newblocksarray pq.Int64Array
			for _, b := range newblocks {
				newblocksarray = append(newblocksarray, int64(b))
			}

			var newNID types.StateSnapshotNID
			err = tx.QueryRow(`
				INSERT INTO roomserver_state_snapshots (state_snapshot_hash, room_nid, state_block_nids)
					VALUES ($1, $2, $3)
					ON CONFLICT (state_snapshot_hash) DO UPDATE SET room_nid=$2
					RETURNING state_snapshot_nid
			`, newblocks.Hash(), snapshotdata.RoomNID, newblocksarray).Scan(&newNID)
			if err != nil {
				return fmt.Errorf("tx.QueryRow.Scan (insert new snapshot): %w", err)
			}

			if _, err = tx.Exec(`UPDATE roomserver_events SET state_snapshot_nid=$1 WHERE state_snapshot_nid=$2 AND state_snapshot_nid<$3`, newNID, snapshotdata.StateSnapshotNID, maxsnapshotid); err != nil {
				return fmt.Errorf("tx.Exec (update events): %w", err)
			}

			if _, err = tx.Exec(`UPDATE roomserver_rooms SET state_snapshot_nid=$1 WHERE state_snapshot_nid=$2 AND state_snapshot_nid<$3`, newNID, snapshotdata.StateSnapshotNID, maxsnapshotid); err != nil {
				return fmt.Errorf("tx.Exec (update rooms): %w", err)
			}
		}
	}

	// By this point we should have no more state_snapshot_nids below maxsnapshotid in either roomserver_rooms or roomserver_events
	// If we do, this is a problem if Dendrite tries to load the snapshot as it will not exist
	// in roomserver_state_snapshots
	var count int64
	if err = tx.QueryRow(`SELECT COUNT(*) FROM roomserver_events WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, maxsnapshotid).Scan(&count); err != nil {
		return fmt.Errorf("assertion query failed: %s", err)
	}
	if count > 0 {
		return fmt.Errorf("%d events exist in roomserver_events which have not been converted to a new state_snapshot_nid; this is a bug, please report", count)
	}
	if err = tx.QueryRow(`SELECT COUNT(*) FROM roomserver_rooms WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, maxsnapshotid).Scan(&count); err != nil {
		return fmt.Errorf("assertion query failed: %s", err)
	}
	if count > 0 {
		return fmt.Errorf("%d rooms exist in roomserver_rooms which have not been converted to a new state_snapshot_nid; this is a bug, please report", count)
	}

	if _, err = tx.Exec(`
		DROP TABLE _roomserver_state_snapshots;
		DROP SEQUENCE roomserver_state_snapshot_nid_seq;
	`); err != nil {
		return fmt.Errorf("tx.Exec (delete old snapshot table): %w", err)
	}
	if _, err = tx.Exec(`
		DROP TABLE _roomserver_state_block;
		DROP SEQUENCE roomserver_state_block_nid_seq;
	`); err != nil {
		return fmt.Errorf("tx.Exec (delete old block table): %w", err)
	}

	return nil
}

func DownStateBlocksRefactor(tx *sql.Tx) error {
	panic("Downgrading state storage is not supported")
}
<|endoftext|>


<commit_before>package defaults

import (
	"testing"

	"github.com/goreleaser/goreleaser/internal/testlib"
	"github.com/goreleaser/goreleaser/pkg/config"
	"github.com/goreleaser/goreleaser/pkg/context"
	"github.com/stretchr/testify/require"
)

func TestDescription(t *testing.T) {
	require.NotEmpty(t, Pipe{}.String())
}

func TestFillBasicData(t *testing.T) {
	testlib.Mktmp(t)
	testlib.GitInit(t)
	testlib.GitRemoteAdd(t, "git@github.com:goreleaser/goreleaser.git")

	ctx := &context.Context{
		TokenType: context.TokenTypeGitHub,
		Config:    config.Project{},
	}

	require.NoError(t, Pipe{}.Run(ctx))
	require.Equal(t, "goreleaser", ctx.Config.Release.GitHub.Owner)
	require.Equal(t, "goreleaser", ctx.Config.Release.GitHub.Name)
	require.NotEmpty(t, ctx.Config.Builds)
	require.Equal(t, "goreleaser", ctx.Config.Builds[0].Binary)
	require.Equal(t, ".", ctx.Config.Builds[0].Main)
	require.Contains(t, ctx.Config.Builds[0].Goos, "darwin")
	require.Contains(t, ctx.Config.Builds[0].Goos, "linux")
	require.Contains(t, ctx.Config.Builds[0].Goarch, "386")
	require.Contains(t, ctx.Config.Builds[0].Goarch, "amd64")
	require.Equal(t, "tar.gz", ctx.Config.Archives[0].Format)
	require.Empty(t, ctx.Config.Dockers)
	require.Equal(t, "https://github.com", ctx.Config.GitHubURLs.Download)
	require.NotEmpty(t, ctx.Config.Archives[0].NameTemplate)
	require.NotEmpty(t, ctx.Config.Builds[0].Ldflags)
	require.NotEmpty(t, ctx.Config.Archives[0].Files)
	require.NotEmpty(t, ctx.Config.Dist)
}

func TestFillPartial(t *testing.T) {
	testlib.Mktmp(t)
	testlib.GitInit(t)
	testlib.GitRemoteAdd(t, "git@github.com:goreleaser/goreleaser.git")

	ctx := &context.Context{
		Config: config.Project{
			GitHubURLs: config.GitHubURLs{
				Download: "https://github.company.com",
			},
			Dist: "disttt",
			Release: config.Release{
				GitHub: config.Repo{
					Owner: "goreleaser",
					Name:  "test",
				},
			},
			Archives: []config.Archive{
				{
					Files: []config.File{
						{Source: "glob/*"},
					},
				},
			},
			Builds: []config.Build{
				{
					ID:     "build1",
					Binary: "testreleaser",
				},
				{Goos: []string{"linux"}},
				{
					ID:     "build3",
					Binary: "another",
					Ignore: []config.IgnoredBuild{
						{Goos: "darwin", Goarch: "amd64"},
					},
				},
			},
			Dockers: []config.Docker{
				{
					ImageTemplates: []string{"a/b"},
				},
			},
			Brews: []config.Homebrew{
				{
					Description: "foo",
				},
			},
		},
	}
	require.NoError(t, Pipe{}.Run(ctx))
	require.Len(t, ctx.Config.Archives[0].Files, 1)
	require.Equal(t, `bin.install "test"`, ctx.Config.Brews[0].Install)
	require.NotEmpty(t, ctx.Config.Dockers[0].Goos)
	require.NotEmpty(t, ctx.Config.Dockers[0].Goarch)
	require.NotEmpty(t, ctx.Config.Dockers[0].Dockerfile)
	require.Empty(t, ctx.Config.Dockers[0].Goarm)
	require.Equal(t, "disttt", ctx.Config.Dist)
	require.NotEqual(t, "https://github.com", ctx.Config.GitHubURLs.Download)

	ctx = &context.Context{
		TokenType: context.TokenTypeGitea,

		Config: config.Project{
			GiteaURLs: config.GiteaURLs{
				API: "https://gitea.com/api/v1",
			},
		},
	}
	require.NoError(t, Pipe{}.Run(ctx))
	require.Equal(t, "https://gitea.com", ctx.Config.GiteaURLs.Download)
}

func TestGiteaTemplateDownloadURL(t *testing.T) {
	tests := []struct {
		name    string
		apiURL  string
		wantErr bool
	}{
		{
			name:   "string_url",
			apiURL: "https://gitea.com/api/v1",
		},
		{
			name:   "download_url_template",
			apiURL: "{{ .Env.GORELEASER_TEST_GITEA_URLS_API }}",
		},
		{
			name:    "download_url_template_invalid_value",
			apiURL:  "{{ .Env.GORELEASER_NOT_EXISTS }}",
			wantErr: true,
		},
		{
			name:    "download_url_template_invalid",
			apiURL:  "{{.dddddddddd",
			wantErr: true,
		},
	}

	for _, tt := range tests {
		ctx := &context.Context{
			TokenType: context.TokenTypeGitea,
			Env: context.Env{
				"GORELEASER_TEST_GITEA_URLS_API": "https://gitea.com/api/v1",
			},
			Config: config.Project{
				GiteaURLs: config.GiteaURLs{
					API: tt.apiURL,
				},
			},
		}

		err := Pipe{}.Run(ctx)
		if tt.wantErr {
			require.Error(t, err)
			return
		}

		require.NoError(t, err)
		require.Equal(t, "https://gitea.com", ctx.Config.GiteaURLs.Download)
	}
}
<commit_msg>test: fix test<commit_after>package defaults

import (
	"testing"

	"github.com/goreleaser/goreleaser/internal/testlib"
	"github.com/goreleaser/goreleaser/pkg/config"
	"github.com/goreleaser/goreleaser/pkg/context"
	"github.com/stretchr/testify/require"
)

func TestDescription(t *testing.T) {
	require.NotEmpty(t, Pipe{}.String())
}

func TestFillBasicData(t *testing.T) {
	testlib.Mktmp(t)
	testlib.GitInit(t)
	testlib.GitRemoteAdd(t, "git@github.com:goreleaser/goreleaser.git")

	ctx := &context.Context{
		TokenType: context.TokenTypeGitHub,
		Config:    config.Project{},
	}

	require.NoError(t, Pipe{}.Run(ctx))
	require.Equal(t, "goreleaser", ctx.Config.Release.GitHub.Owner)
	require.Equal(t, "goreleaser", ctx.Config.Release.GitHub.Name)
	require.NotEmpty(t, ctx.Config.Builds)
	require.Equal(t, "goreleaser", ctx.Config.Builds[0].Binary)
	require.Equal(t, ".", ctx.Config.Builds[0].Main)
	require.Contains(t, ctx.Config.Builds[0].Goos, "darwin")
	require.Contains(t, ctx.Config.Builds[0].Goos, "linux")
	require.Contains(t, ctx.Config.Builds[0].Goarch, "386")
	require.Contains(t, ctx.Config.Builds[0].Goarch, "amd64")
	require.Equal(t, "tar.gz", ctx.Config.Archives[0].Format)
	require.Empty(t, ctx.Config.Dockers)
	require.Equal(t, "https://github.com", ctx.Config.GitHubURLs.Download)
	require.NotEmpty(t, ctx.Config.Archives[0].NameTemplate)
	require.NotEmpty(t, ctx.Config.Builds[0].Ldflags)
	require.NotEmpty(t, ctx.Config.Archives[0].Files)
	require.NotEmpty(t, ctx.Config.Dist)
}

func TestFillPartial(t *testing.T) {
	testlib.Mktmp(t)
	testlib.GitInit(t)
	testlib.GitRemoteAdd(t, "git@github.com:goreleaser/goreleaser.git")

	ctx := &context.Context{
		Config: config.Project{
			GitHubURLs: config.GitHubURLs{
				Download: "https://github.company.com",
			},
			Dist: "disttt",
			Release: config.Release{
				GitHub: config.Repo{
					Owner: "goreleaser",
					Name:  "test",
				},
			},
			Archives: []config.Archive{
				{
					Files: []config.File{
						{Source: "glob/*"},
					},
				},
			},
			Builds: []config.Build{
				{
					ID:     "build1",
					Binary: "testreleaser",
				},
				{Goos: []string{"linux"}},
				{
					ID:     "build3",
					Binary: "another",
					Ignore: []config.IgnoredBuild{
						{Goos: "darwin", Goarch: "amd64"},
					},
				},
			},
			Dockers: []config.Docker{
				{
					ImageTemplates: []string{"a/b"},
				},
			},
			Brews: []config.Homebrew{
				{
					Description: "foo",
				},
			},
		},
	}
	require.NoError(t, Pipe{}.Run(ctx))
	require.Len(t, ctx.Config.Archives[0].Files, 1)
	require.NotEmpty(t, ctx.Config.Dockers[0].Goos)
	require.NotEmpty(t, ctx.Config.Dockers[0].Goarch)
	require.NotEmpty(t, ctx.Config.Dockers[0].Dockerfile)
	require.Empty(t, ctx.Config.Dockers[0].Goarm)
	require.Equal(t, "disttt", ctx.Config.Dist)
	require.NotEqual(t, "https://github.com", ctx.Config.GitHubURLs.Download)

	ctx = &context.Context{
		TokenType: context.TokenTypeGitea,

		Config: config.Project{
			GiteaURLs: config.GiteaURLs{
				API: "https://gitea.com/api/v1",
			},
		},
	}
	require.NoError(t, Pipe{}.Run(ctx))
	require.Equal(t, "https://gitea.com", ctx.Config.GiteaURLs.Download)
}

func TestGiteaTemplateDownloadURL(t *testing.T) {
	tests := []struct {
		name    string
		apiURL  string
		wantErr bool
	}{
		{
			name:   "string_url",
			apiURL: "https://gitea.com/api/v1",
		},
		{
			name:   "download_url_template",
			apiURL: "{{ .Env.GORELEASER_TEST_GITEA_URLS_API }}",
		},
		{
			name:    "download_url_template_invalid_value",
			apiURL:  "{{ .Env.GORELEASER_NOT_EXISTS }}",
			wantErr: true,
		},
		{
			name:    "download_url_template_invalid",
			apiURL:  "{{.dddddddddd",
			wantErr: true,
		},
	}

	for _, tt := range tests {
		ctx := &context.Context{
			TokenType: context.TokenTypeGitea,
			Env: context.Env{
				"GORELEASER_TEST_GITEA_URLS_API": "https://gitea.com/api/v1",
			},
			Config: config.Project{
				GiteaURLs: config.GiteaURLs{
					API: tt.apiURL,
				},
			},
		}

		err := Pipe{}.Run(ctx)
		if tt.wantErr {
			require.Error(t, err)
			return
		}

		require.NoError(t, err)
		require.Equal(t, "https://gitea.com", ctx.Config.GiteaURLs.Download)
	}
}
<|endoftext|>


<commit_before>package interceptor

import (
	"encoding/json"
	"github.com/datawire/teleproxy/internal/pkg/nat"
	rt "github.com/datawire/teleproxy/internal/pkg/route"
	"log"
	"net"
	"strings"
	"sync"
)

type Interceptor struct {
	port       string
	tables     map[string]rt.Table
	translator *nat.Translator
	domains    sync.Map
	work       chan func()
	done       chan empty
}

type empty interface{}

func NewInterceptor(name string) *Interceptor {
	return &Interceptor{
		tables:     make(map[string]rt.Table),
		translator: nat.NewTranslator(name),
		work:       make(chan func()),
		done:       make(chan empty),
	}
}

func (i *Interceptor) Start() {
	go func() {
		defer close(i.done)
		i.translator.Enable()
		defer i.translator.Disable()
		for {
			action, ok := <-i.work
			if ok {
				action()
			} else {
				return
			}
		}
	}()
}

func (i *Interceptor) Stop() {
	close(i.work)
	<-i.done
}

func (i *Interceptor) Resolve(name string) *rt.Route {
	value, ok := i.domains.Load(strings.ToLower(name))
	if ok {
		return value.(*rt.Route)
	} else {
		return nil
	}
}

func (i *Interceptor) Destination(conn *net.TCPConn) (string, error) {
	_, host, err := i.translator.GetOriginalDst(conn)
	return host, err
}

func (i *Interceptor) Render(table string) string {
	result := make(chan string, 1)
	i.work <- func() {
		var obj interface{}

		if table == "" {
			var tables []rt.Table
			for _, t := range i.tables {
				tables = append(tables, t)
			}
			obj = tables
		} else {
			var ok bool
			obj, ok = i.tables[table]
			if !ok {
				result <- ""
				return
			}
		}

		bytes, err := json.MarshalIndent(obj, "", "  ")
		if err != nil {
			result <- err.Error()
		} else {
			result <- string(bytes)
		}
	}
	return <-result
}

func (i *Interceptor) Delete(table string) bool {
	result := make(chan bool, 1)
	i.work <- func() {
		var names []string
		if table == "" {
			for name := range i.tables {
				names = append(names, name)
			}
		} else if _, ok := i.tables[table]; ok {
			names = []string{table}
		} else {
			result <- false
		}

		for _, name := range names {
			if name != "bootstrap" {
				i.update(rt.Table{Name: name})
			}
		}

		result <- true
	}
	return <-result
}

func (i *Interceptor) Update(table rt.Table) {
	i.work <- func() {
		i.update(table)
	}
}

func (i *Interceptor) update(table rt.Table) {
	old, ok := i.tables[table.Name]

	routes := make(map[string]rt.Route)
	if ok {
		for _, route := range old.Routes {
			routes[route.Name] = route
		}
	}

	for _, route := range table.Routes {
		existing, ok := routes[route.Name]
		if ok && route != existing {

			switch route.Proto {
			case "tcp":
				i.translator.ClearTCP(existing.Ip)
			case "udp":
				i.translator.ClearUDP(existing.Ip)
			default:
				log.Printf("INT: unrecognized protocol: %v", route)
			}

		}

		if !ok || route != existing {

			if route.Target != "" {
				switch route.Proto {
				case "tcp":
					i.translator.ForwardTCP(route.Ip, route.Target)
				case "udp":
					i.translator.ForwardUDP(route.Ip, route.Target)
				default:
					log.Printf("INT: unrecognized protocol: %v", route)
				}
			}

			if route.Name != "" {
				log.Printf("INT: STORE %v->%v", route.Domain(), route)
				copy := route
				i.domains.Store(route.Domain(), &copy)
			}

		}

		if ok {
			// remove the route from our map of
			// old routes so we don't end up
			// deleting it below
			delete(routes, route.Name)
		}
	}

	for _, route := range routes {
		log.Printf("INT: CLEAR %v->%v", route.Domain(), route)
		i.domains.Delete(route.Domain())

		switch route.Proto {
		case "tcp":
			i.translator.ClearTCP(route.Ip)
		case "udp":
			i.translator.ClearUDP(route.Ip)
		default:
			log.Printf("INT: unrecognized protocol: %v", route)
		}

	}

	if table.Routes == nil || len(table.Routes) == 0 {
		delete(i.tables, table.Name)
	} else {
		i.tables[table.Name] = table
	}
}
<commit_msg>Add DNS search path handling for the resolver<commit_after>package interceptor

import (
	"encoding/json"
	"github.com/datawire/teleproxy/internal/pkg/nat"
	rt "github.com/datawire/teleproxy/internal/pkg/route"
	"log"
	"net"
	"strings"
	"sync"
)

type Interceptor struct {
	port       string
	tables     map[string]rt.Table
	translator *nat.Translator
	domains    sync.Map
	work       chan func()
	done       chan empty
	search     []string
}

type empty interface{}

func NewInterceptor(name string) *Interceptor {
	return &Interceptor{
		tables:     make(map[string]rt.Table),
		translator: nat.NewTranslator(name),
		work:       make(chan func()),
		done:       make(chan empty),
		search:     []string{""},
	}
}

func (i *Interceptor) Start() {
	go func() {
		defer close(i.done)
		i.translator.Enable()
		defer i.translator.Disable()
		for {
			action, ok := <-i.work
			if ok {
				action()
			} else {
				return
			}
		}
	}()
}

func (i *Interceptor) Stop() {
	close(i.work)
	<-i.done
}

// Resolve looks up the given query in the (FIXME: somewhere), trying
// all the suffixes in the search path, and returns a Route on success
// or nil on failure. This implementation does not count the number of
// dots in the query.
func (i *Interceptor) Resolve(query string) *rt.Route {
	if !strings.HasSuffix(query, ".") {
		query += "."
	}
	for _, suffix := range i.GetSearchPath() {
		name := query + suffix
		value, ok := i.domains.Load(strings.ToLower(name))
		if ok {
			return value.(*rt.Route)
		}
	}
	return nil
}

func (i *Interceptor) Destination(conn *net.TCPConn) (string, error) {
	_, host, err := i.translator.GetOriginalDst(conn)
	return host, err
}

func (i *Interceptor) Render(table string) string {
	result := make(chan string, 1)
	i.work <- func() {
		var obj interface{}

		if table == "" {
			var tables []rt.Table
			for _, t := range i.tables {
				tables = append(tables, t)
			}
			obj = tables
		} else {
			var ok bool
			obj, ok = i.tables[table]
			if !ok {
				result <- ""
				return
			}
		}

		bytes, err := json.MarshalIndent(obj, "", "  ")
		if err != nil {
			result <- err.Error()
		} else {
			result <- string(bytes)
		}
	}
	return <-result
}

func (i *Interceptor) Delete(table string) bool {
	result := make(chan bool, 1)
	i.work <- func() {
		var names []string
		if table == "" {
			for name := range i.tables {
				names = append(names, name)
			}
		} else if _, ok := i.tables[table]; ok {
			names = []string{table}
		} else {
			result <- false
		}

		for _, name := range names {
			if name != "bootstrap" {
				i.update(rt.Table{Name: name})
			}
		}

		result <- true
	}
	return <-result
}

func (i *Interceptor) Update(table rt.Table) {
	i.work <- func() {
		i.update(table)
	}
}

func (i *Interceptor) update(table rt.Table) {
	old, ok := i.tables[table.Name]

	routes := make(map[string]rt.Route)
	if ok {
		for _, route := range old.Routes {
			routes[route.Name] = route
		}
	}

	for _, route := range table.Routes {
		existing, ok := routes[route.Name]
		if ok && route != existing {

			switch route.Proto {
			case "tcp":
				i.translator.ClearTCP(existing.Ip)
			case "udp":
				i.translator.ClearUDP(existing.Ip)
			default:
				log.Printf("INT: unrecognized protocol: %v", route)
			}

		}

		if !ok || route != existing {

			if route.Target != "" {
				switch route.Proto {
				case "tcp":
					i.translator.ForwardTCP(route.Ip, route.Target)
				case "udp":
					i.translator.ForwardUDP(route.Ip, route.Target)
				default:
					log.Printf("INT: unrecognized protocol: %v", route)
				}
			}

			if route.Name != "" {
				log.Printf("INT: STORE %v->%v", route.Domain(), route)
				copy := route
				i.domains.Store(route.Domain(), &copy)
			}

		}

		if ok {
			// remove the route from our map of
			// old routes so we don't end up
			// deleting it below
			delete(routes, route.Name)
		}
	}

	for _, route := range routes {
		log.Printf("INT: CLEAR %v->%v", route.Domain(), route)
		i.domains.Delete(route.Domain())

		switch route.Proto {
		case "tcp":
			i.translator.ClearTCP(route.Ip)
		case "udp":
			i.translator.ClearUDP(route.Ip)
		default:
			log.Printf("INT: unrecognized protocol: %v", route)
		}

	}

	if table.Routes == nil || len(table.Routes) == 0 {
		delete(i.tables, table.Name)
	} else {
		i.tables[table.Name] = table
	}
}

// SetSearchPath updates the DNS search path used by the resolver
func (i *Interceptor) SetSearchPath(paths []string) {
	i.work <- func() {
		i.search = paths
	}
}

// GetSearchPath retrieves the current search path
func (i *Interceptor) GetSearchPath() []string {
	result := make(chan []string, 1)
	i.work <- func() {
		result <- i.search
	}
	return <-result
}
<|endoftext|>


<commit_before>package generatorControllers

import (
	. "eaciit/wfdemo-git/library/helper"
	. "eaciit/wfdemo-git/library/models"
	. "eaciit/wfdemo-git/processapp/summaryGenerator/controllers"
	"eaciit/wfdemo-git/web/helper"
	_ "fmt"
	"log"
	"os"
	_ "strings"
	"time"

	"strings"

	"github.com/eaciit/dbox"
	_ "github.com/eaciit/dbox/dbc/mongo"
	tk "github.com/eaciit/toolkit"
)

type GenScadaLast24 struct {
	*BaseController
}

func (d *GenScadaLast24) Generate(base *BaseController) {
	if base != nil {
		d.BaseController = base
		ctx, e := PrepareConnection()
		if e != nil {
			ErrorHandler(e, "Scada Summary")
			os.Exit(0)
		}

		d.BaseController.Ctx.DeleteMany(new(ScadaLastUpdate), dbox.And(dbox.Ne("_id", "")))

		projectList, _ := helper.GetProjectList()

		inprojectactive := func(str string) bool {
			for _, v := range projectList {
				if v.Value == str {
					return true
				}
			}
			return false
		}

		mapbudget := map[string]float64{}
		csrBudget, _ := ctx.NewQuery().From(new(ExpPValueModel).TableName()).
			Cursor(nil)

		budgets := make([]ExpPValueModel, 0)
		_ = csrBudget.Fetch(&budgets, 0, false)
		csrBudget.Close()

		for _, budget := range budgets {
			mapbudget[tk.Sprintf("%s_%d", budget.ProjectName, budget.MonthNo)] = budget.P75NetGenMWH
			if inprojectactive(budget.ProjectName) {
				mapbudget[tk.Sprintf("fleet_%d", budget.MonthNo)] = budget.P75NetGenMWH
			}
		}

		for _, proj := range d.BaseController.ProjectList {
			projectName := proj.Value
			turbineList := []TurbineOut{}

			if projectName != "Fleet" {
				turbineList, _ = helper.GetTurbineList([]interface{}{projectName})
			} else {
				turbineList, _ = helper.GetTurbineList(nil)
			}

			totalTurbine := len(turbineList)

			filter := dbox.Eq("available", 1)
			if projectName != "Fleet" {
				filter = dbox.And(dbox.Eq("projectname", projectName), filter)
			}

			/*for _, v := range filter {
				log.Printf(">> %#v \n", v)
			}*/

			csr, e := ctx.NewQuery().
				From(new(ScadaData).TableName()).
				Where(filter).
				Aggr(dbox.AggrMax, "$timestamp", "timestamp").
				Aggr(dbox.AggrMax, "$dateinfo.dateid", "dateid").
				Group("").
				Cursor(nil)

			if e != nil {
				log.Printf("Error: %v \n", e.Error())
			} else {
				datas := []tk.M{}
				e = csr.Fetch(&datas, 0, false)
				csr.Close()

				tk.Printf(">> %#v \n", datas)

				if len(datas) > 0 {
					dateId := datas[0].Get("dateid", time.Time{}).(time.Time).UTC()
					dtInfo := GetDateInfo(dateId)
					maxTimeStamp := datas[0].Get("timestamp", time.Time{}).(time.Time).UTC()

					var budgetCurrMonthDaily float64

					_id := tk.Sprintf("%s_%d", projectName, dateId.Month())
					if val, cond := mapbudget[_id]; cond {
						budgetCurrMonths := val * 1000.0
						noOfDay := float64(daysIn(dateId.Month(), dateId.Year()))
						budgetCurrMonthDaily = tk.Div(budgetCurrMonths, noOfDay)
					}

					mdl := new(ScadaLastUpdate).New()

					if projectName != "Fleet" {
						mdl.ID = "SCADALASTUPDATE_" + strings.ToUpper(projectName)
						mdl.ProjectName = projectName
						mdl.NoOfProjects = 1
					} else {
						mdl.ID = "SCADALASTUPDATE_FLEET"
						mdl.ProjectName = "Fleet"
						mdl.NoOfProjects = len(d.BaseController.ProjectList) - 1
					}

					for _, t := range turbineList {
						mdl.TotalMaxCapacity += t.Capacity
					}

					mdl.TotalMaxCapacity = tk.ToFloat64(mdl.TotalMaxCapacity*1000.0, 2, tk.RoundingAuto)
					mdl.LastUpdate = maxTimeStamp
					mdl.DateInfo = dtInfo
					mdl.NoOfTurbines = totalTurbine

					items := make([]LastData24Hours, 0)
					cdatehour := dateId.UTC()
					for i := 0; i < 24; i++ {
						cdatehour = cdatehour.Add(time.Duration(i) * time.Hour)

						// year := strconv.Itoa(dateId.Year())
						// month := dateId.Month().String()
						// day := strconv.Itoa(dateId.Day())
						// strTime := year + "-" + month + "-" + day + " " + strconv.Itoa(i) + ":00:00"
						// timeHr, _ := time.Parse("2006-January-2 15:04:05", strTime)

						// timeHrStart := timeHr.Add(-1 * time.Hour)

						filterSub := []*dbox.Filter{}
						filterSub = append(filterSub, dbox.Gt("timestamp", cdatehour.Add(time.Hour*-1)))
						filterSub = append(filterSub, dbox.Lte("timestamp", cdatehour))
						filterSub = append(filterSub, dbox.Eq("available", 1))

						if projectName != "Fleet" {
							filterSub = append(filterSub, dbox.Eq("projectname", projectName))
						}

						csr, e = ctx.NewQuery().From(new(ScadaData).TableName()).
							Where(dbox.And(filterSub...)).
							Aggr(dbox.AggrSum, "$power", "totalpower").
							Aggr(dbox.AggrSum, "$powerlost", "totalpowerlost").
							Aggr(dbox.AggrSum, "$energylost", "energylost").
							Aggr(dbox.AggrSum, "$denpower", "denpower").
							Aggr(dbox.AggrSum, "$oktime", "totaloktime").
							Aggr(dbox.AggrSum, "$griddowntime", "totalgriddowntime").
							Aggr(dbox.AggrAvr, "$windspeed", "avgwindspeed").
							Group("projectname").
							Cursor(nil)
						defer csr.Close()

						scadas := []tk.M{}
						e = csr.Fetch(&scadas, 0, false)

						var last LastData24Hours
						if len(scadas) > 0 {
							data := scadas[0]
							trueAvail := 0.0
							gridAvail := 0.0

							ipower := data["totalpower"]
							power := 0.0
							if ipower != nil {
								power = ipower.(float64)
							}

							ipotentialpower := data["denpower"]
							potentialpower := 0.0
							if ipotentialpower != nil {
								potentialpower = ipotentialpower.(float64)
							}

							iwindspeed := data["avgwindspeed"]
							windspeed := 0.0
							if iwindspeed != nil {
								windspeed = iwindspeed.(float64)
							}
							last.Hour = i
							last.TimeHour = cdatehour
							last.AvgWindSpeed = windspeed
							last.PowerKw = power
							last.EnergyKwh = power / 6
							last.Potential = potentialpower
							last.PotentialKwh = potentialpower / 6
							last.TrueAvail = trueAvail
							last.GridAvail = gridAvail
						} else {
							last.Hour = i
							last.TimeHour = cdatehour
							last.AvgWindSpeed = 0.0
							last.PowerKw = 0.0
							last.EnergyKwh = 0.0
							last.Potential = 0.0
							last.PotentialKwh = 0.0
							last.TrueAvail = 0.0
							last.GridAvail = 0.0
						}

						items = append(items, last)
					}

					match := tk.M{}

					match.Set("dateinfo.monthid", tk.M{}.Set("$eq", dtInfo.MonthId)).Set("power", tk.M{}.Set("$gte", -200))

					if projectName != "Fleet" {
						match.Set("projectname", projectName)
					}

					pipe := []tk.M{tk.M{}.Set("$match", match), tk.M{}.Set("$group", tk.M{}.Set("_id", "$dateinfo.dateid").Set("totalpower", tk.M{}.Set("$sum", "$power"))), tk.M{}.Set("$sort", tk.M{}.Set("_id", 1))}

					csr, _ := ctx.NewQuery().
						Command("pipe", pipe).
						From(new(ScadaData).TableName()).
						Cursor(nil)
					defer csr.Close()

					scadas := []tk.M{}
					e = csr.Fetch(&scadas, 0, false)

					item30s := make([]Last30Days, 0)
					dateData := dateId
					cummProd := 0.0
					cummBudget := 0.0
					for _, data := range scadas {
						dateData = data["_id"].(time.Time)
						var last30 Last30Days
						last30.DateId = dateData
						last30.DayNo = dateData.Day()

						currProd := 0.0
						currBudget := budgetCurrMonthDaily // 565160.32
						if data != nil {
							ipower := data["totalpower"]
							power := 0.0
							if ipower != nil {
								power = ipower.(float64)
							}
							currProd = power / 6
						}
						cummProd = cummProd + currProd
						cummBudget = cummBudget + currBudget

						last30.CurrBudget = currBudget
						last30.CurrProduction = currProd
						last30.CumBudget = cummBudget / 1000000
						last30.CumProduction = cummProd / 1000000

						item30s = append(item30s, last30)

						dateData = dateId.Add(-1)
					}

					mdl.Productions = items
					mdl.CummulativeProductions = item30s

					d.BaseController.Ctx.Insert(mdl)
				}
			}
		}
	}
}
<commit_msg>filter available only<commit_after>package generatorControllers

import (
	. "eaciit/wfdemo-git/library/helper"
	. "eaciit/wfdemo-git/library/models"
	. "eaciit/wfdemo-git/processapp/summaryGenerator/controllers"
	"eaciit/wfdemo-git/web/helper"
	_ "fmt"
	"log"
	"os"
	_ "strings"
	"time"

	"strings"

	"github.com/eaciit/dbox"
	_ "github.com/eaciit/dbox/dbc/mongo"
	tk "github.com/eaciit/toolkit"
)

type GenScadaLast24 struct {
	*BaseController
}

func (d *GenScadaLast24) Generate(base *BaseController) {
	if base != nil {
		d.BaseController = base
		ctx, e := PrepareConnection()
		if e != nil {
			ErrorHandler(e, "Scada Summary")
			os.Exit(0)
		}

		d.BaseController.Ctx.DeleteMany(new(ScadaLastUpdate), dbox.And(dbox.Ne("_id", "")))

		projectList, _ := helper.GetProjectList()

		inprojectactive := func(str string) bool {
			for _, v := range projectList {
				if v.Value == str {
					return true
				}
			}
			return false
		}

		mapbudget := map[string]float64{}
		csrBudget, _ := ctx.NewQuery().From(new(ExpPValueModel).TableName()).
			Cursor(nil)

		budgets := make([]ExpPValueModel, 0)
		_ = csrBudget.Fetch(&budgets, 0, false)
		csrBudget.Close()

		for _, budget := range budgets {
			mapbudget[tk.Sprintf("%s_%d", budget.ProjectName, budget.MonthNo)] = budget.P75NetGenMWH
			if inprojectactive(budget.ProjectName) {
				mapbudget[tk.Sprintf("fleet_%d", budget.MonthNo)] = budget.P75NetGenMWH
			}
		}

		for _, proj := range d.BaseController.ProjectList {
			projectName := proj.Value
			turbineList := []TurbineOut{}

			if projectName != "Fleet" {
				turbineList, _ = helper.GetTurbineList([]interface{}{projectName})
			} else {
				turbineList, _ = helper.GetTurbineList(nil)
			}

			totalTurbine := len(turbineList)

			filter := dbox.Eq("available", 1)
			if projectName != "Fleet" {
				filter = dbox.And(dbox.Eq("projectname", projectName), filter)
			}

			/*for _, v := range filter {
				log.Printf(">> %#v \n", v)
			}*/

			csr, e := ctx.NewQuery().
				From(new(ScadaData).TableName()).
				Where(filter).
				Aggr(dbox.AggrMax, "$timestamp", "timestamp").
				Aggr(dbox.AggrMax, "$dateinfo.dateid", "dateid").
				Group("").
				Cursor(nil)

			if e != nil {
				log.Printf("Error: %v \n", e.Error())
			} else {
				datas := []tk.M{}
				e = csr.Fetch(&datas, 0, false)
				csr.Close()

				tk.Printf(">> %#v \n", datas)

				if len(datas) > 0 {
					dateId := datas[0].Get("dateid", time.Time{}).(time.Time).UTC()
					dtInfo := GetDateInfo(dateId)
					maxTimeStamp := datas[0].Get("timestamp", time.Time{}).(time.Time).UTC()

					var budgetCurrMonthDaily float64

					_id := tk.Sprintf("%s_%d", projectName, dateId.Month())
					if val, cond := mapbudget[_id]; cond {
						budgetCurrMonths := val * 1000.0
						noOfDay := float64(daysIn(dateId.Month(), dateId.Year()))
						budgetCurrMonthDaily = tk.Div(budgetCurrMonths, noOfDay)
					}

					mdl := new(ScadaLastUpdate).New()

					if projectName != "Fleet" {
						mdl.ID = "SCADALASTUPDATE_" + strings.ToUpper(projectName)
						mdl.ProjectName = projectName
						mdl.NoOfProjects = 1
					} else {
						mdl.ID = "SCADALASTUPDATE_FLEET"
						mdl.ProjectName = "Fleet"
						mdl.NoOfProjects = len(d.BaseController.ProjectList) - 1
					}

					for _, t := range turbineList {
						mdl.TotalMaxCapacity += t.Capacity
					}

					mdl.TotalMaxCapacity = tk.ToFloat64(mdl.TotalMaxCapacity*1000.0, 2, tk.RoundingAuto)
					mdl.LastUpdate = maxTimeStamp
					mdl.DateInfo = dtInfo
					mdl.NoOfTurbines = totalTurbine

					items := make([]LastData24Hours, 0)
					cdatehour := dateId.UTC()
					for i := 0; i < 24; i++ {
						cdatehour = cdatehour.Add(time.Duration(i) * time.Hour)

						// year := strconv.Itoa(dateId.Year())
						// month := dateId.Month().String()
						// day := strconv.Itoa(dateId.Day())
						// strTime := year + "-" + month + "-" + day + " " + strconv.Itoa(i) + ":00:00"
						// timeHr, _ := time.Parse("2006-January-2 15:04:05", strTime)

						// timeHrStart := timeHr.Add(-1 * time.Hour)

						filterSub := []*dbox.Filter{}
						filterSub = append(filterSub, dbox.Gt("timestamp", cdatehour.Add(time.Hour*-1)))
						filterSub = append(filterSub, dbox.Lte("timestamp", cdatehour))
						filterSub = append(filterSub, dbox.Eq("available", 1))

						if projectName != "Fleet" {
							filterSub = append(filterSub, dbox.Eq("projectname", projectName))
						}

						csr, e = ctx.NewQuery().From(new(ScadaData).TableName()).
							Where(dbox.And(filterSub...)).
							Aggr(dbox.AggrSum, "$power", "totalpower").
							Aggr(dbox.AggrSum, "$powerlost", "totalpowerlost").
							Aggr(dbox.AggrSum, "$energylost", "energylost").
							Aggr(dbox.AggrSum, "$denpower", "denpower").
							Aggr(dbox.AggrSum, "$oktime", "totaloktime").
							Aggr(dbox.AggrSum, "$griddowntime", "totalgriddowntime").
							Aggr(dbox.AggrAvr, "$windspeed", "avgwindspeed").
							Group("projectname").
							Cursor(nil)
						defer csr.Close()

						scadas := []tk.M{}
						e = csr.Fetch(&scadas, 0, false)

						var last LastData24Hours
						if len(scadas) > 0 {
							data := scadas[0]
							trueAvail := 0.0
							gridAvail := 0.0

							ipower := data["totalpower"]
							power := 0.0
							if ipower != nil {
								power = ipower.(float64)
							}

							ipotentialpower := data["denpower"]
							potentialpower := 0.0
							if ipotentialpower != nil {
								potentialpower = ipotentialpower.(float64)
							}

							iwindspeed := data["avgwindspeed"]
							windspeed := 0.0
							if iwindspeed != nil {
								windspeed = iwindspeed.(float64)
							}
							last.Hour = i
							last.TimeHour = cdatehour
							last.AvgWindSpeed = windspeed
							last.PowerKw = power
							last.EnergyKwh = power / 6
							last.Potential = potentialpower
							last.PotentialKwh = potentialpower / 6
							last.TrueAvail = trueAvail
							last.GridAvail = gridAvail
						} else {
							last.Hour = i
							last.TimeHour = cdatehour
							last.AvgWindSpeed = 0.0
							last.PowerKw = 0.0
							last.EnergyKwh = 0.0
							last.Potential = 0.0
							last.PotentialKwh = 0.0
							last.TrueAvail = 0.0
							last.GridAvail = 0.0
						}

						items = append(items, last)
					}

					match := tk.M{}

					match.Set("dateinfo.monthid", tk.M{}.Set("$eq", dtInfo.MonthId)).Set("available", tk.M{}.Set("$eq", 1))

					if projectName != "Fleet" {
						match.Set("projectname", projectName)
					}

					pipe := []tk.M{tk.M{}.Set("$match", match), tk.M{}.Set("$group", tk.M{}.Set("_id", "$dateinfo.dateid").Set("totalpower", tk.M{}.Set("$sum", "$power"))), tk.M{}.Set("$sort", tk.M{}.Set("_id", 1))}

					csr, _ := ctx.NewQuery().
						Command("pipe", pipe).
						From(new(ScadaData).TableName()).
						Cursor(nil)
					defer csr.Close()

					scadas := []tk.M{}
					e = csr.Fetch(&scadas, 0, false)

					item30s := make([]Last30Days, 0)
					dateData := dateId
					cummProd := 0.0
					cummBudget := 0.0
					for _, data := range scadas {
						dateData = data["_id"].(time.Time)
						var last30 Last30Days
						last30.DateId = dateData
						last30.DayNo = dateData.Day()

						currProd := 0.0
						currBudget := budgetCurrMonthDaily // 565160.32
						if data != nil {
							ipower := data["totalpower"]
							power := 0.0
							if ipower != nil {
								power = ipower.(float64)
							}
							currProd = power / 6
						}
						cummProd = cummProd + currProd
						cummBudget = cummBudget + currBudget

						last30.CurrBudget = currBudget
						last30.CurrProduction = currProd
						last30.CumBudget = cummBudget / 1000000
						last30.CumProduction = cummProd / 1000000

						item30s = append(item30s, last30)

						dateData = dateId.Add(-1)
					}

					mdl.Productions = items
					mdl.CummulativeProductions = item30s

					d.BaseController.Ctx.Insert(mdl)
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package gateway

import (
	"fmt"
	"log"
	"net/http"

	"encoding/json"

	"github.com/ReneGa/tweetcount-microservices/recorder/domain"
)

// Tweets is a gateway to a tweet producing service
type Tweets interface {
	Tweets(query string) domain.Tweets
}

// HTTPTweets is the gateway to get tweets over http
type HTTPTweets struct {
	Client *http.Client
	URL    string
}

type decodeResult int

const (
	decodeError decodeResult = iota
	decodeStopped
)

func decodeResponse(res *http.Response, data chan domain.Tweet, stop chan bool) decodeResult {
	defer res.Body.Close()
	var tweet domain.Tweet
	jd := json.NewDecoder(res.Body)
	for {
		select {
		case <-stop:
			return decodeStopped
		default:
			err := jd.Decode(&tweet)
			if err != nil {
				return decodeError
			}
			data <- tweet
		}
	}
}

// Tweets return a stream of tweets for a given search query
func (t *HTTPTweets) Tweets(query string) domain.Tweets {
	url := fmt.Sprintf("%s?q=%s", t.URL, query)
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		panic(err)
	}

	data := make(chan domain.Tweet)
	stop := make(chan bool)

	tweets := domain.Tweets{
		Data: data,
		Stop: stop,
	}

	go func() {
		defer close(data)
		reconnect := true
		for reconnect {
			select {
			case <-stop:
				return
			default:
			}
			res, err := t.Client.Do(req)
			if err == nil {
				decodeResult := decodeResponse(res, data, stop)
				reconnect = decodeResult == decodeError
			} else {
				log.Println("Error: ", err)
			}
		}
	}()

	return tweets
}
<commit_msg>Handle stop during tweet sending<commit_after>package gateway

import (
	"fmt"
	"log"
	"net/http"

	"encoding/json"

	"github.com/ReneGa/tweetcount-microservices/recorder/domain"
)

// Tweets is a gateway to a tweet producing service
type Tweets interface {
	Tweets(query string) domain.Tweets
}

// HTTPTweets is the gateway to get tweets over http
type HTTPTweets struct {
	Client *http.Client
	URL    string
}

type decodeResult int

const (
	decodeError decodeResult = iota
	decodeStopped
)

func decodeResponse(res *http.Response, data chan domain.Tweet, stop chan bool) decodeResult {
	defer res.Body.Close()
	jd := json.NewDecoder(res.Body)
	for {
		select {
		case <-stop:
			return decodeStopped
		default:
			var tweet domain.Tweet
			err := jd.Decode(&tweet)
			if err != nil {
				return decodeError
			}
			select {
			case data <- tweet:
			case <-stop:
				return decodeStopped
			}
		}
	}
}

// Tweets return a stream of tweets for a given search query
func (t *HTTPTweets) Tweets(query string) domain.Tweets {
	url := fmt.Sprintf("%s?q=%s", t.URL, query)
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		panic(err)
	}

	data := make(chan domain.Tweet)
	stop := make(chan bool)

	tweets := domain.Tweets{
		Data: data,
		Stop: stop,
	}

	go func() {
		defer close(data)
		reconnect := true
		for reconnect {
			select {
			case <-stop:
				return
			default:
			}
			res, err := t.Client.Do(req)
			if err == nil {
				decodeResult := decodeResponse(res, data, stop)
				reconnect = decodeResult == decodeError
			} else {
				log.Println("Error: ", err)
			}
		}
	}()

	return tweets
}
<|endoftext|>


<commit_before>package bench

import (
	"context"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"time"

	"sort"

	"github.com/pilosa/pilosa/pilosactl"
)

func NewImport(stdin io.Reader, stdout, stderr io.Writer) *Import {
	return &Import{
		ImportCommand: pilosactl.NewImportCommand(stdin, stdout, stderr),
	}
}

// Import sets bits with increasing profile id and bitmap id.
type Import struct {
	Name              string `json:"name"`
	BaseBitmapID      int64  `json:"base-bitmap-id"`
	MaxBitmapID       int64  `json:"max-bitmap-id"`
	BaseProfileID     int64  `json:"base-profile-id"`
	MaxProfileID      int64  `json:"max-profile-id"`
	RandomBitmapOrder bool   `json:"random-bitmap-order"`
	MinBitsPerMap     int64  `json:"min-bits-per-map"`
	MaxBitsPerMap     int64  `json:"max-bits-per-map"`
	AgentControls     string `json:"agent-controls"`
	Seed              int64  `json:"seed"`
	numbits           int

	*pilosactl.ImportCommand
}

func (b *Import) Usage() string {
	return `
import generates an import file and imports using pilosa's bulk import interface

Usage: import [arguments]

The following arguments are available:

	-base-bitmap-id int
		bits being set will all be greater than this

	-maximum-bitmap-id int
		bits being set will all be less than this

	-base-profile-id int
		profile id num to start from

	-max-profile-id int
		maximum profile id to generate

	-random-bitmap-order
		if this option is set, the import file will not be sorted by bitmap id

	-min-bits-per-map int
		minimum number of bits set per bitmap

	-max-bits-per-map int
		maximum number of bits set per bitmap

	-agent-controls string
		can be 'height', 'width', or empty (TODO or square?)- increasing
		number of agents modulates bitmap id range, profile id range,
		or just sets more bits in the same range.

	-seed int
		seed for RNG

	-db string
		pilosa db to use

	-frame string
		frame to import into
`[1:]
}

func (b *Import) ConsumeFlags(args []string) ([]string, error) {
	fs := flag.NewFlagSet("Import", flag.ContinueOnError)
	fs.SetOutput(ioutil.Discard)
	fs.Int64Var(&b.BaseBitmapID, "base-bitmap-id", 0, "")
	fs.Int64Var(&b.MaxBitmapID, "max-bitmap-id", 1000, "")
	fs.Int64Var(&b.BaseProfileID, "base-profile-id", 0, "")
	fs.Int64Var(&b.MaxProfileID, "max-profile-id", 1000, "")
	fs.BoolVar(&b.RandomBitmapOrder, "random-bitmap-order", false, "")
	fs.Int64Var(&b.MinBitsPerMap, "min-bits-per-map", 0, "")
	fs.Int64Var(&b.MaxBitsPerMap, "max-bits-per-map", 10, "")
	fs.StringVar(&b.AgentControls, "agent-controls", "", "")
	fs.Int64Var(&b.Seed, "seed", 0, "")
	fs.StringVar(&b.Database, "db", "benchdb", "")
	fs.StringVar(&b.Frame, "frame", "testframe", "")
	fs.IntVar(&b.BufferSize, "buffer-size", 10000000, "")

	if err := fs.Parse(args); err != nil {
		return nil, err
	}
	return fs.Args(), nil
}

func (b *Import) Init(hosts []string, agentNum int) error {
	if len(hosts) == 0 {
		return fmt.Errorf("Need at least one host")
	}
	b.Name = "import"
	b.Host = hosts[0]
	// generate csv data
	baseBitmapID, maxBitmapID, baseProfileID, maxProfileID := b.BaseBitmapID, b.MaxBitmapID, b.BaseProfileID, b.MaxProfileID
	switch b.AgentControls {
	case "height":
		numBitmapIDs := (b.MaxBitmapID - b.BaseBitmapID)
		baseBitmapID = b.BaseBitmapID + (numBitmapIDs * int64(agentNum))
		maxBitmapID = baseBitmapID + numBitmapIDs
	case "width":
		numProfileIDs := (b.MaxProfileID - b.BaseProfileID)
		baseProfileID = b.BaseProfileID + (numProfileIDs * int64(agentNum))
		maxProfileID = baseProfileID + numProfileIDs
	case "":
		break
	default:
		return fmt.Errorf("agent-controls: '%v' is not supported", b.AgentControls)
	}
	f, err := ioutil.TempFile("", "")
	if err != nil {
		return err
	}
	// set b.Paths)
	num := GenerateImportCSV(f, baseBitmapID, maxBitmapID, baseProfileID, maxProfileID,
		b.MinBitsPerMap, b.MaxBitsPerMap, b.Seed+int64(agentNum), b.RandomBitmapOrder)
	b.numbits = num
	// set b.Paths
	f.Close()
	b.Paths = []string{f.Name()}
	return nil
}

// Run runs the Import benchmark
func (b *Import) Run(ctx context.Context, agentNum int) map[string]interface{} {
	results := make(map[string]interface{})
	results["numbits"] = b.numbits
	results["db"] = b.Database
	start := time.Now()
	err := b.ImportCommand.Run(ctx)

	if err != nil {
		results["error"] = err.Error()
	}
	results["time"] = time.Now().Sub(start)
	return results
}

type Int64Slice []int64

func (s Int64Slice) Len() int           { return len(s) }
func (s Int64Slice) Less(i, j int) bool { return s[i] < s[j] }
func (s Int64Slice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

func GenerateImportCSV(w io.Writer, baseBitmapID, maxBitmapID, baseProfileID, maxProfileID, minBitsPerMap, maxBitsPerMap, seed int64, randomOrder bool) int {
	src := rand.NewSource(seed)
	rng := rand.New(src)

	var bitmapIDs []int
	if randomOrder {
		bitmapIDs = rng.Perm(int(maxBitmapID - baseBitmapID))
	}
	numrows := 0
	profileIDs := make(Int64Slice, maxBitsPerMap)
	for i := baseBitmapID; i < maxBitmapID; i++ {
		var bitmapID int64
		if randomOrder {
			bitmapID = int64(bitmapIDs[i-baseBitmapID])
		} else {
			bitmapID = int64(i)
		}

		numBitsToSet := rng.Int63n(maxBitsPerMap-minBitsPerMap) + minBitsPerMap
		numrows += int(numBitsToSet)
		for j := int64(0); j < numBitsToSet; j++ {
			profileIDs[j] = rng.Int63n(maxProfileID-baseProfileID) + baseProfileID
		}
		profIDs := profileIDs[:numBitsToSet]
		if !randomOrder {
			sort.Sort(profIDs)
		}
		for j := int64(0); j < numBitsToSet; j++ {
			fmt.Fprintf(w, "%d,%d\n", bitmapID, profIDs[j])
		}

	}
	return numrows
}
<commit_msg>fix usage typo maximum->max<commit_after>package bench

import (
	"context"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"time"

	"sort"

	"github.com/pilosa/pilosa/pilosactl"
)

func NewImport(stdin io.Reader, stdout, stderr io.Writer) *Import {
	return &Import{
		ImportCommand: pilosactl.NewImportCommand(stdin, stdout, stderr),
	}
}

// Import sets bits with increasing profile id and bitmap id.
type Import struct {
	Name              string `json:"name"`
	BaseBitmapID      int64  `json:"base-bitmap-id"`
	MaxBitmapID       int64  `json:"max-bitmap-id"`
	BaseProfileID     int64  `json:"base-profile-id"`
	MaxProfileID      int64  `json:"max-profile-id"`
	RandomBitmapOrder bool   `json:"random-bitmap-order"`
	MinBitsPerMap     int64  `json:"min-bits-per-map"`
	MaxBitsPerMap     int64  `json:"max-bits-per-map"`
	AgentControls     string `json:"agent-controls"`
	Seed              int64  `json:"seed"`
	numbits           int

	*pilosactl.ImportCommand
}

func (b *Import) Usage() string {
	return `
import generates an import file and imports using pilosa's bulk import interface

Usage: import [arguments]

The following arguments are available:

	-base-bitmap-id int
		bits being set will all be greater than this

	-max-bitmap-id int
		bits being set will all be less than this

	-base-profile-id int
		profile id num to start from

	-max-profile-id int
		maximum profile id to generate

	-random-bitmap-order
		if this option is set, the import file will not be sorted by bitmap id

	-min-bits-per-map int
		minimum number of bits set per bitmap

	-max-bits-per-map int
		maximum number of bits set per bitmap

	-agent-controls string
		can be 'height', 'width', or empty (TODO or square?)- increasing
		number of agents modulates bitmap id range, profile id range,
		or just sets more bits in the same range.

	-seed int
		seed for RNG

	-db string
		pilosa db to use

	-frame string
		frame to import into
`[1:]
}

func (b *Import) ConsumeFlags(args []string) ([]string, error) {
	fs := flag.NewFlagSet("Import", flag.ContinueOnError)
	fs.SetOutput(ioutil.Discard)
	fs.Int64Var(&b.BaseBitmapID, "base-bitmap-id", 0, "")
	fs.Int64Var(&b.MaxBitmapID, "max-bitmap-id", 1000, "")
	fs.Int64Var(&b.BaseProfileID, "base-profile-id", 0, "")
	fs.Int64Var(&b.MaxProfileID, "max-profile-id", 1000, "")
	fs.BoolVar(&b.RandomBitmapOrder, "random-bitmap-order", false, "")
	fs.Int64Var(&b.MinBitsPerMap, "min-bits-per-map", 0, "")
	fs.Int64Var(&b.MaxBitsPerMap, "max-bits-per-map", 10, "")
	fs.StringVar(&b.AgentControls, "agent-controls", "", "")
	fs.Int64Var(&b.Seed, "seed", 0, "")
	fs.StringVar(&b.Database, "db", "benchdb", "")
	fs.StringVar(&b.Frame, "frame", "testframe", "")
	fs.IntVar(&b.BufferSize, "buffer-size", 10000000, "")

	if err := fs.Parse(args); err != nil {
		return nil, err
	}
	return fs.Args(), nil
}

func (b *Import) Init(hosts []string, agentNum int) error {
	if len(hosts) == 0 {
		return fmt.Errorf("Need at least one host")
	}
	b.Name = "import"
	b.Host = hosts[0]
	// generate csv data
	baseBitmapID, maxBitmapID, baseProfileID, maxProfileID := b.BaseBitmapID, b.MaxBitmapID, b.BaseProfileID, b.MaxProfileID
	switch b.AgentControls {
	case "height":
		numBitmapIDs := (b.MaxBitmapID - b.BaseBitmapID)
		baseBitmapID = b.BaseBitmapID + (numBitmapIDs * int64(agentNum))
		maxBitmapID = baseBitmapID + numBitmapIDs
	case "width":
		numProfileIDs := (b.MaxProfileID - b.BaseProfileID)
		baseProfileID = b.BaseProfileID + (numProfileIDs * int64(agentNum))
		maxProfileID = baseProfileID + numProfileIDs
	case "":
		break
	default:
		return fmt.Errorf("agent-controls: '%v' is not supported", b.AgentControls)
	}
	f, err := ioutil.TempFile("", "")
	if err != nil {
		return err
	}
	// set b.Paths)
	num := GenerateImportCSV(f, baseBitmapID, maxBitmapID, baseProfileID, maxProfileID,
		b.MinBitsPerMap, b.MaxBitsPerMap, b.Seed+int64(agentNum), b.RandomBitmapOrder)
	b.numbits = num
	// set b.Paths
	f.Close()
	b.Paths = []string{f.Name()}
	return nil
}

// Run runs the Import benchmark
func (b *Import) Run(ctx context.Context, agentNum int) map[string]interface{} {
	results := make(map[string]interface{})
	results["numbits"] = b.numbits
	results["db"] = b.Database
	start := time.Now()
	err := b.ImportCommand.Run(ctx)

	if err != nil {
		results["error"] = err.Error()
	}
	results["time"] = time.Now().Sub(start)
	return results
}

type Int64Slice []int64

func (s Int64Slice) Len() int           { return len(s) }
func (s Int64Slice) Less(i, j int) bool { return s[i] < s[j] }
func (s Int64Slice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

func GenerateImportCSV(w io.Writer, baseBitmapID, maxBitmapID, baseProfileID, maxProfileID, minBitsPerMap, maxBitsPerMap, seed int64, randomOrder bool) int {
	src := rand.NewSource(seed)
	rng := rand.New(src)

	var bitmapIDs []int
	if randomOrder {
		bitmapIDs = rng.Perm(int(maxBitmapID - baseBitmapID))
	}
	numrows := 0
	profileIDs := make(Int64Slice, maxBitsPerMap)
	for i := baseBitmapID; i < maxBitmapID; i++ {
		var bitmapID int64
		if randomOrder {
			bitmapID = int64(bitmapIDs[i-baseBitmapID])
		} else {
			bitmapID = int64(i)
		}

		numBitsToSet := rng.Int63n(maxBitsPerMap-minBitsPerMap) + minBitsPerMap
		numrows += int(numBitsToSet)
		for j := int64(0); j < numBitsToSet; j++ {
			profileIDs[j] = rng.Int63n(maxProfileID-baseProfileID) + baseProfileID
		}
		profIDs := profileIDs[:numBitsToSet]
		if !randomOrder {
			sort.Sort(profIDs)
		}
		for j := int64(0); j < numBitsToSet; j++ {
			fmt.Fprintf(w, "%d,%d\n", bitmapID, profIDs[j])
		}

	}
	return numrows
}
<|endoftext|>


<commit_before>package serve

import (
	"fmt"
	"net"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"time"

	"github.com/go-redis/redis/v8"
	"github.com/spf13/pflag"
	"github.com/tarampampam/mikrotik-hosts-parser/v4/internal/pkg/env"
)

type flags struct {
	listen struct {
		ip   string
		port uint16
	}

	resourcesDir string // can be empty
	configPath   string

	cache struct {
		ttl    string
		engine string
	}

	// redisDSN allows to setup redis server using single string. Examples:
	//	redis://<user>:<password>@<host>:<port>/<db_number>
	//	unix://<user>:<password>@</path/to/redis.sock>?db=<db_number>
	redisDSN string
}

func (f *flags) init(flagSet *pflag.FlagSet) {
	exe, _ := os.Executable()
	exe = path.Dir(exe)

	flagSet.StringVarP(
		&f.listen.ip,
		"listen",
		"l",
		"0.0.0.0",
		fmt.Sprintf("IP address to listen on [$%s]", env.ListenAddr),
	)
	flagSet.Uint16VarP(
		&f.listen.port,
		"port",
		"p",
		8080, //nolint:gomnd
		fmt.Sprintf("TCP port number [$%s]", env.ListenPort),
	)
	flagSet.StringVarP(
		&f.resourcesDir,
		"resources-dir",
		"r",
		filepath.Join(exe, "web"),
		fmt.Sprintf("path to the directory with public assets [$%s]", env.ResourcesDir),
	)
	flagSet.StringVarP(
		&f.configPath,
		"config",
		"c",
		filepath.Join(exe, "configs", "config.yml"),
		fmt.Sprintf("config file path [$%s]", env.ConfigPath),
	)
	flagSet.StringVarP(
		&f.cache.engine,
		"caching-engine",
		"",
		cachingEngineMemory,
		fmt.Sprintf("caching engine (%s|%s) [$%s]", cachingEngineMemory, cachingEngineRedis, env.CachingEngine),
	)
	flagSet.StringVarP(
		&f.cache.ttl,
		"cache-ttl",
		"",
		"30m",
		fmt.Sprintf("cache entries lifetime (examples: 50s, 1h30m) [$%s]", env.CacheTTL),
	)
	flagSet.StringVarP(
		&f.redisDSN,
		"redis-dsn",
		"",
		"redis://127.0.0.1:6379/0",
		fmt.Sprintf("redis server DSN (format: \"redis://<user>:<password>@<host>:<port>/<db_number>\") [$%s]", env.RedisDSN), //nolint:lll
	)
}

func (f *flags) overrideUsingEnv() error {
	if envVar, exists := env.ListenAddr.Lookup(); exists {
		f.listen.ip = envVar
	}

	if envVar, exists := env.ListenPort.Lookup(); exists {
		if p, err := strconv.ParseUint(envVar, 10, 16); err == nil { //nolint:gomnd
			f.listen.port = uint16(p)
		} else {
			return fmt.Errorf("wrong TCP port environment variable [%s] value", envVar)
		}
	}

	if envVar, exists := env.ResourcesDir.Lookup(); exists {
		f.resourcesDir = envVar
	}

	if envVar, exists := env.ConfigPath.Lookup(); exists {
		f.configPath = envVar
	}

	if envVar, exists := env.CachingEngine.Lookup(); exists {
		f.cache.engine = envVar
	}

	if envVar, exists := env.CacheTTL.Lookup(); exists {
		f.cache.ttl = envVar
	}

	if envVar, exists := env.RedisDSN.Lookup(); exists {
		f.redisDSN = envVar
	}

	return nil
}

func (f *flags) validate() error {
	if net.ParseIP(f.listen.ip) == nil {
		return fmt.Errorf("wrong IP address [%s] for listening", f.listen.ip)
	}

	if f.resourcesDir != "" {
		if info, err := os.Stat(f.resourcesDir); err != nil || !info.Mode().IsDir() {
			return fmt.Errorf("wrong resources directory [%s] path", f.resourcesDir)
		}
	}

	if info, err := os.Stat(f.configPath); err != nil || !info.Mode().IsRegular() {
		return fmt.Errorf("config file [%s] was not found", f.configPath)
	}

	switch f.cache.engine {
	case cachingEngineMemory:
	case cachingEngineRedis:
		if _, err := redis.ParseURL(f.redisDSN); err != nil {
			return fmt.Errorf("wrong redis DSN [%s]: %w", f.redisDSN, err)
		}
	default:
		return fmt.Errorf("unsupported caching engine: %s", f.cache.engine)
	}

	if _, err := time.ParseDuration(f.cache.ttl); err != nil {
		return fmt.Errorf("wrong cache lifetime [%s] period", f.cache.ttl)
	}

	return nil
}
<commit_msg>wip: fix golangci-lint false-positive issues<commit_after>package serve

import (
	"fmt"
	"net"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"time"

	"github.com/go-redis/redis/v8"
	"github.com/spf13/pflag"

	"github.com/tarampampam/mikrotik-hosts-parser/v4/internal/pkg/env"
)

type flags struct {
	listen struct {
		ip   string
		port uint16
	}

	resourcesDir string // can be empty
	configPath   string

	cache struct {
		ttl    string
		engine string
	}

	// redisDSN allows to setup redis server using single string. Examples:
	//	redis://<user>:<password>@<host>:<port>/<db_number>
	//	unix://<user>:<password>@</path/to/redis.sock>?db=<db_number>
	redisDSN string
}

func (f *flags) init(flagSet *pflag.FlagSet) {
	exe, _ := os.Executable()
	exe = path.Dir(exe)

	flagSet.StringVarP(
		&f.listen.ip,
		"listen",
		"l",
		"0.0.0.0",
		fmt.Sprintf("IP address to listen on [$%s]", env.ListenAddr),
	)
	flagSet.Uint16VarP(
		&f.listen.port,
		"port",
		"p",
		8080, //nolint:gomnd
		fmt.Sprintf("TCP port number [$%s]", env.ListenPort),
	)
	flagSet.StringVarP(
		&f.resourcesDir,
		"resources-dir",
		"r",
		filepath.Join(exe, "web"),
		fmt.Sprintf("path to the directory with public assets [$%s]", env.ResourcesDir),
	)
	flagSet.StringVarP(
		&f.configPath,
		"config",
		"c",
		filepath.Join(exe, "configs", "config.yml"),
		fmt.Sprintf("config file path [$%s]", env.ConfigPath),
	)
	flagSet.StringVarP(
		&f.cache.engine,
		"caching-engine",
		"",
		cachingEngineMemory,
		fmt.Sprintf("caching engine (%s|%s) [$%s]", cachingEngineMemory, cachingEngineRedis, env.CachingEngine),
	)
	flagSet.StringVarP(
		&f.cache.ttl,
		"cache-ttl",
		"",
		"30m",
		fmt.Sprintf("cache entries lifetime (examples: 50s, 1h30m) [$%s]", env.CacheTTL),
	)
	flagSet.StringVarP(
		&f.redisDSN,
		"redis-dsn",
		"",
		"redis://127.0.0.1:6379/0",
		fmt.Sprintf("redis server DSN (format: \"redis://<user>:<password>@<host>:<port>/<db_number>\") [$%s]", env.RedisDSN), //nolint:lll
	)
}

func (f *flags) overrideUsingEnv() error {
	if envVar, exists := env.ListenAddr.Lookup(); exists {
		f.listen.ip = envVar
	}

	if envVar, exists := env.ListenPort.Lookup(); exists {
		if p, err := strconv.ParseUint(envVar, 10, 16); err == nil {
			f.listen.port = uint16(p)
		} else {
			return fmt.Errorf("wrong TCP port environment variable [%s] value", envVar)
		}
	}

	if envVar, exists := env.ResourcesDir.Lookup(); exists {
		f.resourcesDir = envVar
	}

	if envVar, exists := env.ConfigPath.Lookup(); exists {
		f.configPath = envVar
	}

	if envVar, exists := env.CachingEngine.Lookup(); exists {
		f.cache.engine = envVar
	}

	if envVar, exists := env.CacheTTL.Lookup(); exists {
		f.cache.ttl = envVar
	}

	if envVar, exists := env.RedisDSN.Lookup(); exists {
		f.redisDSN = envVar
	}

	return nil
}

func (f *flags) validate() error {
	if net.ParseIP(f.listen.ip) == nil {
		return fmt.Errorf("wrong IP address [%s] for listening", f.listen.ip)
	}

	if f.resourcesDir != "" {
		if info, err := os.Stat(f.resourcesDir); err != nil || !info.Mode().IsDir() {
			return fmt.Errorf("wrong resources directory [%s] path", f.resourcesDir)
		}
	}

	if info, err := os.Stat(f.configPath); err != nil || !info.Mode().IsRegular() {
		return fmt.Errorf("config file [%s] was not found", f.configPath)
	}

	switch f.cache.engine {
	case cachingEngineMemory:
	case cachingEngineRedis:
		if _, err := redis.ParseURL(f.redisDSN); err != nil {
			return fmt.Errorf("wrong redis DSN [%s]: %w", f.redisDSN, err)
		}
	default:
		return fmt.Errorf("unsupported caching engine: %s", f.cache.engine)
	}

	if _, err := time.ParseDuration(f.cache.ttl); err != nil {
		return fmt.Errorf("wrong cache lifetime [%s] period", f.cache.ttl)
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright (C) 2018 Nippon Telegraph and Telephone Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package version

import "fmt"

const MAJOR uint = 3
const MINOR uint = 6
const PATCH uint = 0

var COMMIT string = ""
var IDENTIFIER string = ""
var METADATA string = ""

func Version() string {
	var suffix string = ""
	if len(IDENTIFIER) > 0 {
		suffix = fmt.Sprintf("-%s", IDENTIFIER)
	}

	if len(COMMIT) > 0 || len(METADATA) > 0 {
		suffix = suffix + "+"
	}

	if len(COMMIT) > 0 {
		suffix = fmt.Sprintf("%s"+"commit.%s", suffix, COMMIT)

	}

	if len(METADATA) > 0 {
		if len(COMMIT) > 0 {
			suffix = suffix + "."
		}
		suffix = suffix + METADATA
	}

	return fmt.Sprintf("%d.%d.%d%s", MAJOR, MINOR, PATCH, suffix)
}
<commit_msg>v3.7.0<commit_after>// Copyright (C) 2018 Nippon Telegraph and Telephone Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package version

import "fmt"

const MAJOR uint = 3
const MINOR uint = 7
const PATCH uint = 0

var COMMIT string = ""
var IDENTIFIER string = ""
var METADATA string = ""

func Version() string {
	var suffix string = ""
	if len(IDENTIFIER) > 0 {
		suffix = fmt.Sprintf("-%s", IDENTIFIER)
	}

	if len(COMMIT) > 0 || len(METADATA) > 0 {
		suffix = suffix + "+"
	}

	if len(COMMIT) > 0 {
		suffix = fmt.Sprintf("%s"+"commit.%s", suffix, COMMIT)

	}

	if len(METADATA) > 0 {
		if len(COMMIT) > 0 {
			suffix = suffix + "."
		}
		suffix = suffix + METADATA
	}

	return fmt.Sprintf("%d.%d.%d%s", MAJOR, MINOR, PATCH, suffix)
}
<|endoftext|>