<commit_before>package main

import (
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/Sirupsen/logrus"
)

const (
	cidr = "0.0.0.0/0"

	arinAPIEndpoint = "http://whois.arin.net/rest/ip/%s"
)

var (
	ports = []int{80, 443, 9001, 8001}
)

func main() {
	// On ^C, or SIGTERM handle exit.
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	signal.Notify(c, syscall.SIGTERM)
	go func() {
		for sig := range c {
			logrus.Infof("Received %s, exiting.", sig.String())
			os.Exit(0)
		}
	}()

	// Set the logger to nil so we ignore messages from the Dial that don't matter.
	// See: https://github.com/golang/go/issues/19895#issuecomment-292793756
	log.SetFlags(0)
	log.SetOutput(ioutil.Discard)

	logrus.Infof("Scanning for Kubernetes Dashboards and API Servers on %s over port range %#v", cidr, ports)
	logrus.Infof("This may take a bit...")

	startTime := time.Now()

	ip, ipnet, err := net.ParseCIDR(cidr)
	if err != nil {
		logrus.Fatal(err)
	}

	var wg sync.WaitGroup
	for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); inc(ip) {
		for _, port := range ports {
			wg.Add(1)
			go func(ip string, port int) {
				defer wg.Done()

				scanIP(ip, port)

			}(ip.String(), port)
		}
	}

	wg.Wait()

	since := time.Since(startTime)
	logrus.Infof("Scan took: %s", since.String())
}

func scanIP(ip string, port int) {
	// Check if the port is open.
	ok := portOpen(ip, port)
	if !ok {
		return
	}

	// Check if it's a kubernetes dashboard.
	ok = isKubernetesDashboard(ip, port)
	if !ok {
		return
	}

	fmt.Printf("%s:%d\n", ip, port)
	// Get the info for the ip address.
	info, err := getIPInfo(ip)
	if err != nil {
		logrus.Warnf("ip info err: %v", err)
		return
	}
	fmt.Printf("%s:%d\t%s\t%s\t%s\n",
		ip, port,
		info.Net.Organization.Handle, info.Net.Organization.Name, info.Net.Organization.Reference)
}

func portOpen(ip string, port int) bool {
	c, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", ip, port), 2*time.Second)
	if err != nil {
		// logrus.Warnf("listen at %s:%s failed: %v", ip, port, err)
		return false
	}
	defer c.Close()

	return true
}

func isKubernetesDashboard(ip string, port int) bool {
	client := &http.Client{
		Timeout: time.Second * 10,
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
		},
	}

	tryAddrs := []string{
		fmt.Sprintf("http://%s:%d", ip, port),
		fmt.Sprintf("https://%s:%d", ip, port),
		fmt.Sprintf("http://%s:%d/api/", ip, port),
		fmt.Sprintf("https://%s:%d/api/", ip, port),
	}

	var (
		resp *http.Response
		err  = errors.New("not yet run")
		uri  string
	)

	for i := 0; i < len(tryAddrs) && err != nil; i++ {
		uri = tryAddrs[i]
		resp, err = client.Get(uri)
	}
	if err != nil {
		//logrus.Warnf("getting %s:%s failed: %v", ip, port, err)
		return false
	}
	defer resp.Body.Close()

	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return false
	}

	body := strings.ToLower(string(b))
	if (strings.Contains(body, "kubernetes") && strings.Contains(body, "dashboard")) ||
		(strings.Contains(body, `"versions"`) && strings.Contains(body, `"serverAddress`)) ||
		(strings.Contains(body, `"paths"`) && strings.Contains(body, `"/api"`)) {
		logrus.Infof("uri: %s", uri)
		return true
	}

	return false
}

type ARINResponse struct {
	Net NetJSON `json:"net,omitempty"`
}

type NetJSON struct {
	Organization OrganizationJSON `json:"orgRef,omitempty"`
}

type OrganizationJSON struct {
	Handle    string `json:"@handle,omitempty"`
	Name      string `json:"@name,omitempty"`
	Reference string `json:"$,omitempty"`
}

func getIPInfo(ip string) (b ARINResponse, err error) {
	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(arinAPIEndpoint, ip), nil)
	if err != nil {
		return b, err
	}
	req.Header.Set("Accept", "application/json")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return b, err
	}
	defer resp.Body.Close()

	if err := json.NewDecoder(resp.Body).Decode(&b); err != nil {
		return b, err
	}

	return b, nil
}

func inc(ip net.IP) {
	for j := len(ip) - 1; j >= 0; j-- {
		ip[j]++
		if ip[j] > 0 {
			break
		}
	}
}
<commit_msg>add flags<commit_after>package main

import (
	"crypto/tls"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/Sirupsen/logrus"
	mailgun "github.com/mailgun/mailgun-go"
)

const (
	defaultCIDR = "0.0.0.0/0"

	arinAPIEndpoint = "http://whois.arin.net/rest/ip/%s"

	emailSender = "k8scan@jessfraz.com"
)

var (
	timeoutPing time.Duration
	timeoutGet  time.Duration

	cidr string

	ports = []int{80, 443, 9001, 8001}

	mailgunDomain  string
	mailgunAPIKey  string
	emailRecipient string

	debug bool
)

func init() {
	flag.DurationVar(&timeoutPing, "timeout-ping", 2*time.Second, "Timeout for checking that the port is open")
	flag.DurationVar(&timeoutGet, "timeout-get", 10*time.Second, "Timeout for getting the contents of the URL")

	flag.StringVar(&cidr, "cidr", defaultCIDR, "IP CIDR to scan")

	flag.StringVar(&mailgunAPIKey, "mailgun-api-key", "", "Mailgun API Key to use for sending email (optional)")
	flag.StringVar(&mailgunDomain, "mailgun-domain", "", "Mailgun Domain to use for sending email (optional)")
	flag.StringVar(&emailRecipient, "email-recipient", "", "Recipient for email notifications (optional)")

	flag.BoolVar(&debug, "d", false, "Run in debug mode")

	flag.Usage = func() {
		flag.PrintDefaults()
	}

	flag.Parse()

	// set log level
	if debug {
		logrus.SetLevel(logrus.DebugLevel)
	}
}

func main() {
	// On ^C, or SIGTERM handle exit.
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	signal.Notify(c, syscall.SIGTERM)
	go func() {
		for sig := range c {
			logrus.Infof("Received %s, exiting.", sig.String())
			os.Exit(0)
		}
	}()

	// Set the logger to nil so we ignore messages from the Dial that don't matter.
	// See: https://github.com/golang/go/issues/19895#issuecomment-292793756
	log.SetFlags(0)
	log.SetOutput(ioutil.Discard)

	logrus.Infof("Scanning for Kubernetes Dashboards and API Servers on %s over port range %#v", cidr, ports)
	if len(mailgunDomain) > 0 && len(mailgunAPIKey) > 0 && len(emailRecipient) > 0 {
		logrus.Infof("Using Mailgun Domain %s, API Key %s to send emails to %s", mailgunDomain, mailgunAPIKey, emailRecipient)
	}
	logrus.Infof("This may take a bit...")

	startTime := time.Now()

	ip, ipnet, err := net.ParseCIDR(cidr)
	if err != nil {
		logrus.Fatal(err)
	}

	var wg sync.WaitGroup
	for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); inc(ip) {
		for _, port := range ports {
			wg.Add(1)
			go func(ip string, port int) {
				defer wg.Done()

				scanIP(ip, port)

			}(ip.String(), port)
		}
	}

	wg.Wait()

	since := time.Since(startTime)
	logrus.Infof("Scan took: %s", since.String())
}

func scanIP(ip string, port int) {
	// Check if the port is open.
	ok := portOpen(ip, port)
	if !ok {
		return
	}

	// Check if it's a kubernetes dashboard.
	ok, uri := isKubernetesDashboard(ip, port)
	if !ok {
		return
	}

	// Get the info for the ip address.
	info, err := getIPInfo(ip)
	if err != nil {
		logrus.Warnf("ip info err: %v", err)
	}

	fmt.Printf("%s:%d\t%s\t%s\t%s\n",
		ip, port,
		info.Net.Organization.Handle, info.Net.Organization.Name, info.Net.Organization.Reference)

	// send an email
	if len(mailgunDomain) > 0 && len(mailgunAPIKey) > 0 && len(emailRecipient) > 0 {
		if err := sendEmail(uri, ip, port, info); err != nil {
			logrus.Warn(err)
		}
	}
}

func portOpen(ip string, port int) bool {
	c, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", ip, port), timeoutPing)
	if err != nil {
		// logrus.Warnf("listen at %s:%s failed: %v", ip, port, err)
		return false
	}
	defer c.Close()

	return true
}

func isKubernetesDashboard(ip string, port int) (bool, string) {
	client := &http.Client{
		Timeout: timeoutGet,
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
		},
	}

	tryAddrs := []string{
		fmt.Sprintf("http://%s:%d", ip, port),
		fmt.Sprintf("https://%s:%d", ip, port),
		fmt.Sprintf("http://%s:%d/api/", ip, port),
		fmt.Sprintf("https://%s:%d/api/", ip, port),
	}

	var (
		resp *http.Response
		err  = errors.New("not yet run")
		uri  string
	)

	for i := 0; i < len(tryAddrs) && err != nil; i++ {
		uri = tryAddrs[i]
		resp, err = client.Get(uri)
	}
	if err != nil {
		//logrus.Warnf("getting %s:%s failed: %v", ip, port, err)
		return false, ""
	}
	defer resp.Body.Close()

	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return false, ""
	}

	body := strings.ToLower(string(b))
	if (strings.Contains(body, "kubernetes") && strings.Contains(body, "dashboard")) ||
		(strings.Contains(body, `"versions"`) && strings.Contains(body, `"serverAddress`)) ||
		(strings.Contains(body, `"paths"`) && strings.Contains(body, `"/api"`)) {
		return true, uri
	}

	return false, ""
}

type ARINResponse struct {
	Net NetJSON `json:"net,omitempty"`
}

type NetJSON struct {
	Organization OrganizationJSON `json:"orgRef,omitempty"`
}

type OrganizationJSON struct {
	Handle    string `json:"@handle,omitempty"`
	Name      string `json:"@name,omitempty"`
	Reference string `json:"$,omitempty"`
}

func getIPInfo(ip string) (b ARINResponse, err error) {
	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(arinAPIEndpoint, ip), nil)
	if err != nil {
		return b, err
	}
	req.Header.Set("Accept", "application/json")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return b, err
	}
	defer resp.Body.Close()

	if err := json.NewDecoder(resp.Body).Decode(&b); err != nil {
		return b, err
	}

	return b, nil
}

func inc(ip net.IP) {
	for j := len(ip) - 1; j >= 0; j-- {
		ip[j]++
		if ip[j] > 0 {
			break
		}
	}
}

func sendEmail(uri, ip string, port int, arinInfo ARINResponse) error {
	mailgunClient := mailgun.NewMailgun(mailgunDomain, mailgunAPIKey, "")

	msg, _, err := mailgunClient.Send(mailgunClient.NewMessage(
		/* From */ fmt.Sprintf("%s <%s>", emailSender, emailSender),
		/* Subject */ fmt.Sprintf("[k8scan]: found dashboard %s", uri),
		/* Body */ fmt.Sprintf(`Time: %s

IP: %s:%d
URL: %s

ARIN: %s
	  %s
	  %s
`,
			time.Now().Format(time.UnixDate),
			arinInfo.Net.Organization.Handle,
			arinInfo.Net.Organization.Name,
			arinInfo.Net.Organization.Reference,
		),
		/* To */ emailRecipient,
	))
	if err != nil {
		return fmt.Errorf("sending Mailgun message failed: response: %#v error: %v", msg, err)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"os"
	"time"

	"github.com/kaaryasthan/kaaryasthan/auth"
	_ "github.com/kaaryasthan/kaaryasthan/auth/google"
	_ "github.com/kaaryasthan/kaaryasthan/comment"
	"github.com/kaaryasthan/kaaryasthan/config"
	"github.com/kaaryasthan/kaaryasthan/db"
	_ "github.com/kaaryasthan/kaaryasthan/item"
	"github.com/kaaryasthan/kaaryasthan/middleware"
	_ "github.com/kaaryasthan/kaaryasthan/project"
	"github.com/kaaryasthan/kaaryasthan/route"
	_ "github.com/kaaryasthan/kaaryasthan/web"
	"github.com/urfave/negroni"
)

//go:generate go-bindata -pkg db -o db/bindata.go -nocompress db/migrations/

var migrate = flag.Bool("migrate", false, "perform db migrations")

func init() {
	flag.Parse()
	go func() {
		time.Sleep(5 * time.Second)
		err := db.DB.Ping()
		if err != nil {
			log.Fatal(err.Error())
		}
		if *migrate {
			err = db.SchemaMigrate()
			if err != nil {
				log.Fatal(err.Error())
			}
			log.Println("Migration completed. Program is exiting.")
			os.Exit(0)
		}
	}()
}

func main() {
	route.URT.PathPrefix("/api").Handler(
		negroni.New(negroni.HandlerFunc(auth.JwtMiddleware.HandlerWithNext), negroni.Wrap(route.RT)))
	middleware.Run(config.Config.HTTPAddress)
}
<commit_msg>better exit strategy<commit_after>package main

import (
	"flag"
	"log"
	"os"
	"time"

	"github.com/kaaryasthan/kaaryasthan/auth"
	_ "github.com/kaaryasthan/kaaryasthan/auth/google"
	_ "github.com/kaaryasthan/kaaryasthan/comment"
	"github.com/kaaryasthan/kaaryasthan/config"
	"github.com/kaaryasthan/kaaryasthan/db"
	_ "github.com/kaaryasthan/kaaryasthan/item"
	"github.com/kaaryasthan/kaaryasthan/middleware"
	_ "github.com/kaaryasthan/kaaryasthan/project"
	"github.com/kaaryasthan/kaaryasthan/route"
	_ "github.com/kaaryasthan/kaaryasthan/web"
	"github.com/urfave/negroni"
)

//go:generate go-bindata -pkg db -o db/bindata.go -nocompress db/migrations/

var migrate = flag.Bool("migrate", false, "perform db migrations")

// Exit code for clean exit
type Exit struct {
	Code int
}

// exit code handler
func handleExit() {
	if e := recover(); e != nil {
		if exit, ok := e.(Exit); ok == true {
			os.Exit(exit.Code)
		}
		panic(e) // not an Exit, bubble up
	}
}

func init() {
	flag.Parse()
	go func() {
		defer handleExit()
		defer db.DB.Close()
		time.Sleep(5 * time.Second)
		err := db.DB.Ping()
		if err != nil {
			log.Fatal(err.Error())
		}
		if *migrate {
			err = db.SchemaMigrate()
			if err != nil {
				log.Println(err.Error())
				panic(Exit{4})
			}
			log.Println("Migration completed. Program is exiting.")
			panic(Exit{3})
		}
	}()
}

func main() {
	route.URT.PathPrefix("/api").Handler(
		negroni.New(negroni.HandlerFunc(auth.JwtMiddleware.HandlerWithNext), negroni.Wrap(route.RT)))
	middleware.Run(config.Config.HTTPAddress)
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package whiteblacklist

import (
	"regexp"
	"strings"

	"github.com/pkg/errors"
)

// WhiteBlackList encapsulates the logic needed to filter based on a string.
type WhiteBlackList struct {
	list        map[string]struct{}
	rList       []regexp.Regexp
	isWhiteList bool
}

// New constructs a new WhiteBlackList based on a white- and a
// blacklist. Only one of them can be not empty.
func New(w, b map[string]struct{}) (*WhiteBlackList, error) {
	if len(w) != 0 && len(b) != 0 {
		return nil, errors.New(
			"whitelist and blacklist are both set, they are mutually exclusive, only one of them can be set",
		)
	}

	white := copyList(w)
	black := copyList(b)

	var list map[string]struct{}
	var isWhiteList bool

	// Default to blacklisting
	if len(white) != 0 {
		list = white
		isWhiteList = true
	} else {
		list = black
		isWhiteList = false
	}

	return &WhiteBlackList{
		list:        list,
		isWhiteList: isWhiteList,
	}, nil
}

// Parse parses and compiles all of the regexes in the whiteBlackList.
func (l *WhiteBlackList) Parse() error {
	var regexes []regexp.Regexp
	for item := range l.list {
		r, err := regexp.Compile(item)
		if err != nil {
			return err
		}
		regexes = append(regexes, *r)
	}
	l.rList = regexes
	return nil
}

// Include includes the given items in the list.
func (l *WhiteBlackList) Include(items []string) {
	if l.isWhiteList {
		for _, item := range items {
			l.list[item] = struct{}{}
		}
	} else {
		for _, item := range items {
			delete(l.list, item)
		}
	}
}

// Exclude excludes the given items from the list.
func (l *WhiteBlackList) Exclude(items []string) {
	if l.isWhiteList {
		for _, item := range items {
			delete(l.list, item)
		}
	} else {
		for _, item := range items {
			l.list[item] = struct{}{}
		}
	}
}

// IsIncluded returns if the given item is included.
func (l *WhiteBlackList) IsIncluded(item string) bool {
	var matched bool
	for _, r := range l.rList {
		matched = r.MatchString(item)
		if matched {
			break
		}
	}

	if l.isWhiteList {
		return matched
	}

	return !matched
}

// IsExcluded returns if the given item is excluded.
func (l *WhiteBlackList) IsExcluded(item string) bool {
	return !l.IsIncluded(item)
}

// Status returns the status of the WhiteBlackList that can e.g. be passed into
// a logger.
func (l *WhiteBlackList) Status() string {
	items := []string{}
	for key := range l.list {
		items = append(items, key)
	}

	if l.isWhiteList {
		return "whitelisting the following items: " + strings.Join(items, ", ")
	}

	return "blacklisting the following items: " + strings.Join(items, ", ")
}

func copyList(l map[string]struct{}) map[string]struct{} {
	newList := map[string]struct{}{}
	for k, v := range l {
		newList[k] = v
	}
	return newList
}
<commit_msg>fix go vet issues in whitblacklist.go<commit_after>/*
Copyright 2018 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package whiteblacklist

import (
	"regexp"
	"strings"

	"github.com/pkg/errors"
)

// WhiteBlackList encapsulates the logic needed to filter based on a string.
type WhiteBlackList struct {
	list        map[string]struct{}
	rList       []*regexp.Regexp
	isWhiteList bool
}

// New constructs a new WhiteBlackList based on a white- and a
// blacklist. Only one of them can be not empty.
func New(w, b map[string]struct{}) (*WhiteBlackList, error) {
	if len(w) != 0 && len(b) != 0 {
		return nil, errors.New(
			"whitelist and blacklist are both set, they are mutually exclusive, only one of them can be set",
		)
	}

	white := copyList(w)
	black := copyList(b)

	var list map[string]struct{}
	var isWhiteList bool

	// Default to blacklisting
	if len(white) != 0 {
		list = white
		isWhiteList = true
	} else {
		list = black
		isWhiteList = false
	}

	return &WhiteBlackList{
		list:        list,
		isWhiteList: isWhiteList,
	}, nil
}

// Parse parses and compiles all of the regexes in the whiteBlackList.
func (l *WhiteBlackList) Parse() error {
	var regexes []*regexp.Regexp
	for item := range l.list {
		r, err := regexp.Compile(item)
		if err != nil {
			return err
		}
		regexes = append(regexes, r)
	}
	l.rList = regexes
	return nil
}

// Include includes the given items in the list.
func (l *WhiteBlackList) Include(items []string) {
	if l.isWhiteList {
		for _, item := range items {
			l.list[item] = struct{}{}
		}
	} else {
		for _, item := range items {
			delete(l.list, item)
		}
	}
}

// Exclude excludes the given items from the list.
func (l *WhiteBlackList) Exclude(items []string) {
	if l.isWhiteList {
		for _, item := range items {
			delete(l.list, item)
		}
	} else {
		for _, item := range items {
			l.list[item] = struct{}{}
		}
	}
}

// IsIncluded returns if the given item is included.
func (l *WhiteBlackList) IsIncluded(item string) bool {
	var matched bool
	for _, r := range l.rList {
		matched = r.MatchString(item)
		if matched {
			break
		}
	}

	if l.isWhiteList {
		return matched
	}

	return !matched
}

// IsExcluded returns if the given item is excluded.
func (l *WhiteBlackList) IsExcluded(item string) bool {
	return !l.IsIncluded(item)
}

// Status returns the status of the WhiteBlackList that can e.g. be passed into
// a logger.
func (l *WhiteBlackList) Status() string {
	items := []string{}
	for key := range l.list {
		items = append(items, key)
	}

	if l.isWhiteList {
		return "whitelisting the following items: " + strings.Join(items, ", ")
	}

	return "blacklisting the following items: " + strings.Join(items, ", ")
}

func copyList(l map[string]struct{}) map[string]struct{} {
	newList := map[string]struct{}{}
	for k, v := range l {
		newList[k] = v
	}
	return newList
}
<|endoftext|>


<commit_before>package kafka

type Producer interface {
	Close()
	SendBytes(message []byte, topic, key string)
}

type ConsumerEvent struct {
	Err       error
	Message   []byte
	Offset    int64
	Partition int32
}

type ConsumerChannel interface {
	Events() <-chan *ConsumerEvent
	Close() error
}

type PartitionIndex int32
type Offset int64

type Sender interface {
	SendBytes(message []byte, topic, key string)
}

// DividePartitions - splits the number of partitions in to buckets numbering the splitBy
func DividePartitions(partitions, splitBy int) [][]int {
	buckets := make([][]int, splitBy, splitBy)

	if splitBy > partitions {
		splitBy = partitions
	}

	count := partitions / splitBy
	extra := partitions % splitBy

	for j := 0; j < splitBy; j++ {
		index := 0 + j
		bucketSize := count
		if j < extra {
			bucketSize += 1
		}

		bucket := make([]int, bucketSize, bucketSize)
		buckets[j] = bucket

		for i := 0; i < bucketSize; i++ {
			bucket[i] = index
			index += splitBy
		}
	}
	return buckets
}
<commit_msg>added error back to signature<commit_after>package kafka

type Producer interface {
	Close()
	SendBytes(message []byte, topic, key string) error
}

type ConsumerEvent struct {
	Err       error
	Message   []byte
	Offset    int64
	Partition int32
}

type ConsumerChannel interface {
	Events() <-chan *ConsumerEvent
	Close() error
}

type PartitionIndex int32
type Offset int64

type Sender interface {
	SendBytes(message []byte, topic, key string) error
}

// DividePartitions - splits the number of partitions in to buckets numbering the splitBy
func DividePartitions(partitions, splitBy int) [][]int {
	buckets := make([][]int, splitBy, splitBy)

	if splitBy > partitions {
		splitBy = partitions
	}

	count := partitions / splitBy
	extra := partitions % splitBy

	for j := 0; j < splitBy; j++ {
		index := 0 + j
		bucketSize := count
		if j < extra {
			bucketSize += 1
		}

		bucket := make([]int, bucketSize, bucketSize)
		buckets[j] = bucket

		for i := 0; i < bucketSize; i++ {
			bucket[i] = index
			index += splitBy
		}
	}
	return buckets
}
<|endoftext|>


<commit_before>// Copyright (c) Ilia Kravets, 2015. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package anal

import (
	"my/errs"
	"sort"

	"my/itto/verify/packet/itto"
	"my/itto/verify/sim"
)

type bookStat struct {
	maxLevels int
}

type Analyzer struct {
	observer  observer
	bookStats map[uint64]*bookStat
}

func NewAnalyzer() *Analyzer {
	a := &Analyzer{
		bookStats: make(map[uint64]*bookStat),
	}
	a.observer.analyzer = a
	return a
}

type observer struct {
	analyzer *Analyzer
}

var _ sim.Observer = &observer{}

func (*observer) MessageArrived(*sim.IttoDbMessage)            {}
func (*observer) OperationAppliedToOrders(sim.IttoOperation)   {}
func (*observer) BeforeBookUpdate(sim.Book, sim.IttoOperation) {}
func (o *observer) AfterBookUpdate(book sim.Book, op sim.IttoOperation) {
	oid := op.GetOptionId()
	if oid.Invalid() {
		return
	}
	bs := o.analyzer.book(oid, op.GetSide())
	//bookSize := len(book.GetTop(oid, op.GetSide(), 0))
	b := book.GetTop(oid, op.GetSide(), 0)
	bookSize := len(b)
	if bs.maxLevels < bookSize {
		//log.Printf("%d %s %d: %v\n", oid, op.GetSide(), bookSize, b)
		bs.maxLevels = bookSize
	}
}

func (a *Analyzer) Observer() sim.Observer {
	return &a.observer
}
func (a *Analyzer) book(oid itto.OptionId, side itto.MarketSide) (bs *bookStat) {
	errs.Check(side == itto.MarketSideBid || side == itto.MarketSideAsk)
	key := uint64(oid) | uint64(side)<<32
	var ok bool
	if bs, ok = a.bookStats[key]; !ok {
		bs = &bookStat{}
		a.bookStats[key] = bs
	}
	return
}

type OptionSide struct {
	Oid  itto.OptionId
	Side itto.MarketSide
}
type BSVal struct {
	Levels       int
	OptionNumber int
	Sample       []OptionSide
}
type BSHist []BSVal

func (a *Analyzer) BookSizeHist() BSHist {
	bsv := make(map[int]BSVal)
	var levels []int
	for k, bs := range a.bookStats {
		v := bsv[bs.maxLevels]
		v.OptionNumber++
		if len(v.Sample) < 10 {
			v.Sample = append(v.Sample, OptionSide{
				Oid:  itto.OptionId(k),
				Side: itto.MarketSide(k >> 32),
			})
		}
		if v.OptionNumber == 1 {
			v.Levels = bs.maxLevels
			levels = append(levels, v.Levels)
		}
		bsv[v.Levels] = v
	}
	sort.Ints(levels)

	var bsh BSHist
	for _, l := range levels {
		bsh = append(bsh, bsv[l])
	}
	return bsh
}
<commit_msg>anal: add order-id hash collisions histogram<commit_after>// Copyright (c) Ilia Kravets, 2015. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package anal

import (
	"sort"

	"my/errs"

	"my/itto/verify/packet/itto"
	"my/itto/verify/sim"
)

type bookStat struct {
	maxLevels int
}

type HashFunc func(uint64) uint64

type orderHashStat struct {
	f             HashFunc
	bucketSize    map[uint64]int
	maxBucketSize map[uint64]int
}
type Analyzer struct {
	observer      observer
	bookStats     map[uint64]*bookStat
	orderHashStat []orderHashStat
}

func NewAnalyzer() *Analyzer {
	a := &Analyzer{
		bookStats: make(map[uint64]*bookStat),
	}
	a.observer.analyzer = a
	return a
}

type observer struct {
	analyzer *Analyzer
}

var _ sim.Observer = &observer{}

func (*observer) MessageArrived(*sim.IttoDbMessage) {}
func (o *observer) OperationAppliedToOrders(op sim.IttoOperation) {
	sess := op.GetMessage().Session.Index()
	errs.Check(sess < 4, sess)
	var key uint64
	var delta int
	switch o := op.(type) {
	case *sim.OperationAdd:
		key = uint64(sess)<<32 | uint64(o.RefNumD.Delta())
		delta = 1
	case *sim.OperationRemove:
		key = uint64(sess)<<32 | uint64(o.GetOrigRef().Delta())
		delta = -1
	}
	for _, ohs := range o.analyzer.orderHashStat {
		keyHash := ohs.f(key)
		ohs.bucketSize[keyHash] += delta
		if ohs.bucketSize[keyHash] > ohs.maxBucketSize[keyHash] {
			ohs.maxBucketSize[keyHash] = ohs.bucketSize[keyHash]
		} else if ohs.bucketSize[keyHash] == 0 {
			delete(ohs.bucketSize, keyHash)
		}
	}
}
func (*observer) BeforeBookUpdate(sim.Book, sim.IttoOperation) {}
func (o *observer) AfterBookUpdate(book sim.Book, op sim.IttoOperation) {
	oid := op.GetOptionId()
	if oid.Invalid() {
		return
	}
	bs := o.analyzer.book(oid, op.GetSide())
	//bookSize := len(book.GetTop(oid, op.GetSide(), 0))
	b := book.GetTop(oid, op.GetSide(), 0)
	bookSize := len(b)
	if bs.maxLevels < bookSize {
		//log.Printf("%d %s %d: %v\n", oid, op.GetSide(), bookSize, b)
		bs.maxLevels = bookSize
	}
}

func (a *Analyzer) Observer() sim.Observer {
	return &a.observer
}
func (a *Analyzer) AddOrderHashFunction(f HashFunc) {
	ohs := orderHashStat{
		f:             f,
		bucketSize:    make(map[uint64]int),
		maxBucketSize: make(map[uint64]int),
	}
	a.orderHashStat = append(a.orderHashStat, ohs)
}

func (a *Analyzer) book(oid itto.OptionId, side itto.MarketSide) (bs *bookStat) {
	errs.Check(side == itto.MarketSideBid || side == itto.MarketSideAsk)
	key := uint64(oid) | uint64(side)<<32
	var ok bool
	if bs, ok = a.bookStats[key]; !ok {
		bs = &bookStat{}
		a.bookStats[key] = bs
	}
	return
}

type OptionSide struct {
	Oid  itto.OptionId
	Side itto.MarketSide
}
type BSVal struct {
	Levels       int
	OptionNumber int
	Sample       []OptionSide
}
type BSHist []BSVal

func (a *Analyzer) BookSizeHist() BSHist {
	bsv := make(map[int]BSVal)
	var levels []int
	for k, bs := range a.bookStats {
		v := bsv[bs.maxLevels]
		v.OptionNumber++
		if len(v.Sample) < 10 {
			v.Sample = append(v.Sample, OptionSide{
				Oid:  itto.OptionId(k),
				Side: itto.MarketSide(k >> 32),
			})
		}
		if v.OptionNumber == 1 {
			v.Levels = bs.maxLevels
			levels = append(levels, v.Levels)
		}
		bsv[v.Levels] = v
	}
	sort.Ints(levels)

	var bsh BSHist
	for _, l := range levels {
		bsh = append(bsh, bsv[l])
	}
	return bsh
}

type HistVal struct {
	Bin   int
	Count int
}
type Hist []HistVal

func (a *Analyzer) OrdersHashCollisionHist() []Hist {
	var hists []Hist
	for _, ohs := range a.orderHashStat {
		collisionHist := make(map[int]int)
		for _, c := range ohs.maxBucketSize {
			collisionHist[c]++
		}
		var chKeys []int
		for k, _ := range collisionHist {
			chKeys = append(chKeys, k)
		}
		sort.Ints(chKeys)
		var hist Hist
		for _, k := range chKeys {
			hist = append(hist, HistVal{k, collisionHist[k]})
		}
		hists = append(hists, hist)
	}
	return hists
}
<|endoftext|>


<commit_before>package ddoc

import (
	"bytes"
	"errors"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"text/template"
)

type DesignDoc struct {
	ID  string `json:"_id"`
	Rev string `json:"_rev,omitempty"`

	Language string                 `json:"language"`
	Views    map[string]interface{} `json:"views"`
}

// type View is used to fill the design doc template
// with javascript files
// it is not marshaled to or from JSON itself
type View struct {
	Name           string
	MapFunction    string
	ReduceFunction string
}

// views is passed to the template
var views []View

// BuildDesignDocument takes a design document name (do not include _design/)
// and a file path where javascript views can be found; these views should be
// map.js (TODO: reduce.js) files under subdirectories named after the view
//
// Example:
// path/views/customers/map.js
//
// From these, it composes a CouchDB design document having the map and reduce
// functions as named views
func Build(name, path string) ([]byte, error) {

	// Error checking
	if strings.ContainsRune(name, '/') {
		return nil, errors.New("Do not include / in design document name")
	} else if strings.Contains(name, "_design") {
		return nil, errors.New("Do not begin design document name with _design")
	}

	ddocTemplate := filepath.Join(path, "ddoc.tmpl")
	if _, err := os.Stat(ddocTemplate); os.IsNotExist(err) {
		// path/ddoc.tmpl does not exist
		return nil, err
	}

	ddocID := "_design/" + name
	//ddoc := &DesignDoc{ID: ddocID}
	_ = &DesignDoc{ID: ddocID}

	// Glob files
	mapFiles := filepath.Join(path, "views/*/", "map.js")

	matches, err := filepath.Glob(mapFiles)
	if err != nil {
		// only if malformed pattern
		log.Fatalln(err)
	}
	if len(matches) < 1 {
		log.Fatalln("No javascript views found in " + path)
	}

	// Build views
	for i := 0; i < len(matches); i++ {
		v := View{}
		// take the last directory from the path by removing filename (.Dir)
		// and then taking the last element (.Base)
		v.Name = filepath.Base(filepath.Dir(matches[i]))

		// read map.js
		filebuf, err := ioutil.ReadFile(matches[i])
		if err != nil {
			return nil, err
		}

		mapFunc := string(filebuf)
		mapFunc = strings.Replace(mapFunc, "\n", "\\n", -1) // replace newline with literal \n
		mapFunc = strings.Replace(mapFunc, "\r", "", -1)    // replace carriage ret with nothing

		v.MapFunction = string(mapFunc)
		v.ReduceFunction = "_sum"

		// Append to 'views' slice
		views = append(views, v)
	}

	// Fill template
	t, err := template.ParseFiles(ddocTemplate)
	if err != nil {
		log.Fatalln(err)
	}
	// print results
	buf := make([]byte, 0, 8192) // byte slice with capacity of 8k
	b := bytes.NewBuffer(buf)
	t.Execute(b, views)

	return b.Bytes(), nil
}
<commit_msg>Filter out tabs (invalid JSON)<commit_after>package ddoc

import (
	"bytes"
	"errors"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"text/template"
)

type DesignDoc struct {
	ID  string `json:"_id"`
	Rev string `json:"_rev,omitempty"`

	Language string                 `json:"language"`
	Views    map[string]interface{} `json:"views"`
}

// type View is used to fill the design doc template
// with javascript files
// it is not marshaled to or from JSON itself
type View struct {
	Name           string
	MapFunction    string
	ReduceFunction string
}

// views is passed to the template
var views []View

// BuildDesignDocument takes a design document name (do not include _design/)
// and a file path where javascript views can be found; these views should be
// map.js (TODO: reduce.js) files under subdirectories named after the view
//
// Example:
// path/views/customers/map.js
//
// From these, it composes a CouchDB design document having the map and reduce
// functions as named views
func Build(name, path string) ([]byte, error) {

	// Error checking
	if strings.ContainsRune(name, '/') {
		return nil, errors.New("Do not include / in design document name")
	} else if strings.Contains(name, "_design") {
		return nil, errors.New("Do not begin design document name with _design")
	}

	ddocTemplate := filepath.Join(path, "ddoc.tmpl")
	if _, err := os.Stat(ddocTemplate); os.IsNotExist(err) {
		// path/ddoc.tmpl does not exist
		return nil, err
	}

	ddocID := "_design/" + name
	//ddoc := &DesignDoc{ID: ddocID}
	_ = &DesignDoc{ID: ddocID}

	// Glob files
	mapFiles := filepath.Join(path, "views/*/", "map.js")

	matches, err := filepath.Glob(mapFiles)
	if err != nil {
		// only if malformed pattern
		log.Fatalln(err)
	}
	if len(matches) < 1 {
		log.Fatalln("No javascript views found in " + path)
	}

	// Build views
	for i := 0; i < len(matches); i++ {
		v := View{}
		// take the last directory from the path by removing filename (.Dir)
		// and then taking the last element (.Base)
		v.Name = filepath.Base(filepath.Dir(matches[i]))

		// read map.js
		filebuf, err := ioutil.ReadFile(matches[i])
		if err != nil {
			return nil, err
		}

		mapFunc := string(filebuf)
		mapFunc = strings.Replace(mapFunc, "\n", "\\n", -1)  // replace newline with literal \n
		mapFunc = strings.Replace(mapFunc, "\r", "", -1)     // replace carriage ret with nothing
		mapFunc = strings.Replace(mapFunc, "\t", "    ", -1) // JSON spec says no control characters
		// and in fact json.Unmarshal will barf on tab

		v.MapFunction = string(mapFunc)
		v.ReduceFunction = "_sum"

		// Append to 'views' slice
		views = append(views, v)
	}

	// Fill template
	t, err := template.ParseFiles(ddocTemplate)
	if err != nil {
		log.Fatalln(err)
	}
	// print results
	buf := make([]byte, 0, 8192) // byte slice with capacity of 8k
	b := bytes.NewBuffer(buf)
	t.Execute(b, views)

	return b.Bytes(), nil
}
<|endoftext|>


<commit_before>package cvmfs

import (
	"fmt"
	"strings"

	exec "github.com/cvmfs/ducc/exec"
	log "github.com/sirupsen/logrus"
)

type TransactionOption interface {
	ToString() string
}

type TemplateTransaction struct {
	source      string
	destination string
}

func (t TemplateTransaction) ToString() string {
	return fmt.Sprintf("-T %s=%s", t.source, t.destination)
}

func OpenTransaction(CVMFSRepo string, options ...TransactionOption) error {
	cmd := []string{"cvmfs_server", "transaction"}
	for _, opt := range options {
		cmd = append(cmd, opt.ToString())
	}
	cmd = append(cmd, CVMFSRepo)
	err := exec.ExecCommand("cvmfs_server", "transaction", CVMFSRepo).Start()
	if err != nil {
		LogE(err).WithFields(
			log.Fields{"repo": CVMFSRepo}).
			Error("Error in opening the transaction")
		abort(CVMFSRepo)
	}
	return err
}

func Publish(CVMFSRepo string) error {
	err := exec.ExecCommand("cvmfs_server", "publish", CVMFSRepo).Start()
	if err != nil {
		LogE(err).WithFields(
			log.Fields{"repo": CVMFSRepo}).
			Error("Error in publishing the repository")
		abort(CVMFSRepo)
	}
	return err
}

func Abort(CVMFSRepo string) error {
	err := abort(CVMFSRepo)
	if err != nil {
		LogE(err).WithFields(
			log.Fields{"repo": CVMFSRepo}).
			Error("Error in abort the transaction")
	}
	return err
}

func abort(CVMFSRepo string) error {
	return exec.ExecCommand("cvmfs_server", "abort", "-f", CVMFSRepo).Start()
}

func RepositoryExists(CVMFSRepo string) bool {
	cmd := exec.ExecCommand("cvmfs_server", "list")
	err, stdout, _ := cmd.StartWithOutput()
	if err != nil {
		LogE(fmt.Errorf("Error in listing the repository")).
			Error("Repo not present")
		return false
	}
	stdoutString := string(stdout.Bytes())

	if strings.Contains(stdoutString, CVMFSRepo) {
		return true
	} else {
		return false
	}
}

func WithinTransaction(CVMFSRepo string, f func() error, opts ...TransactionOption) error {
	err := OpenTransaction(CVMFSRepo, opts...)
	if err != nil {
		return err
	}
	err = f()
	if err != nil {
		return Abort(CVMFSRepo)
	}
	return Publish(CVMFSRepo)
}
<commit_msg>implement lock for repository<commit_after>package cvmfs

import (
	"fmt"
	"io"
	"strings"
	"sync"

	exec "github.com/cvmfs/ducc/exec"
	log "github.com/sirupsen/logrus"
)

type TransactionOption interface {
	ToString() string
}

type TemplateTransaction struct {
	source      string
	destination string
}

func (t TemplateTransaction) ToString() string {
	return fmt.Sprintf("-T %s=%s", t.source, t.destination)
}

var locksMap = make(map[string]*sync.Mutex)
var lockMap = &sync.Mutex{}

func getLock(CVMFSRepo string) {
	lockMap.Lock()
	l := locksMap[CVMFSRepo]
	if l == nil {
		locksMap[CVMFSRepo] = &sync.Mutex{}
		l = locksMap[CVMFSRepo]
	}
	lockMap.Unlock()
	l.Lock()
}

func unlock(CVMFSRepo string) {
	lockMap.Lock()
	l := locksMap[CVMFSRepo]
	lockMap.Unlock()
	l.Unlock()
}

func OpenTransaction(CVMFSRepo string, options ...TransactionOption) error {
	cmd := []string{"cvmfs_server", "transaction"}
	for _, opt := range options {
		cmd = append(cmd, opt.ToString())
	}
	cmd = append(cmd, CVMFSRepo)
	getLock(CVMFSRepo)
	err := exec.ExecCommand("cvmfs_server", "transaction", CVMFSRepo).Start()
	if err != nil {
		LogE(err).WithFields(
			log.Fields{"repo": CVMFSRepo}).
			Error("Error in opening the transaction")
		abort(CVMFSRepo)
	}
	return err
}

func Publish(CVMFSRepo string) error {
	defer unlock(CVMFSRepo)
	err := exec.ExecCommand("cvmfs_server", "publish", CVMFSRepo).Start()
	if err != nil {
		LogE(err).WithFields(
			log.Fields{"repo": CVMFSRepo}).
			Error("Error in publishing the repository")
		abort(CVMFSRepo)
	}
	return err
}

func Abort(CVMFSRepo string) error {
	defer unlock(CVMFSRepo)
	err := abort(CVMFSRepo)
	if err != nil {
		LogE(err).WithFields(
			log.Fields{"repo": CVMFSRepo}).
			Error("Error in abort the transaction")
	}
	return err
}

func abort(CVMFSRepo string) error {
	return exec.ExecCommand("cvmfs_server", "abort", "-f", CVMFSRepo).Start()
}

func RepositoryExists(CVMFSRepo string) bool {
	cmd := exec.ExecCommand("cvmfs_server", "list")
	err, stdout, _ := cmd.StartWithOutput()
	if err != nil {
		LogE(fmt.Errorf("Error in listing the repository")).
			Error("Repo not present")
		return false
	}
	stdoutString := string(stdout.Bytes())

	if strings.Contains(stdoutString, CVMFSRepo) {
		return true
	} else {
		return false
	}
}

func WithinTransaction(CVMFSRepo string, f func() error, opts ...TransactionOption) error {
	err := OpenTransaction(CVMFSRepo, opts...)
	if err != nil {
		return err
	}
	err = f()
	if err != nil {
		return Abort(CVMFSRepo)
	}
	return Publish(CVMFSRepo)
}
<|endoftext|>


<commit_before>package cvmfs

import (
	"fmt"
	"io"
	"strings"
	"sync"
	"time"

	exec "github.com/cvmfs/ducc/exec"
	l "github.com/cvmfs/ducc/log"
	log "github.com/sirupsen/logrus"
)

type TransactionOption interface {
	ToString() string
}

type TemplateTransaction struct {
	source      string
	destination string
}

func NewTemplateTransaction(source, destination string) TemplateTransaction {
	return TemplateTransaction{source, destination}
}

func (t TemplateTransaction) ToString() string {
	return fmt.Sprintf("-T %s=%s", t.source, t.destination)
}

var locksMap = make(map[string]*sync.Mutex)
var locksFile = make(map[string]fSLock)
var lockMap = &sync.Mutex{}

func getLock(CVMFSRepo string) {
	lockMap.Lock()
	lc := locksMap[CVMFSRepo]
	if lc == nil {
		locksMap[CVMFSRepo] = &sync.Mutex{}
		lc = locksMap[CVMFSRepo]
	}
	f := locksFile[CVMFSRepo]
	if f == nil {
		f = newFSLock("/tmp/DUCC.lock")
		locksFile[CVMFSRepo] = f
		f = locksFile[CVMFSRepo]
	}
	lc.Lock()
	lockMap.Unlock()

	err := f.LockWriteB()
	for err != nil {
		// this may happen if the kernel detect a deadlock
		// it should never happen in our case, (of a single global lock)
		// but still we can protect against it
		l.LogE(err).Info("Error in getting the FS lock")
		time.Sleep(100 * time.Millisecond)
		err = f.LockWriteB()
	}
}

func unlock(CVMFSRepo string) {
	lockMap.Lock()
	l := locksMap[CVMFSRepo]
	f := locksFile[CVMFSRepo]
	lockMap.Unlock()
	l.Unlock()
	f.Unlock()
}

func ExecuteAndOpenTransaction(CVMFSRepo string, f func() error, options ...TransactionOption) error {
	cmd := []string{"cvmfs_server", "transaction"}
	for _, opt := range options {
		cmd = append(cmd, opt.ToString())
	}
	cmd = append(cmd, CVMFSRepo)
	getLock(CVMFSRepo)
	if err := f(); err != nil {
		unlock(CVMFSRepo)
		return err
	}
	err := exec.ExecCommand(cmd...).Start()
	if err != nil {
		l.LogE(err).WithFields(
			log.Fields{"repo": CVMFSRepo}).
			Error("Error in opening the transaction")
		Abort(CVMFSRepo)
	}
	return err

}

func OpenTransaction(CVMFSRepo string, options ...TransactionOption) error {
	return ExecuteAndOpenTransaction(CVMFSRepo, func() error { return nil }, options...)
}

func Publish(CVMFSRepo string) error {
	defer unlock(CVMFSRepo)
	err := exec.ExecCommand("cvmfs_server", "publish", CVMFSRepo).Start()
	if err != nil {
		l.LogE(err).WithFields(
			log.Fields{"repo": CVMFSRepo}).
			Error("Error in publishing the repository")
		abort(CVMFSRepo)
		return err
	}

	l.LogE(err).WithFields(
		log.Fields{"repo": CVMFSRepo}).
		Info("Publish complete")
	return nil
}

func Abort(CVMFSRepo string) error {
	defer unlock(CVMFSRepo)
	err := abort(CVMFSRepo)
	if err != nil {
		l.LogE(err).WithFields(
			log.Fields{"repo": CVMFSRepo}).
			Error("Error in abort the transaction")
	}
	return err
}

func abort(CVMFSRepo string) error {
	return exec.ExecCommand("cvmfs_server", "abort", "-f", CVMFSRepo).Start()
}

func RepositoryExists(CVMFSRepo string) bool {
	cmd := exec.ExecCommand("cvmfs_server", "list")
	err, stdout, _ := cmd.StartWithOutput()
	if err != nil {
		l.LogE(fmt.Errorf("Error in listing the repository")).
			Error("Repo not present")
		return false
	}
	stdoutString := string(stdout.Bytes())

	if strings.Contains(stdoutString, CVMFSRepo) {
		return true
	} else {
		return false
	}
}

func WithinTransaction(CVMFSRepo string, f func() error, opts ...TransactionOption) error {
	err := OpenTransaction(CVMFSRepo, opts...)
	if err != nil {
		return err
	}
	err = f()
	if err != nil {
		return Abort(CVMFSRepo)
	}
	return Publish(CVMFSRepo)
}

func Ingest(CVMFSRepo string, input io.ReadCloser, options ...string) error {
	cmd := []string{"cvmfs_server", "ingest"}
	for _, opt := range options {
		cmd = append(cmd, opt)
	}
	cmd = append(cmd, CVMFSRepo)
	getLock(CVMFSRepo)
	defer unlock(CVMFSRepo)
	return exec.ExecCommand(cmd...).StdIn(input).Start()
}

func IngestDelete(CVMFSRepo string, path string) error {
	getLock(CVMFSRepo)
	defer unlock(CVMFSRepo)
	return exec.ExecCommand("cvmfs_server", "ingest", "--delete", path, CVMFSRepo).Start()
}
<commit_msg>hold as little as possible the lock<commit_after>package cvmfs

import (
	"fmt"
	"io"
	"strings"
	"sync"
	"time"

	exec "github.com/cvmfs/ducc/exec"
	l "github.com/cvmfs/ducc/log"
	log "github.com/sirupsen/logrus"
)

type TransactionOption interface {
	ToString() string
}

type TemplateTransaction struct {
	source      string
	destination string
}

func NewTemplateTransaction(source, destination string) TemplateTransaction {
	return TemplateTransaction{source, destination}
}

func (t TemplateTransaction) ToString() string {
	return fmt.Sprintf("-T %s=%s", t.source, t.destination)
}

var locksMap = make(map[string]*sync.Mutex)
var locksFile = make(map[string]fSLock)
var lockMap = &sync.Mutex{}

func getLock(CVMFSRepo string) {
	lockMap.Lock()
	lc := locksMap[CVMFSRepo]
	if lc == nil {
		locksMap[CVMFSRepo] = &sync.Mutex{}
		lc = locksMap[CVMFSRepo]
	}
	f := locksFile[CVMFSRepo]
	if f == nil {
		f = newFSLock("/tmp/DUCC.lock")
		locksFile[CVMFSRepo] = f
		f = locksFile[CVMFSRepo]
	}
	lockMap.Unlock()

	lc.Lock()
	err := f.LockWriteB()
	for err != nil {
		// this may happen if the kernel detect a deadlock
		// it should never happen in our case, (of a single global lock)
		// but still we can protect against it
		l.LogE(err).Info("Error in getting the FS lock")
		time.Sleep(100 * time.Millisecond)
		err = f.LockWriteB()
	}
}

func unlock(CVMFSRepo string) {
	lockMap.Lock()
	l := locksMap[CVMFSRepo]
	f := locksFile[CVMFSRepo]
	lockMap.Unlock()

	l.Unlock()
	f.Unlock()
}

func ExecuteAndOpenTransaction(CVMFSRepo string, f func() error, options ...TransactionOption) error {
	cmd := []string{"cvmfs_server", "transaction"}
	for _, opt := range options {
		cmd = append(cmd, opt.ToString())
	}
	cmd = append(cmd, CVMFSRepo)
	getLock(CVMFSRepo)
	if err := f(); err != nil {
		unlock(CVMFSRepo)
		return err
	}
	err := exec.ExecCommand(cmd...).Start()
	if err != nil {
		l.LogE(err).WithFields(
			log.Fields{"repo": CVMFSRepo}).
			Error("Error in opening the transaction")
		Abort(CVMFSRepo)
	}
	return err

}

func OpenTransaction(CVMFSRepo string, options ...TransactionOption) error {
	return ExecuteAndOpenTransaction(CVMFSRepo, func() error { return nil }, options...)
}

func Publish(CVMFSRepo string) error {
	defer unlock(CVMFSRepo)
	err := exec.ExecCommand("cvmfs_server", "publish", CVMFSRepo).Start()
	if err != nil {
		l.LogE(err).WithFields(
			log.Fields{"repo": CVMFSRepo}).
			Error("Error in publishing the repository")
		abort(CVMFSRepo)
		return err
	}

	l.LogE(err).WithFields(
		log.Fields{"repo": CVMFSRepo}).
		Info("Publish complete")
	return nil
}

func Abort(CVMFSRepo string) error {
	defer unlock(CVMFSRepo)
	err := abort(CVMFSRepo)
	if err != nil {
		l.LogE(err).WithFields(
			log.Fields{"repo": CVMFSRepo}).
			Error("Error in abort the transaction")
	}
	return err
}

func abort(CVMFSRepo string) error {
	return exec.ExecCommand("cvmfs_server", "abort", "-f", CVMFSRepo).Start()
}

func RepositoryExists(CVMFSRepo string) bool {
	cmd := exec.ExecCommand("cvmfs_server", "list")
	err, stdout, _ := cmd.StartWithOutput()
	if err != nil {
		l.LogE(fmt.Errorf("Error in listing the repository")).
			Error("Repo not present")
		return false
	}
	stdoutString := string(stdout.Bytes())

	if strings.Contains(stdoutString, CVMFSRepo) {
		return true
	} else {
		return false
	}
}

func WithinTransaction(CVMFSRepo string, f func() error, opts ...TransactionOption) error {
	err := OpenTransaction(CVMFSRepo, opts...)
	if err != nil {
		return err
	}
	err = f()
	if err != nil {
		return Abort(CVMFSRepo)
	}
	return Publish(CVMFSRepo)
}

func Ingest(CVMFSRepo string, input io.ReadCloser, options ...string) error {
	cmd := []string{"cvmfs_server", "ingest"}
	for _, opt := range options {
		cmd = append(cmd, opt)
	}
	cmd = append(cmd, CVMFSRepo)
	getLock(CVMFSRepo)
	defer unlock(CVMFSRepo)
	return exec.ExecCommand(cmd...).StdIn(input).Start()
}

func IngestDelete(CVMFSRepo string, path string) error {
	getLock(CVMFSRepo)
	defer unlock(CVMFSRepo)
	return exec.ExecCommand("cvmfs_server", "ingest", "--delete", path, CVMFSRepo).Start()
}
<|endoftext|>


<commit_before>package action

import (
	"errors"
	"fmt"
	"strings"

	"golang.org/x/net/websocket"
)

var (
	errNotImplemented = errors.New("Not Implemented!")
)

// Exec execute the js code on WebBrowser
func Exec(ws *websocket.Conn, js string) error {
	return websocket.Message.Send(ws, js)
}

//Set update a form element (textbox, dropdown, checkbox, etc) to set text value of TargetID.
func Set(ws *websocket.Conn, target, value string) error {
	return errNotImplemented
}

//Enable the target form field or button.
func Enable(ws *websocket.Conn, target string) error {
	return errNotImplemented
}

//Disable  the target form field or button.
func Disable(ws *websocket.Conn, target string) error {
	return errNotImplemented
}

//Replace target with new content
func Replace(ws *websocket.Conn, target, content string) error {
	c := strings.Replace(content, "\n", "\\n", -1)
	c = strings.Replace(c, "\"", "\\\"", -1)
	js := fmt.Sprintf("$( \"#%s\" ).replaceWith(\"%s\");", target, c)
	return websocket.Message.Send(ws, js)
}

//Replace target with new content
func Html(ws *websocket.Conn, target, content string) error {
	c := strings.Replace(content, "\n", "\\n", -1)
	c = strings.Replace(c, "\"", "\\\"", -1)
	js := fmt.Sprintf("$( \"#%s\" ).html(\"%s\");", target, c)
	return websocket.Message.Send(ws, js)
}

//Remove target from the DOM
func Remove(ws *websocket.Conn, target string) error {
	return errNotImplemented
}

//InsertTop Insert content at the top of target
func InsertTop(ws *websocket.Conn, target, content string) error {
	return errNotImplemented
}

//InsertBottom Insert content at the bottom of target
func InsertBottom(ws *websocket.Conn, target, content string) error {
	return errNotImplemented
}

//InsertBefore Insert content at the before of target
func InsertBefore(ws *websocket.Conn, target, content string) error {
	return errNotImplemented
}

//InsertAfter Insert content at the after of target
func InsertAfter(ws *websocket.Conn, target, content string) error {
	return errNotImplemented
}

//Redirect to url
func Redirect(ws *websocket.Conn, url string) error {
	return errNotImplemented
}

//Get content of form element
func Get(ws *websocket.Conn, target string) (string, error) {
	ret := ""
	js := fmt.Sprintf("ws.send($('#%s').val());", target)
	err := websocket.Message.Send(ws, js)
	if err != nil {
		return "", err
	}

	err = websocket.Message.Receive(ws, &ret)
	if err != nil {
		return "", err
	}

	return ret, nil
}

//Wire bind an action to an event on target
func Wire(ws *websocket.Conn, target, event, act string) error {
	return errNotImplemented
}

func SendEvent(ws *websocket.Conn, event string) error {
	js := fmt.Sprintf("sendEvent('%s');", event)
	err := websocket.Message.Send(ws, js)
	if err != nil {
		return err
	}

	return nil
}

func Alert(ws *websocket.Conn, message string) error {
	js := fmt.Sprintf("alert('%s');", message)
	err := websocket.Message.Send(ws, js)
	if err != nil {
		return err
	}

	return nil
}

//BlockUI block page interaction
func BlockUI(ws *websocket.Conn) {
	Exec(ws, "$.blockUI();")
	/*	Exec(ws, "$.blockUI();ws.send('blocked');")

		ret := ""
		err := websocket.Message.Receive(ws, &ret)
		if err != nil {
			log.Println("ERROR blockUI", err)
		}*/
}

//UnblockUI block page interaction
func UnblockUI(ws *websocket.Conn) {
	Exec(ws, "$.unblockUI();")
}
<commit_msg>action.Set implemented<commit_after>package action

import (
	"errors"
	"fmt"
	"strings"

	"golang.org/x/net/websocket"
)

var (
	errNotImplemented = errors.New("Not Implemented!")
)

// Exec execute the js code on WebBrowser
func Exec(ws *websocket.Conn, js string) error {
	return websocket.Message.Send(ws, js)
}

//Enable the target form field or button.
func Enable(ws *websocket.Conn, target string) error {
	return errNotImplemented
}

//Disable  the target form field or button.
func Disable(ws *websocket.Conn, target string) error {
	return errNotImplemented
}

//Replace target with new content
func Replace(ws *websocket.Conn, target, content string) error {
	c := strings.Replace(content, "\n", "\\n", -1)
	c = strings.Replace(c, "\"", "\\\"", -1)
	js := fmt.Sprintf("$( \"#%s\" ).replaceWith(\"%s\");", target, c)
	return websocket.Message.Send(ws, js)
}

//Replace target with new content
func Html(ws *websocket.Conn, target, content string) error {
	c := strings.Replace(content, "\n", "\\n", -1)
	c = strings.Replace(c, "\"", "\\\"", -1)
	js := fmt.Sprintf("$( \"#%s\" ).html(\"%s\");", target, c)
	return websocket.Message.Send(ws, js)
}

//Remove target from the DOM
func Remove(ws *websocket.Conn, target string) error {
	return errNotImplemented
}

//InsertTop Insert content at the top of target
func InsertTop(ws *websocket.Conn, target, content string) error {
	return errNotImplemented
}

//InsertBottom Insert content at the bottom of target
func InsertBottom(ws *websocket.Conn, target, content string) error {
	return errNotImplemented
}

//InsertBefore Insert content at the before of target
func InsertBefore(ws *websocket.Conn, target, content string) error {
	return errNotImplemented
}

//InsertAfter Insert content at the after of target
func InsertAfter(ws *websocket.Conn, target, content string) error {
	return errNotImplemented
}

//Redirect to url
func Redirect(ws *websocket.Conn, url string) error {
	return errNotImplemented
}

//Set update a form element (textbox, dropdown, checkbox, etc) to set text value of TargetID.
func Set(ws *websocket.Conn, target, value string) error {
	js := fmt.Sprintf("ws.send($('#%s').val('%s'));", target, value)
	err := websocket.Message.Send(ws, js)
	if err != nil {
		return err
	}
	return nil
}

//Get content of form element
func Get(ws *websocket.Conn, target string) (string, error) {
	ret := ""
	js := fmt.Sprintf("ws.send($('#%s').val());", target)
	err := websocket.Message.Send(ws, js)
	if err != nil {
		return "", err
	}

	err = websocket.Message.Receive(ws, &ret)
	if err != nil {
		return "", err
	}

	return ret, nil
}

//Wire bind an action to an event on target
func Wire(ws *websocket.Conn, target, event, act string) error {
	return errNotImplemented
}

func SendEvent(ws *websocket.Conn, event string) error {
	js := fmt.Sprintf("sendEvent('%s');", event)
	err := websocket.Message.Send(ws, js)
	if err != nil {
		return err
	}

	return nil
}

func Alert(ws *websocket.Conn, message string) error {
	js := fmt.Sprintf("alert('%s');", message)
	err := websocket.Message.Send(ws, js)
	if err != nil {
		return err
	}

	return nil
}

//BlockUI block page interaction
func BlockUI(ws *websocket.Conn) {
	Exec(ws, "$.blockUI();")
	/*	Exec(ws, "$.blockUI();ws.send('blocked');")

		ret := ""
		err := websocket.Message.Receive(ws, &ret)
		if err != nil {
			log.Println("ERROR blockUI", err)
		}*/
}

//UnblockUI block page interaction
func UnblockUI(ws *websocket.Conn) {
	Exec(ws, "$.unblockUI();")
}
<|endoftext|>


<commit_before>// Copyright (c) 2012 - Cloud Instruments Co. Ltd.
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
//    list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
//    this list of conditions and the following disclaimer in the documentation
//    and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package main

import (
	"time"
	log "seelog"
)

func adaptiveMain() {
	defer log.Flush()
	loadAdaptiveConfig()
	testMsgIntensity(1)
	testMsgIntensity(5)
	testMsgIntensity(10)
}

func testMsgIntensity(intensity int) {
	log.Default.Info("Intensity test: %d", intensity)

	for j := 0; j < 4; j++ {
		for i := 0; i < intensity; i++ {
			log.Trace("%d", i)
			<-time.After(time.Second / time.Duration(intensity))
		}
	}

	log.Default.Info("Messages sent")

	<-time.After(time.Second * time.Duration(intensity))
}

func loadAdaptiveConfig() {
	testConfig := `<seelog type="adaptive" mininterval="200000000" maxinterval="1000000000" critmsgcount="5">
	<outputs formatid="msg">
		<console/>
	</outputs>
	<formats>
		<format id="msg" format="%Time: %Msg%n"/>
	</formats>
</seelog>`

	logger, _ := log.LoggerFromConfigAsBytes([]byte(testConfig))

	log.ReplaceLogger(logger)
}
<commit_msg>Fixed import<commit_after>// Copyright (c) 2012 - Cloud Instruments Co. Ltd.
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
//    list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
//    this list of conditions and the following disclaimer in the documentation
//    and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package main

import (
	"time"
	log "github.com/cihub/seelog"
)

func adaptiveMain() {
	defer log.Flush()
	loadAdaptiveConfig()
	testMsgIntensity(1)
	testMsgIntensity(5)
	testMsgIntensity(10)
}

func testMsgIntensity(intensity int) {
	log.Default.Info("Intensity test: %d", intensity)

	for j := 0; j < 4; j++ {
		for i := 0; i < intensity; i++ {
			log.Trace("%d", i)
			<-time.After(time.Second / time.Duration(intensity))
		}
	}

	log.Default.Info("Messages sent")

	<-time.After(time.Second * time.Duration(intensity))
}

func loadAdaptiveConfig() {
	testConfig := `<seelog type="adaptive" mininterval="200000000" maxinterval="1000000000" critmsgcount="5">
	<outputs formatid="msg">
		<console/>
	</outputs>
	<formats>
		<format id="msg" format="%Time: %Msg%n"/>
	</formats>
</seelog>`

	logger, _ := log.LoggerFromConfigAsBytes([]byte(testConfig))

	log.ReplaceLogger(logger)
}
<|endoftext|>


<commit_before>package articles

import (
	"errors"
	"fmt"
	"strings"
	"time"
)

const (
	TimeFormat = "January 2, 2006"
	readMore   = "<!--readmore-->"
)

type Articles []*Article
type Article struct {
	Date     time.Time
	Edit     time.Time
	Title    string
	Slug     string
	Body     string
	Tags     Tags
	Enabled  bool
	Author   string
	Comments Comments
}

type YearMap map[int]Articles
type MonthMap map[int]Articles

func (a *Article) makeSlug() {
	r := strings.NewReplacer(" ", "-")
	a.Slug = r.Replace(strings.TrimSpace(a.Title))
}

func (a *Article) Publish() {
	a.Date = time.Now()
	a.Enabled = true
}

func (a *Article) Suppress() {
	a.Enabled = false
}

func (a *Article) AddComment(c *Comment) {
	a.Comments.Add(c)
}

func (a Articles) Len() int           { return len(a) }
func (a Articles) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a Articles) Less(i, j int) bool { return a[i].Date.Before(a[j].Date) }

func (a *Articles) Add(article *Article) error {
	article.Date = time.Now()
	if article.Slug == "" {
		article.makeSlug()
	}
	if _, ok := a.Find(article.Slug); ok {
		return errors.New("duplicate slug " + article.Slug)
	}
	*a = append(*a, article)
	return nil
}

func (a *Articles) Update(article *Article) error {
	article.Edit = time.Now()
	if article.Slug == "" {
		article.makeSlug()
	}
	if i, ok := a.locate(article.Slug); ok {
		article.Date = (*a)[i].Date
		(*a)[i] = article
	} else {
		return a.Add(article)
	}
	return nil
}

func (a Articles) locate(slug string) (int, bool) {
	for i, ar := range a {
		if ar.Slug == slug {
			return i, true
		}
	}
	return 0, false
}

func (a Articles) Find(slug string) (*Article, bool) {
	if i, ok := a.locate(slug); ok {
		return a[i], true
	}
	return nil, false
}

// Pager
func (a Articles) Page(page, app int) (_ Articles, next int, prev int) {
	if page <= 1 {
		page = 1
	} else {
		prev = page - 1
	}

	lastpage := len(a)/app + 1
	if page >= lastpage {
		page = lastpage
	} else {
		next = page + 1
	}

	from := (page - 1) * app
	to := from + app - 1
	if to > len(a) {
		to = len(a)
	}

	return a[from:to], next, prev
}

// Format Date with TimeFormat
func (a Article) PostDate() string {
	return a.Date.Local().Format(TimeFormat)
}

func (a Article) Edited() bool {
	return !a.Edit.IsZero()
}
func (a Article) EditDate() string {
	return a.Edit.Local().Format(TimeFormat)
}

// Format Date for RSS
func (a Article) RssDate() string {
	return a.Date.Local().Format(time.RFC1123Z)
}

func (a Article) Spoiler() string {
	if i := strings.Index(a.Body, readMore); i > 0 {
		return a.Body[:i]
	}
	return a.Body
}

func (a Article) HasMore() bool {
	return strings.Contains(a.Body, readMore)
}

func (a Article) Year() int {
	return a.Date.Year()
}

func (a Article) Month() time.Month {
	return a.Date.Month()
}

func (a Articles) Year(year int) (A Articles) {
	if year == 0 {
		year = time.Now().Year()
	}
	for _, v := range a {
		if v.Date.Year() == year {
			A = append(A, v)
		}
	}
	return A
}

func (a Articles) Month(month time.Month) (A Articles) {
	if month == 0 {
		month = time.Now().Month()
	}
	for _, v := range a {
		if v.Date.Month() == month {
			A = append(A, v)
		}
	}
	return A
}

func (a Articles) Enabled() (A Articles) {
	for _, v := range a {
		if v.Enabled {
			A = append(A, v)
		}
	}
	return A
}

func (a Articles) Skip(n int) Articles {
	if n > len(a) {
		return nil
	}
	return a[n:]
}

func (a Articles) Limit(n int) Articles {
	if n > len(a) {
		n = len(a)
	}
	return a[:n]
}

func (a Articles) Head() Article {
	if len(a) > 0 {
		return *a[0]
	}
	return Article{}
}

func (a Articles) Tail() Article {
	if len(a) > 0 {
		return *a[len(a)-1]
	}
	return Article{}
}

func (a Articles) YearMap() YearMap {
	ym := make(YearMap)
	for _, v := range a {
		y := v.Date.Year()
		ym[y] = append(ym[y], v)
	}
	return ym
}

func (a Articles) MonthMap() MonthMap {
	mm := make(MonthMap)
	for _, v := range a {
		m := int(v.Date.Month())
		mm[m] = append(mm[m], v)
	}
	return mm
}

func (a Article) FullPath() string {
	return fmt.Sprintf("/%.4d/%.2d/%s", a.Date.Year(), a.Date.Month(), a.Slug)
}
<commit_msg>fit into 80 chars<commit_after>package articles

import (
	"errors"
	"fmt"
	"strings"
	"time"
)

const (
	TimeFormat = "January 2, 2006"
	readMore   = "<!--readmore-->"
)

type Articles []*Article
type Article struct {
	Date     time.Time
	Edit     time.Time
	Title    string
	Slug     string
	Body     string
	Tags     Tags
	Enabled  bool
	Author   string
	Comments Comments
}

type YearMap map[int]Articles
type MonthMap map[int]Articles

func (a *Article) makeSlug() {
	r := strings.NewReplacer(" ", "-")
	a.Slug = r.Replace(strings.TrimSpace(a.Title))
}

func (a *Article) Publish() {
	a.Date = time.Now()
	a.Enabled = true
}

func (a *Article) Suppress() {
	a.Enabled = false
}

func (a *Article) AddComment(c *Comment) {
	a.Comments.Add(c)
}

func (a Articles) Len() int           { return len(a) }
func (a Articles) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a Articles) Less(i, j int) bool { return a[i].Date.Before(a[j].Date) }

func (a *Articles) Add(article *Article) error {
	article.Date = time.Now()
	if article.Slug == "" {
		article.makeSlug()
	}
	if _, ok := a.Find(article.Slug); ok {
		return errors.New("duplicate slug " + article.Slug)
	}
	*a = append(*a, article)
	return nil
}

func (a *Articles) Update(article *Article) error {
	article.Edit = time.Now()
	if article.Slug == "" {
		article.makeSlug()
	}
	if i, ok := a.locate(article.Slug); ok {
		article.Date = (*a)[i].Date
		(*a)[i] = article
	} else {
		return a.Add(article)
	}
	return nil
}

func (a Articles) locate(slug string) (int, bool) {
	for i, ar := range a {
		if ar.Slug == slug {
			return i, true
		}
	}
	return 0, false
}

func (a Articles) Find(slug string) (*Article, bool) {
	if i, ok := a.locate(slug); ok {
		return a[i], true
	}
	return nil, false
}

// Pager
func (a Articles) Page(page, app int) (_ Articles, next int, prev int) {
	if page <= 1 {
		page = 1
	} else {
		prev = page - 1
	}

	lastpage := len(a)/app + 1
	if page >= lastpage {
		page = lastpage
	} else {
		next = page + 1
	}

	from := (page - 1) * app
	to := from + app - 1
	if to > len(a) {
		to = len(a)
	}

	return a[from:to], next, prev
}

// Format Date with TimeFormat
func (a Article) PostDate() string {
	return a.Date.Local().Format(TimeFormat)
}

func (a Article) Edited() bool {
	return !a.Edit.IsZero()
}
func (a Article) EditDate() string {
	return a.Edit.Local().Format(TimeFormat)
}

// Format Date for RSS
func (a Article) RssDate() string {
	return a.Date.Local().Format(time.RFC1123Z)
}

func (a Article) Spoiler() string {
	if i := strings.Index(a.Body, readMore); i > 0 {
		return a.Body[:i]
	}
	return a.Body
}

func (a Article) HasMore() bool {
	return strings.Contains(a.Body, readMore)
}

func (a Article) Year() int {
	return a.Date.Year()
}

func (a Article) Month() time.Month {
	return a.Date.Month()
}

func (a Articles) Year(year int) (A Articles) {
	if year == 0 {
		year = time.Now().Year()
	}
	for _, v := range a {
		if v.Date.Year() == year {
			A = append(A, v)
		}
	}
	return A
}

func (a Articles) Month(month time.Month) (A Articles) {
	if month == 0 {
		month = time.Now().Month()
	}
	for _, v := range a {
		if v.Date.Month() == month {
			A = append(A, v)
		}
	}
	return A
}

func (a Articles) Enabled() (A Articles) {
	for _, v := range a {
		if v.Enabled {
			A = append(A, v)
		}
	}
	return A
}

func (a Articles) Skip(n int) Articles {
	if n > len(a) {
		return nil
	}
	return a[n:]
}

func (a Articles) Limit(n int) Articles {
	if n > len(a) {
		n = len(a)
	}
	return a[:n]
}

func (a Articles) Head() Article {
	if len(a) > 0 {
		return *a[0]
	}
	return Article{}
}

func (a Articles) Tail() Article {
	if len(a) > 0 {
		return *a[len(a)-1]
	}
	return Article{}
}

func (a Articles) YearMap() YearMap {
	ym := make(YearMap)
	for _, v := range a {
		y := v.Date.Year()
		ym[y] = append(ym[y], v)
	}
	return ym
}

func (a Articles) MonthMap() MonthMap {
	mm := make(MonthMap)
	for _, v := range a {
		m := int(v.Date.Month())
		mm[m] = append(mm[m], v)
	}
	return mm
}

func (a Article) FullPath() string {
	return fmt.Sprintf("/%.4d/%.2d/%s",
		a.Date.Year(), a.Date.Month(), a.Slug)
}
<|endoftext|>


<commit_before>package articles

import (
	"errors"
	"fmt"
	"strings"
	"time"
)

const (
	TimeFormat = "January 2, 2006"
	readMore   = "<!--readmore-->"
)

type Articles []*Article
type Article struct {
	Date     time.Time
	Edit     time.Time
	Title    string
	Slug     string
	Body     string
	Tags     Tags
	Enabled  bool
	Author   string
	Comments Comments
}

type YearMap map[int]Articles
type MonthMap map[int]Articles

func (a *Article) makeSlug() {
	r := strings.NewReplacer(" ", "-")
	a.Slug = r.Replace(strings.TrimSpace(a.Title))
}

func (a *Article) Publish() {
	a.Date = time.Now()
	a.Enabled = true
}

func (a *Article) Suppress() {
	a.Enabled = false
}

func (a *Article) AddComment(c *Comment) {
	a.Comments.Add(c)
}

func (a Articles) Len() int           { return len(a) }
func (a Articles) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a Articles) Less(i, j int) bool { return a[i].Date.Before(a[j].Date) }

func (a *Articles) Add(article *Article) error {
	article.Date = time.Now()
	if article.Slug == "" {
		article.makeSlug()
	}
	_, err := a.Find(article.Slug)
	if err == nil {
		return errors.New("duplicate slug " + article.Slug)
	}
	*a = append(*a, article)
	return nil
}

func (a *Articles) Update(article *Article) error {
	article.Edit = time.Now()
	if article.Slug == "" {
		article.makeSlug()
	}
	i, err := a.locate(article.Slug)
	if err != nil {
		return a.Add(article)
	}
	article.Date = (*a)[i].Date
	(*a)[i] = article
	return nil
}

func (a Articles) locate(slug string) (int, error) {
	for i, ar := range a {
		if ar.Slug == slug {
			return i, nil
		}
	}
	return 0, errors.New("not found " + slug)
}

func (a Articles) Find(slug string) (*Article, error) {
	i, err := a.locate(slug)
	if err != nil {
		return nil, err
	}
	return a[i], nil
}

// Pager
func (a Articles) Page(page, app int) (Articles, next int, prev int) {
	lastpage := len(a)/app + 1

	if page <= 1 {
		page = 1
	} else {
		prev = page - 1
	}

	if page >= lastpage {
		page = lastpage
	} else {
		next = page + 1
	}

	from := (page - 1) * app
	to := from + app - 1
	if to > len(a) {
		to = len(a)
	}

	return a[from:to], next, prev
}

// Format Date with TimeFormat
func (a Article) PostDate() string {
	return a.Date.Local().Format(TimeFormat)
}

func (a Article) Edited() bool {
	return !a.Edit.IsZero()
}
func (a Article) EditDate() string {
	return a.Edit.Local().Format(TimeFormat)
}

// Format Date for RSS
func (a Article) RssDate() string {
	return a.Date.Local().Format(time.RFC1123Z)
}

func (a Article) Spoiler() string {
	if i := strings.Index(a.Body, readMore); i > 0 {
		return a.Body[:i]
	}
	return a.Body
}

func (a Article) HasMore() bool {
	return strings.Contains(a.Body, readMore)
}

func (a Article) Year() int {
	return a.Date.Year()
}

func (a Article) Month() time.Month {
	return a.Date.Month()
}

func (a Articles) Year(year int) (A Articles) {
	if year == 0 {
		year = time.Now().Year()
	}
	for _, v := range a {
		if v.Date.Year() == year {
			A = append(A, v)
		}
	}
	return A
}

func (a Articles) Month(month time.Month) (A Articles) {
	if month == 0 {
		month = time.Now().Month()
	}
	for _, v := range a {
		if v.Date.Month() == month {
			A = append(A, v)
		}
	}
	return A
}

func (a Articles) Enabled() (A Articles) {
	for _, v := range a {
		if v.Enabled {
			A = append(A, v)
		}
	}
	return A
}

func (a Articles) Skip(n int) Articles {
	if n > len(a) {
		return nil
	}
	return a[n:]
}

func (a Articles) Limit(n int) Articles {
	if n > len(a) {
		n = len(a)
	}
	return a[:n]
}

func (a Articles) Head() Article {
	if len(a) > 0 {
		return *a[0]
	}
	return Article{}
}

func (a Articles) Tail() Article {
	if len(a) > 0 {
		return *a[len(a)-1]
	}
	return Article{}
}

func (a Articles) YearMap() YearMap {
	ym := make(YearMap)
	for _, v := range a {
		y := v.Date.Year()
		ym[y] = append(ym[y], v)
	}
	return ym
}

func (a Articles) MonthMap() MonthMap {
	mm := make(MonthMap)
	for _, v := range a {
		m := int(v.Date.Month())
		mm[m] = append(mm[m], v)
	}
	return mm
}

func (a Article) FullPath() string {
	return fmt.Sprintf("/%.4d/%.2d/%s", a.Date.Year(), a.Date.Month(), a.Slug)
}
<commit_msg>fix def<commit_after>package articles

import (
	"errors"
	"fmt"
	"strings"
	"time"
)

const (
	TimeFormat = "January 2, 2006"
	readMore   = "<!--readmore-->"
)

type Articles []*Article
type Article struct {
	Date     time.Time
	Edit     time.Time
	Title    string
	Slug     string
	Body     string
	Tags     Tags
	Enabled  bool
	Author   string
	Comments Comments
}

type YearMap map[int]Articles
type MonthMap map[int]Articles

func (a *Article) makeSlug() {
	r := strings.NewReplacer(" ", "-")
	a.Slug = r.Replace(strings.TrimSpace(a.Title))
}

func (a *Article) Publish() {
	a.Date = time.Now()
	a.Enabled = true
}

func (a *Article) Suppress() {
	a.Enabled = false
}

func (a *Article) AddComment(c *Comment) {
	a.Comments.Add(c)
}

func (a Articles) Len() int           { return len(a) }
func (a Articles) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a Articles) Less(i, j int) bool { return a[i].Date.Before(a[j].Date) }

func (a *Articles) Add(article *Article) error {
	article.Date = time.Now()
	if article.Slug == "" {
		article.makeSlug()
	}
	_, err := a.Find(article.Slug)
	if err == nil {
		return errors.New("duplicate slug " + article.Slug)
	}
	*a = append(*a, article)
	return nil
}

func (a *Articles) Update(article *Article) error {
	article.Edit = time.Now()
	if article.Slug == "" {
		article.makeSlug()
	}
	i, err := a.locate(article.Slug)
	if err != nil {
		return a.Add(article)
	}
	article.Date = (*a)[i].Date
	(*a)[i] = article
	return nil
}

func (a Articles) locate(slug string) (int, error) {
	for i, ar := range a {
		if ar.Slug == slug {
			return i, nil
		}
	}
	return 0, errors.New("not found " + slug)
}

func (a Articles) Find(slug string) (*Article, error) {
	i, err := a.locate(slug)
	if err != nil {
		return nil, err
	}
	return a[i], nil
}

// Pager
func (a Articles) Page(page, app int) (A Articles, next int, prev int) {
	lastpage := len(a)/app + 1

	if page <= 1 {
		page = 1
	} else {
		prev = page - 1
	}

	if page >= lastpage {
		page = lastpage
	} else {
		next = page + 1
	}

	from := (page - 1) * app
	to := from + app - 1
	if to > len(a) {
		to = len(a)
	}

	return a[from:to], next, prev
}

// Format Date with TimeFormat
func (a Article) PostDate() string {
	return a.Date.Local().Format(TimeFormat)
}

func (a Article) Edited() bool {
	return !a.Edit.IsZero()
}
func (a Article) EditDate() string {
	return a.Edit.Local().Format(TimeFormat)
}

// Format Date for RSS
func (a Article) RssDate() string {
	return a.Date.Local().Format(time.RFC1123Z)
}

func (a Article) Spoiler() string {
	if i := strings.Index(a.Body, readMore); i > 0 {
		return a.Body[:i]
	}
	return a.Body
}

func (a Article) HasMore() bool {
	return strings.Contains(a.Body, readMore)
}

func (a Article) Year() int {
	return a.Date.Year()
}

func (a Article) Month() time.Month {
	return a.Date.Month()
}

func (a Articles) Year(year int) (A Articles) {
	if year == 0 {
		year = time.Now().Year()
	}
	for _, v := range a {
		if v.Date.Year() == year {
			A = append(A, v)
		}
	}
	return A
}

func (a Articles) Month(month time.Month) (A Articles) {
	if month == 0 {
		month = time.Now().Month()
	}
	for _, v := range a {
		if v.Date.Month() == month {
			A = append(A, v)
		}
	}
	return A
}

func (a Articles) Enabled() (A Articles) {
	for _, v := range a {
		if v.Enabled {
			A = append(A, v)
		}
	}
	return A
}

func (a Articles) Skip(n int) Articles {
	if n > len(a) {
		return nil
	}
	return a[n:]
}

func (a Articles) Limit(n int) Articles {
	if n > len(a) {
		n = len(a)
	}
	return a[:n]
}

func (a Articles) Head() Article {
	if len(a) > 0 {
		return *a[0]
	}
	return Article{}
}

func (a Articles) Tail() Article {
	if len(a) > 0 {
		return *a[len(a)-1]
	}
	return Article{}
}

func (a Articles) YearMap() YearMap {
	ym := make(YearMap)
	for _, v := range a {
		y := v.Date.Year()
		ym[y] = append(ym[y], v)
	}
	return ym
}

func (a Articles) MonthMap() MonthMap {
	mm := make(MonthMap)
	for _, v := range a {
		m := int(v.Date.Month())
		mm[m] = append(mm[m], v)
	}
	return mm
}

func (a Article) FullPath() string {
	return fmt.Sprintf("/%.4d/%.2d/%s", a.Date.Year(), a.Date.Month(), a.Slug)
}
<|endoftext|>


<commit_before>// bugsnagrevel adds Bugsnag to revel. It lets you pass *revel.Controller into bugsnag.Notify(),
// and provides a Filter to catch errors.
package bugsnagrevel

import (
	"github.com/bugsnag/bugsnag-go"
	"github.com/revel/revel"
	"strings"
	"sync"
)

var once sync.Once

// This Filter should be added to the filter chain just after the PanicFilter.
// It sends errors to Bugsnag automatically. Configuration is read out of
// conf/app.conf, you should set bugsnag.apikey, and can also set
// bugsnag.endpoint, bugsnag.releasestage, bugsnag.appversion,
// bugsnag.projectroot, bugsnag.projectpackages if needed.
func Filter(c *revel.Controller, fc []revel.Filter) {
	defer bugsnag.AutoNotify(c)
	fc[0](c, fc[1:])
}

// Add support to bugsnag for reading data out of *revel.Controllers
func middleware(event *bugsnag.Event, config *bugsnag.Configuration) bool {
	for _, datum := range event.RawData {
		if controller, ok := datum.(*revel.Controller); ok {
			// make the request visible to the builtin HttpMIddleware
			event.RawData = append(event.RawData, controller.Request.Request)
			event.Context = controller.Action
			event.MetaData.AddStruct("Session", controller.Session)
		}
	}

	return true
}

func init() {
	revel.OnAppStart(func () {
		bugsnag.OnBeforeNotify(middleware)

		var projectPackages []string
		if packages, ok := revel.Config.String("bugsnag.projectpackages"); ok {
			projectPackages = strings.Split(packages, ",")
		} else {
			projectPackages = []string{revel.ImportPath + "/app/*"}
		}

		bugsnag.Configure(bugsnag.Configuration{
			APIKey:          revel.Config.StringDefault("bugsnag.apikey", ""),
			Endpoint:        revel.Config.StringDefault("bugsnag.endpoint", ""),
			AppVersion:      revel.Config.StringDefault("bugsnag.appversion", ""),
			ReleaseStage:    revel.Config.StringDefault("bugsnag.releasestage", revel.RunMode),
			ProjectPackages: projectPackages,
			Logger:          revel.ERROR,
		})
	})
}
<commit_msg>Also include app path in revel<commit_after>// bugsnagrevel adds Bugsnag to revel. It lets you pass *revel.Controller into bugsnag.Notify(),
// and provides a Filter to catch errors.
package bugsnagrevel

import (
	"github.com/bugsnag/bugsnag-go"
	"github.com/revel/revel"
	"strings"
	"sync"
)

var once sync.Once

// This Filter should be added to the filter chain just after the PanicFilter.
// It sends errors to Bugsnag automatically. Configuration is read out of
// conf/app.conf, you should set bugsnag.apikey, and can also set
// bugsnag.endpoint, bugsnag.releasestage, bugsnag.appversion,
// bugsnag.projectroot, bugsnag.projectpackages if needed.
func Filter(c *revel.Controller, fc []revel.Filter) {
	defer bugsnag.AutoNotify(c)
	fc[0](c, fc[1:])
}

// Add support to bugsnag for reading data out of *revel.Controllers
func middleware(event *bugsnag.Event, config *bugsnag.Configuration) bool {
	for _, datum := range event.RawData {
		if controller, ok := datum.(*revel.Controller); ok {
			// make the request visible to the builtin HttpMIddleware
			event.RawData = append(event.RawData, controller.Request.Request)
			event.Context = controller.Action
			event.MetaData.AddStruct("Session", controller.Session)
		}
	}

	return true
}

func init() {
	revel.OnAppStart(func () {
		bugsnag.OnBeforeNotify(middleware)

		var projectPackages []string
		if packages, ok := revel.Config.String("bugsnag.projectpackages"); ok {
			projectPackages = strings.Split(packages, ",")
		} else {
			projectPackages = []string{revel.ImportPath + "/app/*", revel.ImportPath + "/app"}
		}

		bugsnag.Configure(bugsnag.Configuration{
			APIKey:          revel.Config.StringDefault("bugsnag.apikey", ""),
			Endpoint:        revel.Config.StringDefault("bugsnag.endpoint", ""),
			AppVersion:      revel.Config.StringDefault("bugsnag.appversion", ""),
			ReleaseStage:    revel.Config.StringDefault("bugsnag.releasestage", revel.RunMode),
			ProjectPackages: projectPackages,
			Logger:          revel.ERROR,
		})
	})
}
<|endoftext|>


<commit_before>package vmwpatch

const VERSION = "4.2.2"
const COPYRIGHT = "© 2014-2022 David Parsons"
<commit_msg>4.2.3: Bump the version number.<commit_after>package vmwpatch

const VERSION = "4.2.3"
const COPYRIGHT = "© 2014-2022 David Parsons"
<|endoftext|>


<commit_before>/**
 * Copyright 2017 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package volume

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"strings"

	"github.com/IBM/ubiquity/resources"
	"github.com/kubernetes-incubator/external-storage/lib/controller"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/client-go/pkg/api/v1"
)

const (

	// are we allowed to set this? else make up our own
	annCreatedBy = "kubernetes.io/createdby"
	createdBy    = "ubiquity-provisioner"

	// Name of the file where an nfsProvisioner will store its identity
	identityFile = "ubiquity-provisioner.identity"

	// VolumeGidAnnotationKey is the key of the annotation on the PersistentVolume
	// object that specifies a supplemental GID.
	VolumeGidAnnotationKey = "pv.beta.kubernetes.io/gid"

	// A PV annotation for the identity of the flexProvisioner that provisioned it
	annProvisionerId = "Provisioner_Id"

	podIPEnv     = "POD_IP"
	serviceEnv   = "SERVICE_NAME"
	namespaceEnv = "POD_NAMESPACE"
	nodeEnv      = "NODE_NAME"
)

func NewFlexProvisioner(logger *log.Logger, ubiquityClient resources.StorageClient, config resources.UbiquityPluginConfig) (controller.Provisioner, error) {
	return newFlexProvisionerInternal(logger, ubiquityClient, config)
}

func newFlexProvisionerInternal(logger *log.Logger, ubiquityClient resources.StorageClient, config resources.UbiquityPluginConfig) (*flexProvisioner, error) {
	var identity types.UID
	identityPath := path.Join(config.LogPath, identityFile)
	if _, err := os.Stat(identityPath); os.IsNotExist(err) {
		identity = uuid.NewUUID()
		err := ioutil.WriteFile(identityPath, []byte(identity), 0600)
		if err != nil {
			logger.Printf("Error writing identity file %s! %v", identityPath, err)
		}
	} else {
		read, err := ioutil.ReadFile(identityPath)
		if err != nil {
			logger.Printf("Error reading identity file %s! %v", config.LogPath, err)
		}
		identity = types.UID(strings.TrimSpace(string(read)))
	}
	provisioner := &flexProvisioner{
		logger:         logger,
		identity:       identity,
		ubiquityClient: ubiquityClient,
		ubiquityConfig: config,
		podIPEnv:       podIPEnv,
		serviceEnv:     serviceEnv,
		namespaceEnv:   namespaceEnv,
		nodeEnv:        nodeEnv,
	}

	activateRequest := resources.ActivateRequest{Backends: config.Backends}
	err := provisioner.ubiquityClient.Activate(activateRequest)

	return provisioner, err
}

type flexProvisioner struct {
	logger   *log.Logger
	identity types.UID
	// Whether the provisioner is running out of cluster and so cannot rely on
	// the existence of any of the pod, service, namespace, node env variables.
	outOfCluster bool

	ubiquityClient resources.StorageClient
	ubiquityConfig resources.UbiquityPluginConfig

	// Environment variables the provisioner pod needs valid values for in order to
	// put a service cluster IP as the server of provisioned NFS PVs, passed in
	// via downward API. If serviceEnv is set, namespaceEnv must be too.
	podIPEnv     string
	serviceEnv   string
	namespaceEnv string
	nodeEnv      string
}

// Provision creates a volume i.e. the storage asset and returns a PV object for
// the volume.
func (p *flexProvisioner) Provision(options controller.VolumeOptions) (*v1.PersistentVolume, error) {
	if options.PVC == nil {
		return nil, fmt.Errorf("options missing PVC %#v", options)
	}
	capacity, exists := options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
	if !exists {
		return nil, fmt.Errorf("options.PVC.Spec.Resources.Requests does not contain capacity")
	}
	fmt.Printf("PVC with capacity %d", capacity.Value())
	capacityMB := capacity.Value() / (1024 * 1024)

	volume_details, err := p.createVolume(options, capacityMB)
	if err != nil {
		return nil, err
	}

	annotations := make(map[string]string)
	annotations[annCreatedBy] = createdBy
	annotations[annProvisionerId] = "ubiquity-provisioner"

	pv := &v1.PersistentVolume{
		ObjectMeta: v1.ObjectMeta{
			Name:        options.PVName,
			Labels:      map[string]string{},
			Annotations: annotations,
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeReclaimPolicy: options.PersistentVolumeReclaimPolicy,
			AccessModes:                   options.PVC.Spec.AccessModes,
			Capacity: v1.ResourceList{
				v1.ResourceName(v1.ResourceStorage): options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)],
			},
			PersistentVolumeSource: v1.PersistentVolumeSource{
				FlexVolume: &v1.FlexVolumeSource{
					Driver:    "ibm/ubiquity",
					FSType:    "",
					SecretRef: nil,
					ReadOnly:  false,
					Options:   volume_details,
				},
			},
		},
	}

	return pv, nil
}

// Delete removes the directory that was created by Provision backing the given
// PV.
func (p *flexProvisioner) Delete(volume *v1.PersistentVolume) error {
	if volume.Name == "" {
		return fmt.Errorf("volume name cannot be empty %#v", volume)
	}

	if volume.Spec.PersistentVolumeReclaimPolicy != v1.PersistentVolumeReclaimRetain {
		getVolumeRequest := resources.GetVolumeRequest{Name: volume.Name}
		volume, err := p.ubiquityClient.GetVolume(getVolumeRequest)
		if err != nil {
			fmt.Printf("error-retrieving-volume-info")
			return err
		}
		removeVolumeRequest := resources.RemoveVolumeRequest{Name: volume.Name}
		err = p.ubiquityClient.RemoveVolume(removeVolumeRequest)
		if err != nil {
			return err
		}
		return nil

	}

	return nil
}

func (p *flexProvisioner) createVolume(options controller.VolumeOptions, capacity int64) (map[string]string, error) {
	ubiquityParams := make(map[string]interface{})
	if capacity != 0 {
		ubiquityParams["quota"] = fmt.Sprintf("%dM", capacity)    // SSc backend expect quota option
		ubiquityParams["size"] = fmt.Sprintf("%d", capacity*1024) // SCBE backend expect size option
	}
	for key, value := range options.Parameters {
		ubiquityParams[key] = value
	}
	backendName, exists := ubiquityParams["backend"]
	if !exists {
		return nil, fmt.Errorf("backend is not specified")
	}
	b := backendName.(string)
	createVolumeRequest := resources.CreateVolumeRequest{Name: options.PVName, Backend: b, Opts: ubiquityParams}
	err := p.ubiquityClient.CreateVolume(createVolumeRequest)
	if err != nil {
		return nil, fmt.Errorf("error creating volume: %v", err)
	}

	getVolumeConfigRequest := resources.GetVolumeConfigRequest{Name: options.PVName}
	volumeConfig, err := p.ubiquityClient.GetVolumeConfig(getVolumeConfigRequest)
	if err != nil {
		return nil, fmt.Errorf("error getting volume config details: %v", err)
	}

	flexVolumeConfig := make(map[string]string)
	flexVolumeConfig["volumeName"] = options.PVName
	for key, value := range volumeConfig {
		flexVolumeConfig[key] = fmt.Sprintf("%v", value)
	}

	return flexVolumeConfig, nil
}
<commit_msg>#34 : fix size for GiB<commit_after>/**
 * Copyright 2017 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package volume

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"strings"

	"github.com/IBM/ubiquity/resources"
	"github.com/kubernetes-incubator/external-storage/lib/controller"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/client-go/pkg/api/v1"
)

const (

	// are we allowed to set this? else make up our own
	annCreatedBy = "kubernetes.io/createdby"
	createdBy    = "ubiquity-provisioner"

	// Name of the file where an nfsProvisioner will store its identity
	identityFile = "ubiquity-provisioner.identity"

	// VolumeGidAnnotationKey is the key of the annotation on the PersistentVolume
	// object that specifies a supplemental GID.
	VolumeGidAnnotationKey = "pv.beta.kubernetes.io/gid"

	// A PV annotation for the identity of the flexProvisioner that provisioned it
	annProvisionerId = "Provisioner_Id"

	podIPEnv     = "POD_IP"
	serviceEnv   = "SERVICE_NAME"
	namespaceEnv = "POD_NAMESPACE"
	nodeEnv      = "NODE_NAME"
)

func NewFlexProvisioner(logger *log.Logger, ubiquityClient resources.StorageClient, config resources.UbiquityPluginConfig) (controller.Provisioner, error) {
	return newFlexProvisionerInternal(logger, ubiquityClient, config)
}

func newFlexProvisionerInternal(logger *log.Logger, ubiquityClient resources.StorageClient, config resources.UbiquityPluginConfig) (*flexProvisioner, error) {
	var identity types.UID
	identityPath := path.Join(config.LogPath, identityFile)
	if _, err := os.Stat(identityPath); os.IsNotExist(err) {
		identity = uuid.NewUUID()
		err := ioutil.WriteFile(identityPath, []byte(identity), 0600)
		if err != nil {
			logger.Printf("Error writing identity file %s! %v", identityPath, err)
		}
	} else {
		read, err := ioutil.ReadFile(identityPath)
		if err != nil {
			logger.Printf("Error reading identity file %s! %v", config.LogPath, err)
		}
		identity = types.UID(strings.TrimSpace(string(read)))
	}
	provisioner := &flexProvisioner{
		logger:         logger,
		identity:       identity,
		ubiquityClient: ubiquityClient,
		ubiquityConfig: config,
		podIPEnv:       podIPEnv,
		serviceEnv:     serviceEnv,
		namespaceEnv:   namespaceEnv,
		nodeEnv:        nodeEnv,
	}

	activateRequest := resources.ActivateRequest{Backends: config.Backends}
	err := provisioner.ubiquityClient.Activate(activateRequest)

	return provisioner, err
}

type flexProvisioner struct {
	logger   *log.Logger
	identity types.UID
	// Whether the provisioner is running out of cluster and so cannot rely on
	// the existence of any of the pod, service, namespace, node env variables.
	outOfCluster bool

	ubiquityClient resources.StorageClient
	ubiquityConfig resources.UbiquityPluginConfig

	// Environment variables the provisioner pod needs valid values for in order to
	// put a service cluster IP as the server of provisioned NFS PVs, passed in
	// via downward API. If serviceEnv is set, namespaceEnv must be too.
	podIPEnv     string
	serviceEnv   string
	namespaceEnv string
	nodeEnv      string
}

// Provision creates a volume i.e. the storage asset and returns a PV object for
// the volume.
func (p *flexProvisioner) Provision(options controller.VolumeOptions) (*v1.PersistentVolume, error) {
	if options.PVC == nil {
		return nil, fmt.Errorf("options missing PVC %#v", options)
	}
	capacity, exists := options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
	if !exists {
		return nil, fmt.Errorf("options.PVC.Spec.Resources.Requests does not contain capacity")
	}
	fmt.Printf("PVC with capacity %d", capacity.Value())
	capacityMB := capacity.Value() / (1024 * 1024)

	volume_details, err := p.createVolume(options, capacityMB)
	if err != nil {
		return nil, err
	}

	annotations := make(map[string]string)
	annotations[annCreatedBy] = createdBy
	annotations[annProvisionerId] = "ubiquity-provisioner"

	pv := &v1.PersistentVolume{
		ObjectMeta: v1.ObjectMeta{
			Name:        options.PVName,
			Labels:      map[string]string{},
			Annotations: annotations,
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeReclaimPolicy: options.PersistentVolumeReclaimPolicy,
			AccessModes:                   options.PVC.Spec.AccessModes,
			Capacity: v1.ResourceList{
				v1.ResourceName(v1.ResourceStorage): options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)],
			},
			PersistentVolumeSource: v1.PersistentVolumeSource{
				FlexVolume: &v1.FlexVolumeSource{
					Driver:    "ibm/ubiquity",
					FSType:    "",
					SecretRef: nil,
					ReadOnly:  false,
					Options:   volume_details,
				},
			},
		},
	}

	return pv, nil
}

// Delete removes the directory that was created by Provision backing the given
// PV.
func (p *flexProvisioner) Delete(volume *v1.PersistentVolume) error {
	if volume.Name == "" {
		return fmt.Errorf("volume name cannot be empty %#v", volume)
	}

	if volume.Spec.PersistentVolumeReclaimPolicy != v1.PersistentVolumeReclaimRetain {
		getVolumeRequest := resources.GetVolumeRequest{Name: volume.Name}
		volume, err := p.ubiquityClient.GetVolume(getVolumeRequest)
		if err != nil {
			fmt.Printf("error-retrieving-volume-info")
			return err
		}
		removeVolumeRequest := resources.RemoveVolumeRequest{Name: volume.Name}
		err = p.ubiquityClient.RemoveVolume(removeVolumeRequest)
		if err != nil {
			return err
		}
		return nil

	}

	return nil
}

func (p *flexProvisioner) createVolume(options controller.VolumeOptions, capacity int64) (map[string]string, error) {
	ubiquityParams := make(map[string]interface{})
	if capacity != 0 {
		ubiquityParams["quota"] = fmt.Sprintf("%dM", capacity)    // SSc backend expect quota option
		ubiquityParams["size"] = fmt.Sprintf("%d", capacity/1024) // SCBE backend expect size option
	}
	for key, value := range options.Parameters {
		ubiquityParams[key] = value
	}
	backendName, exists := ubiquityParams["backend"]
	if !exists {
		return nil, fmt.Errorf("backend is not specified")
	}
	b := backendName.(string)
	createVolumeRequest := resources.CreateVolumeRequest{Name: options.PVName, Backend: b, Opts: ubiquityParams}
	err := p.ubiquityClient.CreateVolume(createVolumeRequest)
	if err != nil {
		return nil, fmt.Errorf("error creating volume: %v", err)
	}

	getVolumeConfigRequest := resources.GetVolumeConfigRequest{Name: options.PVName}
	volumeConfig, err := p.ubiquityClient.GetVolumeConfig(getVolumeConfigRequest)
	if err != nil {
		return nil, fmt.Errorf("error getting volume config details: %v", err)
	}

	flexVolumeConfig := make(map[string]string)
	flexVolumeConfig["volumeName"] = options.PVName
	for key, value := range volumeConfig {
		flexVolumeConfig[key] = fmt.Sprintf("%v", value)
	}

	return flexVolumeConfig, nil
}
<|endoftext|>