<commit_before>package main

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"

	"github.com/ghodss/yaml"
	"github.com/jessevdk/go-flags"
	"github.com/lestrrat/go-jshschema"
	"github.com/minodisk/go-jstmpl"
)

func main() {
	os.Exit(_main())
}

type options struct {
	Schema   string `short:"s" long:"schema" description:"the source JSON Schema file"`
	OutDir   string `short:"o" long:"outfile" description:"output file to generate"`
	Template string `short:"t" log:"tmpl" description:"template file to generate document"`
}

func _main() int {
	var opts options
	if _, err := flags.Parse(&opts); err != nil {
		log.Printf("fail to parse flags: %s", err)
		return 1
	}

	f, err := os.Open(opts.Schema)
	if err != nil {
		log.Printf("fail to open the source JSON Schema file: %s", err)
		return 1
	}
	defer f.Close()

	var m map[string]interface{}
	switch ext := filepath.Ext(opts.Schema); ext {
	case ".json":
		if err := json.NewDecoder(f).Decode(&m); err != nil {
			log.Printf("fail to decode JSON: %s", err)
			return 1
		}
	case ".yml", ".yaml":
		b, err := ioutil.ReadFile(opts.Schema)
		if err != nil {
			log.Printf("fail to read the source JSON Schema file: %s", err)
			return 1
		}
		if err := yaml.Unmarshal(b, &m); err != nil {
			log.Printf("fail to unmarshal YAML: %s", err)
			return 1
		}
	default:
		log.Printf("undefined extension: %s", ext)
		return 1
	}

	hs := hschema.New()
	if err := hs.Extract(m); err != nil {
		log.Printf("fail to extract JSON Schema: %s", err)
		return 1
	}

	b := jstmpl.NewBuilder()
	ts, err := b.Build(hs)
	if err != nil {
		log.Printf("fail to build: %s", err)
		return 1
	}

	err = filepath.Walk(opts.Template, func(i string, info os.FileInfo, err error) error {
		if info.IsDir() {
			return nil
		}
		r, err := filepath.Rel(opts.Template, i)
		if err != nil {
			return err
		}
		o := filepath.Join(opts.OutDir, r)

		var tmpl []byte
		if i != "" {
			f, err := os.Open(i)
			if err != nil {
				return err
			}
			defer f.Close()
			tmpl, err = ioutil.ReadAll(f)
			if err != nil {
				return err
			}
		}

		var out io.Writer
		if o != "" {
			d := filepath.Dir(o)
			_, err := os.Stat(d)
			if err != nil {
				err := os.MkdirAll(d, 0755)
				if err != nil {
					return err
				}
			}
			f, err := os.Create(o)
			if err != nil {
				return err
			}
			defer f.Close()
			out = f
		}

		g := jstmpl.NewGenerator()
		if err := g.Process(out, ts, tmpl); err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		log.Printf("fail to generate: %s", err)
		return 1
	}

	return 0
}
<commit_msg>Update package name to go-jstmpl/go-jstmpl<commit_after>package main

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"

	"github.com/ghodss/yaml"
	"github.com/go-jstmpl/go-jstmpl"
	"github.com/jessevdk/go-flags"
	"github.com/lestrrat/go-jshschema"
)

func main() {
	os.Exit(_main())
}

type options struct {
	Schema   string `short:"s" long:"schema" description:"the source JSON Schema file"`
	OutDir   string `short:"o" long:"outfile" description:"output file to generate"`
	Template string `short:"t" log:"tmpl" description:"template file to generate document"`
}

func _main() int {
	var opts options
	if _, err := flags.Parse(&opts); err != nil {
		log.Printf("fail to parse flags: %s", err)
		return 1
	}

	f, err := os.Open(opts.Schema)
	if err != nil {
		log.Printf("fail to open the source JSON Schema file: %s", err)
		return 1
	}
	defer f.Close()

	var m map[string]interface{}
	switch ext := filepath.Ext(opts.Schema); ext {
	case ".json":
		if err := json.NewDecoder(f).Decode(&m); err != nil {
			log.Printf("fail to decode JSON: %s", err)
			return 1
		}
	case ".yml", ".yaml":
		b, err := ioutil.ReadFile(opts.Schema)
		if err != nil {
			log.Printf("fail to read the source JSON Schema file: %s", err)
			return 1
		}
		if err := yaml.Unmarshal(b, &m); err != nil {
			log.Printf("fail to unmarshal YAML: %s", err)
			return 1
		}
	default:
		log.Printf("undefined extension: %s", ext)
		return 1
	}

	hs := hschema.New()
	if err := hs.Extract(m); err != nil {
		log.Printf("fail to extract JSON Schema: %s", err)
		return 1
	}

	b := jstmpl.NewBuilder()
	ts, err := b.Build(hs)
	if err != nil {
		log.Printf("fail to build: %s", err)
		return 1
	}

	err = filepath.Walk(opts.Template, func(i string, info os.FileInfo, err error) error {
		if info.IsDir() {
			return nil
		}
		r, err := filepath.Rel(opts.Template, i)
		if err != nil {
			return err
		}
		o := filepath.Join(opts.OutDir, r)

		var tmpl []byte
		if i != "" {
			f, err := os.Open(i)
			if err != nil {
				return err
			}
			defer f.Close()
			tmpl, err = ioutil.ReadAll(f)
			if err != nil {
				return err
			}
		}

		var out io.Writer
		if o != "" {
			d := filepath.Dir(o)
			_, err := os.Stat(d)
			if err != nil {
				err := os.MkdirAll(d, 0755)
				if err != nil {
					return err
				}
			}
			f, err := os.Create(o)
			if err != nil {
				return err
			}
			defer f.Close()
			out = f
		}

		g := jstmpl.NewGenerator()
		if err := g.Process(out, ts, tmpl); err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		log.Printf("fail to generate: %s", err)
		return 1
	}

	return 0
}
<|endoftext|>


<commit_before>// Copyright 2015 Richard Lehane. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"io"
	"path/filepath"
	"strings"
	"time"

	"archive/tar"
	"archive/zip"
	"compress/gzip"

	"github.com/richardlehane/characterize"
	"github.com/richardlehane/webarchive"

	"github.com/richardlehane/siegfried/pkg/core/siegreader"
)

type decompressor interface {
	next() error // when finished, should return io.EOF
	reader() io.Reader
	path() string
	mime() string
	size() int64
	mod() string
	dirs() []string
}

type zipD struct {
	idx     int
	p       string
	rdr     *zip.Reader
	rc      io.ReadCloser
	written map[string]bool
}

func newZip(ra io.ReaderAt, path string, sz int64) (decompressor, error) {
	zr, err := zip.NewReader(ra, sz)
	return &zipD{idx: -1, p: path, rdr: zr}, err
}

func (z *zipD) close() {
	if z.rc == nil {
		return
	}
	z.rc.Close()
}

func (z *zipD) next() error {
	z.close() // close the previous entry, if any
	// proceed
	z.idx++
	// scan past directories
	for ; z.idx < len(z.rdr.File) && z.rdr.File[z.idx].FileInfo().IsDir(); z.idx++ {
	}
	if z.idx >= len(z.rdr.File) {
		return io.EOF
	}
	var err error
	z.rc, err = z.rdr.File[z.idx].Open()
	return err
}

func (z *zipD) reader() io.Reader {
	return z.rc
}

func (z *zipD) path() string {
	return arcpath(z.p, filepath.FromSlash(characterize.ZipName(z.rdr.File[z.idx].Name)))
}

func (z *zipD) mime() string {
	return ""
}

func (z *zipD) size() int64 {
	return int64(z.rdr.File[z.idx].UncompressedSize64)
}

func (z *zipD) mod() string {
	return z.rdr.File[z.idx].ModTime().Format(time.RFC3339)
}

func (z *zipD) dirs() []string {
	if z.written == nil {
		z.written = make(map[string]bool)
	}
	return dirs(z.p, characterize.ZipName(z.rdr.File[z.idx].Name), z.written)
}

type tarD struct {
	p       string
	hdr     *tar.Header
	rdr     *tar.Reader
	written map[string]bool
}

func newTar(r io.Reader, path string) (decompressor, error) {
	return &tarD{p: path, rdr: tar.NewReader(r)}, nil
}

func (t *tarD) next() error {
	var err error
	// scan past directories
	for t.hdr, err = t.rdr.Next(); err == nil && t.hdr.FileInfo().IsDir(); t.hdr, err = t.rdr.Next() {
	}
	return err
}

func (t *tarD) reader() io.Reader {
	return t.rdr
}

func (t *tarD) path() string {
	return arcpath(t.p, filepath.FromSlash(t.hdr.Name))
}

func (t *tarD) mime() string {
	return ""
}

func (t *tarD) size() int64 {
	return t.hdr.Size
}

func (t *tarD) mod() string {
	return t.hdr.ModTime.Format(time.RFC3339)
}

func (t *tarD) dirs() []string {
	if t.written == nil {
		t.written = make(map[string]bool)
	}
	return dirs(t.p, t.hdr.Name, t.written)
}

type gzipD struct {
	sz   int64
	p    string
	read bool
	rdr  *gzip.Reader
}

func newGzip(b *siegreader.Buffer, path string) (decompressor, error) {
	_ = b.SizeNow()              // in case of a stream, force full read
	buf, err := b.EofSlice(0, 4) // gzip stores uncompressed size in last 4 bytes of the stream
	if err != nil {
		return nil, err
	}
	sz := int64(uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24)
	g, err := gzip.NewReader(siegreader.ReaderFrom(b))
	return &gzipD{sz: sz, p: path, rdr: g}, err
}

func (g *gzipD) next() error {
	if g.read {
		g.rdr.Close()
		return io.EOF
	}
	g.read = true
	return nil
}

func (g *gzipD) reader() io.Reader {
	return g.rdr
}

func (g *gzipD) path() string {
	name := g.rdr.Name
	if len(name) == 0 {
		switch filepath.Ext(g.p) {
		case ".gz", ".z", ".gzip", ".zip":
			name = strings.TrimSuffix(filepath.Base(g.p), filepath.Ext(g.p))
		default:
			name = filepath.Base(g.p)
		}
	}
	return arcpath(g.p, name)
}

func (g *gzipD) mime() string {
	return ""
}

func (g *gzipD) size() int64 {
	return g.sz
}

func (g *gzipD) mod() string {
	return g.rdr.ModTime.Format(time.RFC3339)
}

func (t *gzipD) dirs() []string {
	return nil
}

func trimWebPath(p string) string {
	d, f := filepath.Split(p)
	clean := strings.TrimSuffix(d, string(filepath.Separator))
	_, f1 := filepath.Split(clean)
	if f == strings.TrimSuffix(f1, filepath.Ext(clean)) {
		return clean
	}
	return p
}

var (
	arcReader  *webarchive.ARCReader
	warcReader *webarchive.WARCReader
)

type wa struct {
	p   string
	rec webarchive.Record
	rdr webarchive.Reader
}

func newARC(r io.Reader, path string) (decompressor, error) {
	var err error
	if arcReader == nil {
		arcReader, err = webarchive.NewARCReader(r)
	} else {
		err = arcReader.Reset(r)
	}
	return &wa{p: trimWebPath(path), rdr: arcReader}, err
}

func newWARC(r io.Reader, path string) (decompressor, error) {
	var err error
	if warcReader == nil {
		warcReader, err = webarchive.NewWARCReader(r)
	} else {
		err = warcReader.Reset(r)
	}
	return &wa{p: trimWebPath(path), rdr: warcReader}, err
}

func (w *wa) next() error {
	var err error
	w.rec, err = w.rdr.NextPayload()
	return err
}

func (w *wa) reader() io.Reader {
	return webarchive.DecodePayload(w.rec)
}

func (w *wa) path() string {
	return arcpath(w.p, w.rec.Date().Format(webarchive.ARCTime)+'/'+w.rec.URL())
}

func (w *wa) mime() string {
	return w.rec.MIME()
}

func (w *wa) size() int64 {
	return w.rec.Size()
}

func (w *wa) mod() string {
	return w.rec.Date().Format(time.RFC3339)
}

func (w *wa) dirs() []string {
	return nil
}

func dirs(path, name string, written map[string]bool) []string {
	ds := strings.Split(filepath.ToSlash(name), "/")
	if len(ds) > 1 {
		var ret []string
		for _, p := range ds[:len(ds)-1] {
			path = path + string(filepath.Separator) + p
			if !written[path] {
				ret = append(ret, path)
				written[path] = true
			}
		}
		return ret
	}
	return nil
}

// per https://github.com/richardlehane/siegfried/issues/81
// construct paths for compressed objects acc. to KDE hash notation
func arcpath(base, path string) string {
	if *droido {
		return base + string(filepath.Separator) + path
	}
	return base + "#" + path
}
<commit_msg>string not char<commit_after>// Copyright 2015 Richard Lehane. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"io"
	"path/filepath"
	"strings"
	"time"

	"archive/tar"
	"archive/zip"
	"compress/gzip"

	"github.com/richardlehane/characterize"
	"github.com/richardlehane/webarchive"

	"github.com/richardlehane/siegfried/pkg/core/siegreader"
)

type decompressor interface {
	next() error // when finished, should return io.EOF
	reader() io.Reader
	path() string
	mime() string
	size() int64
	mod() string
	dirs() []string
}

type zipD struct {
	idx     int
	p       string
	rdr     *zip.Reader
	rc      io.ReadCloser
	written map[string]bool
}

func newZip(ra io.ReaderAt, path string, sz int64) (decompressor, error) {
	zr, err := zip.NewReader(ra, sz)
	return &zipD{idx: -1, p: path, rdr: zr}, err
}

func (z *zipD) close() {
	if z.rc == nil {
		return
	}
	z.rc.Close()
}

func (z *zipD) next() error {
	z.close() // close the previous entry, if any
	// proceed
	z.idx++
	// scan past directories
	for ; z.idx < len(z.rdr.File) && z.rdr.File[z.idx].FileInfo().IsDir(); z.idx++ {
	}
	if z.idx >= len(z.rdr.File) {
		return io.EOF
	}
	var err error
	z.rc, err = z.rdr.File[z.idx].Open()
	return err
}

func (z *zipD) reader() io.Reader {
	return z.rc
}

func (z *zipD) path() string {
	return arcpath(z.p, filepath.FromSlash(characterize.ZipName(z.rdr.File[z.idx].Name)))
}

func (z *zipD) mime() string {
	return ""
}

func (z *zipD) size() int64 {
	return int64(z.rdr.File[z.idx].UncompressedSize64)
}

func (z *zipD) mod() string {
	return z.rdr.File[z.idx].ModTime().Format(time.RFC3339)
}

func (z *zipD) dirs() []string {
	if z.written == nil {
		z.written = make(map[string]bool)
	}
	return dirs(z.p, characterize.ZipName(z.rdr.File[z.idx].Name), z.written)
}

type tarD struct {
	p       string
	hdr     *tar.Header
	rdr     *tar.Reader
	written map[string]bool
}

func newTar(r io.Reader, path string) (decompressor, error) {
	return &tarD{p: path, rdr: tar.NewReader(r)}, nil
}

func (t *tarD) next() error {
	var err error
	// scan past directories
	for t.hdr, err = t.rdr.Next(); err == nil && t.hdr.FileInfo().IsDir(); t.hdr, err = t.rdr.Next() {
	}
	return err
}

func (t *tarD) reader() io.Reader {
	return t.rdr
}

func (t *tarD) path() string {
	return arcpath(t.p, filepath.FromSlash(t.hdr.Name))
}

func (t *tarD) mime() string {
	return ""
}

func (t *tarD) size() int64 {
	return t.hdr.Size
}

func (t *tarD) mod() string {
	return t.hdr.ModTime.Format(time.RFC3339)
}

func (t *tarD) dirs() []string {
	if t.written == nil {
		t.written = make(map[string]bool)
	}
	return dirs(t.p, t.hdr.Name, t.written)
}

type gzipD struct {
	sz   int64
	p    string
	read bool
	rdr  *gzip.Reader
}

func newGzip(b *siegreader.Buffer, path string) (decompressor, error) {
	_ = b.SizeNow()              // in case of a stream, force full read
	buf, err := b.EofSlice(0, 4) // gzip stores uncompressed size in last 4 bytes of the stream
	if err != nil {
		return nil, err
	}
	sz := int64(uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24)
	g, err := gzip.NewReader(siegreader.ReaderFrom(b))
	return &gzipD{sz: sz, p: path, rdr: g}, err
}

func (g *gzipD) next() error {
	if g.read {
		g.rdr.Close()
		return io.EOF
	}
	g.read = true
	return nil
}

func (g *gzipD) reader() io.Reader {
	return g.rdr
}

func (g *gzipD) path() string {
	name := g.rdr.Name
	if len(name) == 0 {
		switch filepath.Ext(g.p) {
		case ".gz", ".z", ".gzip", ".zip":
			name = strings.TrimSuffix(filepath.Base(g.p), filepath.Ext(g.p))
		default:
			name = filepath.Base(g.p)
		}
	}
	return arcpath(g.p, name)
}

func (g *gzipD) mime() string {
	return ""
}

func (g *gzipD) size() int64 {
	return g.sz
}

func (g *gzipD) mod() string {
	return g.rdr.ModTime.Format(time.RFC3339)
}

func (t *gzipD) dirs() []string {
	return nil
}

func trimWebPath(p string) string {
	d, f := filepath.Split(p)
	clean := strings.TrimSuffix(d, string(filepath.Separator))
	_, f1 := filepath.Split(clean)
	if f == strings.TrimSuffix(f1, filepath.Ext(clean)) {
		return clean
	}
	return p
}

var (
	arcReader  *webarchive.ARCReader
	warcReader *webarchive.WARCReader
)

type wa struct {
	p   string
	rec webarchive.Record
	rdr webarchive.Reader
}

func newARC(r io.Reader, path string) (decompressor, error) {
	var err error
	if arcReader == nil {
		arcReader, err = webarchive.NewARCReader(r)
	} else {
		err = arcReader.Reset(r)
	}
	return &wa{p: trimWebPath(path), rdr: arcReader}, err
}

func newWARC(r io.Reader, path string) (decompressor, error) {
	var err error
	if warcReader == nil {
		warcReader, err = webarchive.NewWARCReader(r)
	} else {
		err = warcReader.Reset(r)
	}
	return &wa{p: trimWebPath(path), rdr: warcReader}, err
}

func (w *wa) next() error {
	var err error
	w.rec, err = w.rdr.NextPayload()
	return err
}

func (w *wa) reader() io.Reader {
	return webarchive.DecodePayload(w.rec)
}

func (w *wa) path() string {
	return arcpath(w.p, w.rec.Date().Format(webarchive.ARCTime)+"/"+w.rec.URL())
}

func (w *wa) mime() string {
	return w.rec.MIME()
}

func (w *wa) size() int64 {
	return w.rec.Size()
}

func (w *wa) mod() string {
	return w.rec.Date().Format(time.RFC3339)
}

func (w *wa) dirs() []string {
	return nil
}

func dirs(path, name string, written map[string]bool) []string {
	ds := strings.Split(filepath.ToSlash(name), "/")
	if len(ds) > 1 {
		var ret []string
		for _, p := range ds[:len(ds)-1] {
			path = path + string(filepath.Separator) + p
			if !written[path] {
				ret = append(ret, path)
				written[path] = true
			}
		}
		return ret
	}
	return nil
}

// per https://github.com/richardlehane/siegfried/issues/81
// construct paths for compressed objects acc. to KDE hash notation
func arcpath(base, path string) string {
	if *droido {
		return base + string(filepath.Separator) + path
	}
	return base + "#" + path
}
<|endoftext|>


<commit_before>// Copyright (c) 2012-2019 Eli Janssen
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

// url-tool
package main

import (
	"errors"
	"fmt"
	"net/url"
	"os"
	"strings"

	"github.com/cactus/go-camo/pkg/camo/encoding"

	flags "github.com/jessevdk/go-flags"
)

// EncodeCommand holds command options for the encode command
type EncodeCommand struct {
	Base   string `short:"b" long:"base" default:"hex" description:"Encode/Decode base. Either hex or base64"`
	Prefix string `short:"p" long:"prefix" default:"" description:"Optional url prefix used by encode output"`
}

// Execute runs the encode command
func (c *EncodeCommand) Execute(args []string) error {
	if opts.HmacKey == "" {
		return errors.New("empty HMAC")
	}

	if len(args) == 0 {
		return errors.New("no url argument provided")
	}

	oURL := args[0]
	if oURL == "" {
		return errors.New("no url argument provided")
	}

	hmacKeyBytes := []byte(opts.HmacKey)
	var outURL string
	switch c.Base {
	case "base64":
		outURL = encoding.B64EncodeURL(hmacKeyBytes, oURL)
	case "hex":
		outURL = encoding.HexEncodeURL(hmacKeyBytes, oURL)
	default:
		return errors.New("invalid base provided")
	}
	fmt.Println(c.Prefix + outURL)
	return nil
}

// DecodeCommand holds command options for the decode command
type DecodeCommand struct{}

// Execute runs the decode command
func (c *DecodeCommand) Execute(args []string) error {
	if opts.HmacKey == "" {
		return errors.New("empty HMAC")
	}

	if len(args) == 0 {
		return errors.New("no url argument provided")
	}

	oURL := args[0]
	if oURL == "" {
		return errors.New("no url argument provided")
	}

	hmacKeyBytes := []byte(opts.HmacKey)

	u, err := url.Parse(oURL)
	if err != nil {
		return err
	}
	comp := strings.SplitN(u.Path, "/", 3)
	decURL, valid := encoding.DecodeURL(hmacKeyBytes, comp[1], comp[2])
	if !valid {
		return errors.New("hmac is invalid")
	}
	fmt.Println(decURL)
	return nil
}

var opts struct {
	HmacKey string `short:"k" long:"key" description:"HMAC key"`
}

// #nosec G104
func main() {
	parser := flags.NewParser(&opts, flags.Default)
	parser.AddCommand("encode", "Encode a url and print result",
		"Encode a url and print result", &EncodeCommand{})
	parser.AddCommand("decode", "Decode a url and print result",
		"Decode a url and print result", &DecodeCommand{})

	// parse said flags
	_, err := parser.Parse()
	if err != nil {
		if e, ok := err.(*flags.Error); ok {
			if e.Type == flags.ErrHelp {
				os.Exit(0)
			}
		}
		os.Exit(1)
	}
}
<commit_msg>update url-tool with better help output<commit_after>// Copyright (c) 2012-2019 Eli Janssen
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

// url-tool
package main

import (
	"errors"
	"fmt"
	"net/url"
	"os"
	"strings"

	"github.com/cactus/go-camo/pkg/camo/encoding"

	flags "github.com/jessevdk/go-flags"
)

// EncodeCommand holds command options for the encode command
type EncodeCommand struct {
	Base       string `short:"b" long:"base" default:"hex" description:"Encode/Decode base. Either hex or base64"`
	Prefix     string `short:"p" long:"prefix" default:"" description:"Optional url prefix used by encode output"`
	Positional struct {
		Url string `positional-arg-name:"URL"`
	} `positional-args:"yes" required:"true"`
}

// Execute runs the encode command
func (c *EncodeCommand) Execute(args []string) error {
	if opts.HmacKey == "" {
		return errors.New("empty HMAC")
	}

	if len(c.Positional.Url) == 0 {
		return errors.New("no url argument provided")
	}

	hmacKeyBytes := []byte(opts.HmacKey)
	var outURL string
	switch c.Base {
	case "base64":
		outURL = encoding.B64EncodeURL(hmacKeyBytes, c.Positional.Url)
	case "hex":
		outURL = encoding.HexEncodeURL(hmacKeyBytes, c.Positional.Url)
	default:
		return errors.New("invalid base provided")
	}
	fmt.Println(strings.TrimRight(c.Prefix, "/") + outURL)
	return nil
}

// DecodeCommand holds command options for the decode command
type DecodeCommand struct {
	Positional struct {
		Url string `positional-arg-name:"URL"`
	} `positional-args:"yes" required:"true"`
}

// Execute runs the decode command
func (c *DecodeCommand) Execute(args []string) error {
	if opts.HmacKey == "" {
		return errors.New("empty HMAC")
	}

	if len(c.Positional.Url) == 0 {
		return errors.New("no url argument provided")
	}

	hmacKeyBytes := []byte(opts.HmacKey)

	u, err := url.Parse(c.Positional.Url)
	if err != nil {
		return err
	}
	comp := strings.SplitN(u.Path, "/", 3)
	decURL, valid := encoding.DecodeURL(hmacKeyBytes, comp[1], comp[2])
	if !valid {
		return errors.New("hmac is invalid")
	}
	fmt.Println(decURL)
	return nil
}

var opts struct {
	HmacKey string `short:"k" long:"key" description:"HMAC key"`
}

// #nosec G104
func main() {
	parser := flags.NewParser(&opts, flags.Default)
	parser.AddCommand("encode", "Encode a url and print result",
		"Encode a url and print result", &EncodeCommand{})
	parser.AddCommand("decode", "Decode a url and print result",
		"Decode a url and print result", &DecodeCommand{})

	// parse said flags
	_, err := parser.Parse()
	if err != nil {
		if e, ok := err.(*flags.Error); ok {
			if e.Type == flags.ErrHelp {
				os.Exit(0)
			}
		}
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package forge

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"
)

type Section struct {
	parent *Section
	values map[string]Value
}

func NewSection() *Section {
	return &Section{
		values: make(map[string]Value),
	}
}

func NewChildSection(parent *Section) *Section {
	return &Section{
		parent: parent,
		values: make(map[string]Value),
	}
}

func (section *Section) GetType() ValueType {
	return SECTION
}

func (section *Section) GetValue() interface{} {
	return section.values
}

func (section *Section) UpdateValue(value interface{}) error {
	switch value.(type) {
	case map[string]Value:
		section.values = value.(map[string]Value)
		return nil
	}

	msg := fmt.Sprintf("Unsupported type, %s must be of type `map[string]Value`", value)
	return errors.New(msg)
}

func (section *Section) AddSection(name string) *Section {
	childSection := NewChildSection(section)
	section.values[name] = childSection
	return childSection
}

func (section *Section) Exists(name string) bool {
	_, err := section.Get(name)
	return err == nil
}

func (section *Section) Get(name string) (Value, error) {
	value, ok := section.values[name]
	var err error
	if ok == false {
		err = errors.New("Value does not exist")
	}
	return value, err
}

func (section *Section) GetBoolean(name string) (bool, error) {
	value, err := section.Get(name)
	if err != nil {
		return false, err
	}

	switch value.(type) {
	case *Primative:
		return value.(*Primative).AsBoolean()
	case *Section:
		return true, nil
	}

	return false, errors.New("Could not convert unknown value to boolean")
}

func (section *Section) GetFloat(name string) (float64, error) {
	value, err := section.Get(name)
	if err != nil {
		return float64(0), err
	}

	switch value.(type) {
	case *Primative:
		return value.(*Primative).AsFloat()
	}

	return float64(0), errors.New("Could not convert non-primative value to float")
}

func (section *Section) GetInteger(name string) (int64, error) {
	value, err := section.Get(name)
	if err != nil {
		return int64(0), err
	}

	switch value.(type) {
	case *Primative:
		return value.(*Primative).AsInteger()
	}

	return int64(0), errors.New("Could not convert non-primative value to integer")
}

func (section *Section) GetSection(name string) (*Section, error) {
	value, err := section.Get(name)
	if err != nil {
		return nil, err
	}

	if value.GetType() == SECTION {
		return value.(*Section), nil
	}
	return nil, errors.New("Could not fetch value as section")
}

func (section *Section) GetString(name string) (string, error) {
	value, err := section.Get(name)
	if err != nil {
		return "", err
	}

	switch value.(type) {
	case *Primative:
		return value.(*Primative).AsString()
	}

	return "", errors.New("Could not convert non-primative value to string")
}

func (section *Section) GetParent() *Section {
	return section.parent
}

func (section *Section) HasParent() bool {
	return section.parent != nil
}

func (section *Section) Set(name string, value Value) {
	section.values[name] = value
}

func (section *Section) SetBoolean(name string, value bool) {
	current, err := section.Get(name)

	// Exists just update the value/type
	if err == nil {
		current.UpdateValue(value)
	} else {
		section.values[name] = NewBoolean(value)
	}
}

func (section *Section) SetFloat(name string, value float64) {
	current, err := section.Get(name)

	// Exists just update the value/type
	if err == nil {
		current.UpdateValue(value)
	} else {
		section.values[name] = NewFloat(value)
	}
}

func (section *Section) SetInteger(name string, value int64) {
	current, err := section.Get(name)

	// Exists just update the value/type
	if err == nil {
		current.UpdateValue(value)
	} else {
		section.values[name] = NewInteger(value)
	}
}

func (section *Section) SetNull(name string) {
	current, err := section.Get(name)

	// Already is a Null, nothing to do
	if err == nil && current.GetType() == NULL {
		return
	}
	section.Set(name, NewNull())
}

func (section *Section) SetString(name string, value string) {
	current, err := section.Get(name)

	// Exists just update the value/type
	if err == nil {
		current.UpdateValue(value)
	} else {
		section.Set(name, NewString(value))
	}
}

func (section *Section) Resolve(name string) (Value, error) {
	// Used only in error state return value
	var value Value

	parts := strings.Split(name, ".")
	if len(parts) == 0 {
		return value, errors.New("No name provided")
	}

	var current Value
	current = section
	for _, part := range parts {
		if current.GetType() != SECTION {
			return value, errors.New("Trying to resolve value from non-section")
		}

		next_current, err := current.(*Section).Get(part)
		if err != nil {
			return value, errors.New("Could not find value in section")
		}
		current = next_current
	}
	return current, nil
}

func (section *Section) ToJSON() ([]byte, error) {
	data := section.ToMap()
	return json.Marshal(data)
}

func (section *Section) ToMap() map[string]interface{} {
	output := make(map[string]interface{})

	for key, value := range section.values {
		if value.GetType() == SECTION {
			output[key] = value.(*Section).ToMap()
		} else {
			output[key] = value.GetValue()
		}
	}
	return output
}
<commit_msg>remove underscore variable name in section<commit_after>package forge

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"
)

type Section struct {
	parent *Section
	values map[string]Value
}

func NewSection() *Section {
	return &Section{
		values: make(map[string]Value),
	}
}

func NewChildSection(parent *Section) *Section {
	return &Section{
		parent: parent,
		values: make(map[string]Value),
	}
}

func (section *Section) GetType() ValueType {
	return SECTION
}

func (section *Section) GetValue() interface{} {
	return section.values
}

func (section *Section) UpdateValue(value interface{}) error {
	switch value.(type) {
	case map[string]Value:
		section.values = value.(map[string]Value)
		return nil
	}

	msg := fmt.Sprintf("Unsupported type, %s must be of type `map[string]Value`", value)
	return errors.New(msg)
}

func (section *Section) AddSection(name string) *Section {
	childSection := NewChildSection(section)
	section.values[name] = childSection
	return childSection
}

func (section *Section) Exists(name string) bool {
	_, err := section.Get(name)
	return err == nil
}

func (section *Section) Get(name string) (Value, error) {
	value, ok := section.values[name]
	var err error
	if ok == false {
		err = errors.New("Value does not exist")
	}
	return value, err
}

func (section *Section) GetBoolean(name string) (bool, error) {
	value, err := section.Get(name)
	if err != nil {
		return false, err
	}

	switch value.(type) {
	case *Primative:
		return value.(*Primative).AsBoolean()
	case *Section:
		return true, nil
	}

	return false, errors.New("Could not convert unknown value to boolean")
}

func (section *Section) GetFloat(name string) (float64, error) {
	value, err := section.Get(name)
	if err != nil {
		return float64(0), err
	}

	switch value.(type) {
	case *Primative:
		return value.(*Primative).AsFloat()
	}

	return float64(0), errors.New("Could not convert non-primative value to float")
}

func (section *Section) GetInteger(name string) (int64, error) {
	value, err := section.Get(name)
	if err != nil {
		return int64(0), err
	}

	switch value.(type) {
	case *Primative:
		return value.(*Primative).AsInteger()
	}

	return int64(0), errors.New("Could not convert non-primative value to integer")
}

func (section *Section) GetSection(name string) (*Section, error) {
	value, err := section.Get(name)
	if err != nil {
		return nil, err
	}

	if value.GetType() == SECTION {
		return value.(*Section), nil
	}
	return nil, errors.New("Could not fetch value as section")
}

func (section *Section) GetString(name string) (string, error) {
	value, err := section.Get(name)
	if err != nil {
		return "", err
	}

	switch value.(type) {
	case *Primative:
		return value.(*Primative).AsString()
	}

	return "", errors.New("Could not convert non-primative value to string")
}

func (section *Section) GetParent() *Section {
	return section.parent
}

func (section *Section) HasParent() bool {
	return section.parent != nil
}

func (section *Section) Set(name string, value Value) {
	section.values[name] = value
}

func (section *Section) SetBoolean(name string, value bool) {
	current, err := section.Get(name)

	// Exists just update the value/type
	if err == nil {
		current.UpdateValue(value)
	} else {
		section.values[name] = NewBoolean(value)
	}
}

func (section *Section) SetFloat(name string, value float64) {
	current, err := section.Get(name)

	// Exists just update the value/type
	if err == nil {
		current.UpdateValue(value)
	} else {
		section.values[name] = NewFloat(value)
	}
}

func (section *Section) SetInteger(name string, value int64) {
	current, err := section.Get(name)

	// Exists just update the value/type
	if err == nil {
		current.UpdateValue(value)
	} else {
		section.values[name] = NewInteger(value)
	}
}

func (section *Section) SetNull(name string) {
	current, err := section.Get(name)

	// Already is a Null, nothing to do
	if err == nil && current.GetType() == NULL {
		return
	}
	section.Set(name, NewNull())
}

func (section *Section) SetString(name string, value string) {
	current, err := section.Get(name)

	// Exists just update the value/type
	if err == nil {
		current.UpdateValue(value)
	} else {
		section.Set(name, NewString(value))
	}
}

func (section *Section) Resolve(name string) (Value, error) {
	// Used only in error state return value
	var value Value

	parts := strings.Split(name, ".")
	if len(parts) == 0 {
		return value, errors.New("No name provided")
	}

	var current Value
	current = section
	for _, part := range parts {
		if current.GetType() != SECTION {
			return value, errors.New("Trying to resolve value from non-section")
		}

		nextCurrent, err := current.(*Section).Get(part)
		if err != nil {
			return value, errors.New("Could not find value in section")
		}
		current = nextCurrent
	}
	return current, nil
}

func (section *Section) ToJSON() ([]byte, error) {
	data := section.ToMap()
	return json.Marshal(data)
}

func (section *Section) ToMap() map[string]interface{} {
	output := make(map[string]interface{})

	for key, value := range section.values {
		if value.GetType() == SECTION {
			output[key] = value.(*Section).ToMap()
		} else {
			output[key] = value.GetValue()
		}
	}
	return output
}
<|endoftext|>


<commit_before>package lib

/*SiveOfEratosthenes returns an Slice of primenumbers smaller than the given paramter*/
func SiveOfEratosthenes(numberOfPrimes int64) []int64{
	ch := make(chan int64)
	var returnValue []int64
	go generate(ch)
	var a int64 = 0
	for i := a; i < numberOfPrimes; i++ {
		prime := <-ch
		if prime <= numberOfPrimes {
			returnValue = append(returnValue, prime)
			ch1 := make(chan int64)
			go filter(ch, ch1, prime)
			ch = ch1
		} else {
			break
		}
	}
	return returnValue
}

func filter(in <-chan int64, out chan<- int64, prime int64) {
	for {
		i := <-in
		if i%prime != 0 {
			out <- i
		}
	}
}

func generate(ch chan<- int64) {
	var a int64 = 2
	for i := a; ; i++ {
		ch <- i
	}
}<commit_msg>added prime check method<commit_after>package lib

/*SiveOfEratosthenes returns an Slice of primenumbers smaller than the given paramter*/
func SiveOfEratosthenes(numberOfPrimes int64) []int64{
	ch := make(chan int64)
	var returnValue []int64
	go generate(ch)
	var a int64 = 0
	for i := a; i < numberOfPrimes; i++ {
		prime := <-ch
		if prime <= numberOfPrimes {
			returnValue = append(returnValue, prime)
			ch1 := make(chan int64)
			go filter(ch, ch1, prime)
			ch = ch1
		} else {
			break
		}
	}
	return returnValue
}

func IsPrime(number int64) bool{
	if number <=1 {
		return false
	}
	var i int64 = 2
	for i*i <= number {
		if IsPrime(i){
			if number%i==0 {
				return false
			}
		}
		i++
	}
	return true
}

func filter(in <-chan int64, out chan<- int64, prime int64) {
	for {
		i := <-in
		if i%prime != 0 {
			out <- i
		}
	}
}

func generate(ch chan<- int64) {
	var a int64 = 2
	for i := a; ; i++ {
		ch <- i
	}
}<|endoftext|>


<commit_before>package backupservice

/*
Copyright 2017-2018 Crunchy Data Solutions, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

import (
	log "github.com/Sirupsen/logrus"
	crv1 "github.com/crunchydata/postgres-operator/apis/cr/v1"
	"github.com/crunchydata/postgres-operator/apiserver"
	msgs "github.com/crunchydata/postgres-operator/apiservermsgs"
	"github.com/crunchydata/postgres-operator/kubeapi"
	"github.com/crunchydata/postgres-operator/util"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// ShowBackup ...
func ShowBackup(name string) msgs.ShowBackupResponse {
	var err error
	response := msgs.ShowBackupResponse{}
	response.Status = msgs.Status{Code: msgs.Ok, Msg: ""}

	if name == "all" {
		//get a list of all backups
		err = kubeapi.Getpgbackups(apiserver.RESTClient, &response.BackupList, apiserver.Namespace)
		if err != nil {
			response.Status.Code = msgs.Error
			response.Status.Msg = err.Error()
			return response
		}
		log.Debugf("backups found len is %d\n", len(response.BackupList.Items))
	} else {
		backup := crv1.Pgbackup{}
		_, err := kubeapi.Getpgbackup(apiserver.RESTClient, &backup, name, apiserver.Namespace)
		if err != nil {
			response.Status.Code = msgs.Error
			response.Status.Msg = err.Error()
			return response
		}
		response.BackupList.Items = make([]crv1.Pgbackup, 1)
		response.BackupList.Items[0] = backup
	}

	return response

}

// DeleteBackup ...
func DeleteBackup(backupName string) msgs.DeleteBackupResponse {
	resp := msgs.DeleteBackupResponse{}
	resp.Status.Code = msgs.Ok
	resp.Status.Msg = ""
	resp.Results = make([]string, 0)

	var err error

	if backupName == "all" {
		resp.Status.Code = msgs.Error
		resp.Status.Msg = "all not a valid cluster name"
		return resp
	}

	err = kubeapi.Deletepgbackup(apiserver.RESTClient, backupName, apiserver.Namespace)

	if err != nil {
		resp.Status.Code = msgs.Error
		resp.Status.Msg = err.Error()
		return resp
	}
	resp.Results = append(resp.Results, backupName)

	//create a pgtask to remove the PVC and its data
	pvcName := backupName + "-backup"
	dataRoots := []string{backupName + "-backups"}

	storageSpec := crv1.PgStorageSpec{}
	err = apiserver.CreateRMDataTask(storageSpec, backupName, pvcName, dataRoots)
	if err != nil {
		resp.Status.Code = msgs.Error
		resp.Status.Msg = err.Error()
		return resp
	}

	return resp

}

//  CreateBackup ...
// pgo backup mycluster
// pgo backup all
// pgo backup --selector=name=mycluster
func CreateBackup(request *msgs.CreateBackupRequest) msgs.CreateBackupResponse {
	resp := msgs.CreateBackupResponse{}
	resp.Status.Code = msgs.Ok
	resp.Status.Msg = ""
	resp.Results = make([]string, 0)

	var newInstance *crv1.Pgbackup

	log.Info("CreateBackup sc " + request.StorageConfig)
	if request.StorageConfig != "" {
		if apiserver.IsValidStorageName(request.StorageConfig) == false {
			log.Info("CreateBackup sc error is found " + request.StorageConfig)
			resp.Status.Code = msgs.Error
			resp.Status.Msg = request.StorageConfig + " Storage config was not found "
			return resp
		}
	}

	if request.Selector != "" {
		//use the selector instead of an argument list to filter on

		clusterList := crv1.PgclusterList{}

		err := kubeapi.GetpgclustersBySelector(apiserver.RESTClient, &clusterList, request.Selector, apiserver.Namespace)
		if err != nil {
			resp.Status.Code = msgs.Error
			resp.Status.Msg = err.Error()
			return resp
		}

		if len(clusterList.Items) == 0 {
			log.Debug("no clusters found")
			resp.Results = append(resp.Results, "no clusters found with that selector")
			return resp
		} else {
			newargs := make([]string, 0)
			for _, cluster := range clusterList.Items {
				newargs = append(newargs, cluster.Spec.Name)
			}
			request.Args = newargs
		}

	}

	for _, arg := range request.Args {
		log.Debugf("create backup called for %s", arg)

		cluster := crv1.Pgcluster{}
		found, err := kubeapi.Getpgcluster(apiserver.RESTClient, &cluster, arg, apiserver.Namespace)
		if !found {
			resp.Status.Code = msgs.Error
			resp.Status.Msg = arg + " was not found, verify cluster name"
			return resp
		} else if err != nil {
			resp.Status.Code = msgs.Error
			resp.Status.Msg = err.Error()
			return resp
		}

		//remove any existing backup job
		RemoveBackupJob("backup-" + arg)

		result := crv1.Pgbackup{}

		// error if it already exists
		found, err = kubeapi.Getpgbackup(apiserver.RESTClient, &result, arg, apiserver.Namespace)
		if !found {
			log.Debugf("pgbackup %s was not found so we will create it", arg)
		} else if err != nil {
			resp.Results = append(resp.Results, "error getting pgbackup for "+arg)
			break
		} else {
			log.Debugf("pgbackup %s was found so we will recreate it", arg)
			dels := make([]string, 1)
			dels[0] = arg
			DeleteBackup(arg)
		}

		// Create an instance of our CRD
		newInstance, err = getBackupParams(arg, request.StorageConfig)
		if err != nil {
			msg := "error creating backup for " + arg
			log.Error(err)
			resp.Results = append(resp.Results, msg)
			break
		}
		if request.PVCName != "" {
			log.Debugf("backuppvc is %s", request.PVCName)
			newInstance.Spec.BackupPVC = request.PVCName
		}

		err = kubeapi.Createpgbackup(apiserver.RESTClient, newInstance, apiserver.Namespace)
		if err != nil {
			resp.Status.Code = msgs.Error
			resp.Status.Msg = err.Error()
			return resp
		}
		resp.Results = append(resp.Results, "created Pgbackup "+arg)

	}

	return resp
}

func getBackupParams(name, storageConfig string) (*crv1.Pgbackup, error) {
	var err error
	var newInstance *crv1.Pgbackup

	spec := crv1.PgbackupSpec{}
	spec.Name = name
	if storageConfig != "" {
		spec.StorageSpec, _ = apiserver.Pgo.GetStorageSpec(storageConfig)
	} else {
		spec.StorageSpec, _ = apiserver.Pgo.GetStorageSpec(apiserver.Pgo.BackupStorage)
	}
	spec.CCPImageTag = apiserver.Pgo.Cluster.CCPImageTag
	spec.BackupStatus = "initial"
	spec.BackupHost = "basic"
	spec.BackupUser = "primaryuser"
	spec.BackupPass = "password"
	spec.BackupPort = "5432"

	cluster := crv1.Pgcluster{}
	_, err = kubeapi.Getpgcluster(apiserver.RESTClient, &cluster, name, apiserver.Namespace)
	if err == nil {
		spec.BackupHost = cluster.Spec.Name
		spec.BackupPass, err = util.GetSecretPassword(apiserver.Clientset, cluster.Spec.Name, crv1.PrimarySecretSuffix, apiserver.Namespace)
		if err != nil {
			return newInstance, err
		}
		spec.BackupPort = cluster.Spec.Port
	} else {
		return newInstance, err
	}

	newInstance = &crv1.Pgbackup{
		ObjectMeta: meta_v1.ObjectMeta{
			Name: name,
		},
		Spec: spec,
	}
	return newInstance, nil
}

func RemoveBackupJob(name string) {

	_, found := kubeapi.GetJob(apiserver.Clientset, name, apiserver.Namespace)
	if !found {
		return
	}

	log.Debugf("found backup job %s will remove\n", name)

	kubeapi.DeleteJob(apiserver.Clientset, name, apiserver.Namespace)
}
<commit_msg>provide better message when a not found backup is caught in pgo show backup<commit_after>package backupservice

/*
Copyright 2017-2018 Crunchy Data Solutions, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

import (
	log "github.com/Sirupsen/logrus"
	crv1 "github.com/crunchydata/postgres-operator/apis/cr/v1"
	"github.com/crunchydata/postgres-operator/apiserver"
	msgs "github.com/crunchydata/postgres-operator/apiservermsgs"
	"github.com/crunchydata/postgres-operator/kubeapi"
	"github.com/crunchydata/postgres-operator/util"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// ShowBackup ...
func ShowBackup(name string) msgs.ShowBackupResponse {
	var err error
	response := msgs.ShowBackupResponse{}
	response.Status = msgs.Status{Code: msgs.Ok, Msg: ""}

	if name == "all" {
		//get a list of all backups
		err = kubeapi.Getpgbackups(apiserver.RESTClient, &response.BackupList, apiserver.Namespace)
		if err != nil {
			response.Status.Code = msgs.Error
			response.Status.Msg = err.Error()
			return response
		}
		log.Debugf("backups found len is %d\n", len(response.BackupList.Items))
	} else {
		backup := crv1.Pgbackup{}
		found, err := kubeapi.Getpgbackup(apiserver.RESTClient, &backup, name, apiserver.Namespace)
		if !found {
			response.Status.Code = msgs.Error
			response.Status.Msg = "backup not found"
			return response
		}
		if err != nil {
			response.Status.Code = msgs.Error
			response.Status.Msg = err.Error()
			return response
		}
		response.BackupList.Items = make([]crv1.Pgbackup, 1)
		response.BackupList.Items[0] = backup
	}

	return response

}

// DeleteBackup ...
func DeleteBackup(backupName string) msgs.DeleteBackupResponse {
	resp := msgs.DeleteBackupResponse{}
	resp.Status.Code = msgs.Ok
	resp.Status.Msg = ""
	resp.Results = make([]string, 0)

	var err error

	if backupName == "all" {
		resp.Status.Code = msgs.Error
		resp.Status.Msg = "all not a valid cluster name"
		return resp
	}

	err = kubeapi.Deletepgbackup(apiserver.RESTClient, backupName, apiserver.Namespace)

	if err != nil {
		resp.Status.Code = msgs.Error
		resp.Status.Msg = err.Error()
		return resp
	}
	resp.Results = append(resp.Results, backupName)

	//create a pgtask to remove the PVC and its data
	pvcName := backupName + "-backup"
	dataRoots := []string{backupName + "-backups"}

	storageSpec := crv1.PgStorageSpec{}
	err = apiserver.CreateRMDataTask(storageSpec, backupName, pvcName, dataRoots)
	if err != nil {
		resp.Status.Code = msgs.Error
		resp.Status.Msg = err.Error()
		return resp
	}

	return resp

}

//  CreateBackup ...
// pgo backup mycluster
// pgo backup all
// pgo backup --selector=name=mycluster
func CreateBackup(request *msgs.CreateBackupRequest) msgs.CreateBackupResponse {
	resp := msgs.CreateBackupResponse{}
	resp.Status.Code = msgs.Ok
	resp.Status.Msg = ""
	resp.Results = make([]string, 0)

	var newInstance *crv1.Pgbackup

	log.Info("CreateBackup sc " + request.StorageConfig)
	if request.StorageConfig != "" {
		if apiserver.IsValidStorageName(request.StorageConfig) == false {
			log.Info("CreateBackup sc error is found " + request.StorageConfig)
			resp.Status.Code = msgs.Error
			resp.Status.Msg = request.StorageConfig + " Storage config was not found "
			return resp
		}
	}

	if request.Selector != "" {
		//use the selector instead of an argument list to filter on

		clusterList := crv1.PgclusterList{}

		err := kubeapi.GetpgclustersBySelector(apiserver.RESTClient, &clusterList, request.Selector, apiserver.Namespace)
		if err != nil {
			resp.Status.Code = msgs.Error
			resp.Status.Msg = err.Error()
			return resp
		}

		if len(clusterList.Items) == 0 {
			log.Debug("no clusters found")
			resp.Results = append(resp.Results, "no clusters found with that selector")
			return resp
		} else {
			newargs := make([]string, 0)
			for _, cluster := range clusterList.Items {
				newargs = append(newargs, cluster.Spec.Name)
			}
			request.Args = newargs
		}

	}

	for _, arg := range request.Args {
		log.Debugf("create backup called for %s", arg)

		cluster := crv1.Pgcluster{}
		found, err := kubeapi.Getpgcluster(apiserver.RESTClient, &cluster, arg, apiserver.Namespace)
		if !found {
			resp.Status.Code = msgs.Error
			resp.Status.Msg = arg + " was not found, verify cluster name"
			return resp
		} else if err != nil {
			resp.Status.Code = msgs.Error
			resp.Status.Msg = err.Error()
			return resp
		}

		//remove any existing backup job
		RemoveBackupJob("backup-" + arg)

		result := crv1.Pgbackup{}

		// error if it already exists
		found, err = kubeapi.Getpgbackup(apiserver.RESTClient, &result, arg, apiserver.Namespace)
		if !found {
			log.Debugf("pgbackup %s was not found so we will create it", arg)
		} else if err != nil {
			resp.Results = append(resp.Results, "error getting pgbackup for "+arg)
			break
		} else {
			log.Debugf("pgbackup %s was found so we will recreate it", arg)
			dels := make([]string, 1)
			dels[0] = arg
			DeleteBackup(arg)
		}

		// Create an instance of our CRD
		newInstance, err = getBackupParams(arg, request.StorageConfig)
		if err != nil {
			msg := "error creating backup for " + arg
			log.Error(err)
			resp.Results = append(resp.Results, msg)
			break
		}
		if request.PVCName != "" {
			log.Debugf("backuppvc is %s", request.PVCName)
			newInstance.Spec.BackupPVC = request.PVCName
		}

		err = kubeapi.Createpgbackup(apiserver.RESTClient, newInstance, apiserver.Namespace)
		if err != nil {
			resp.Status.Code = msgs.Error
			resp.Status.Msg = err.Error()
			return resp
		}
		resp.Results = append(resp.Results, "created Pgbackup "+arg)

	}

	return resp
}

func getBackupParams(name, storageConfig string) (*crv1.Pgbackup, error) {
	var err error
	var newInstance *crv1.Pgbackup

	spec := crv1.PgbackupSpec{}
	spec.Name = name
	if storageConfig != "" {
		spec.StorageSpec, _ = apiserver.Pgo.GetStorageSpec(storageConfig)
	} else {
		spec.StorageSpec, _ = apiserver.Pgo.GetStorageSpec(apiserver.Pgo.BackupStorage)
	}
	spec.CCPImageTag = apiserver.Pgo.Cluster.CCPImageTag
	spec.BackupStatus = "initial"
	spec.BackupHost = "basic"
	spec.BackupUser = "primaryuser"
	spec.BackupPass = "password"
	spec.BackupPort = "5432"

	cluster := crv1.Pgcluster{}
	_, err = kubeapi.Getpgcluster(apiserver.RESTClient, &cluster, name, apiserver.Namespace)
	if err == nil {
		spec.BackupHost = cluster.Spec.Name
		spec.BackupPass, err = util.GetSecretPassword(apiserver.Clientset, cluster.Spec.Name, crv1.PrimarySecretSuffix, apiserver.Namespace)
		if err != nil {
			return newInstance, err
		}
		spec.BackupPort = cluster.Spec.Port
	} else {
		return newInstance, err
	}

	newInstance = &crv1.Pgbackup{
		ObjectMeta: meta_v1.ObjectMeta{
			Name: name,
		},
		Spec: spec,
	}
	return newInstance, nil
}

func RemoveBackupJob(name string) {

	_, found := kubeapi.GetJob(apiserver.Clientset, name, apiserver.Namespace)
	if !found {
		return
	}

	log.Debugf("found backup job %s will remove\n", name)

	kubeapi.DeleteJob(apiserver.Clientset, name, apiserver.Namespace)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"log"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/inominate/apicache"
	"github.com/kr/pretty"
)

var apiClient apicache.Client

// worker tracking
var activeWorkerCount, workerCount int32
var workCount []int32

type apiReq struct {
	apiReq  *apicache.Request
	apiResp *apicache.Response

	apiErr apicache.APIError

	expires time.Time

	worker   int
	httpCode int
	err      error
	respChan chan apiReq
}

// Channel for sending jobs to workers
var workChan chan apiReq

func APIReq(url string, params map[string]string) (*apicache.Response, error) {
	var errorStr string

	if atomic.LoadInt32(&workerCount) <= 0 {
		panic("No workers!")
	}

	// Build the request
	apireq := apicache.NewRequest(url)
	for k, v := range params {
		switch k {
		case "force":
			if v != "" {
				apireq.Force = true
			}
		default:
			apireq.Set(k, v)
		}
	}

	workerID := "C"
	// Don't send it to a worker if we can just yank it fromm the cache
	apiResp, err := apireq.GetCached()
	// 221's are the bane of my existence.
	if err != nil || apireq.Force || apiResp.Error.ErrorCode == 221 {
		for i := 0; i < conf.Retries; i++ {
			respChan := make(chan apiReq)
			req := apiReq{apiReq: apireq, respChan: respChan}
			workChan <- req

			resp := <-respChan
			close(respChan)

			apiResp = resp.apiResp
			err = resp.err
			workerID = fmt.Sprintf("%d", resp.worker)

			// Attempt to recover from server issues, invalidate flag means we
			// believe this is not a server failure.
			// 418 is the tempban code
			// 500/900 are panic codes
			if apiResp.Invalidate || apiResp.HTTPCode == 418 || apiResp.HTTPCode == 500 || apiResp.HTTPCode == 900 {
				break
			}
			time.Sleep(2 * time.Second)
			apireq.Force = true
		}
	}
	var singleDebug bool
	if apiResp.Error.ErrorCode == 221 {
		log.Printf("Failed to recover from error 221.")
		singleDebug = true
	}

	// This is similar to the request log, but knows more about where it came from.
	if debug || singleDebug {
		if apiResp.Error.ErrorCode != 0 {
			errorStr = fmt.Sprintf(" Error %d: %s", apiResp.Error.ErrorCode, apiResp.Error.ErrorText)
		}
		logParams := ""
		var paramVal string
		for k, _ := range params {
			if conf.Logging.CensorLog && strings.ToLower(k) == "vcode" && len(params[k]) > 8 {
				paramVal = params[k][0:8] + "..."
			} else {
				paramVal = params[k]
			}
			logParams = fmt.Sprintf("%s&%s=%s", logParams, k, paramVal)
		}
		if logParams != "" {
			logParams = "?" + logParams[1:]
		}

		debugLog.Printf("w%s: %s%s HTTP: %d Expires: %s%s", workerID, url, logParams, apiResp.HTTPCode, apiResp.Expires.Format("2006-01-02 15:04:05"), errorStr)
	}
	return apiResp, err
}

func worker(reqChan chan apiReq, workerID int) {
	atomic.AddInt32(&workerCount, 1)
	var err error

	var eErr error
	var rErr error

	for req := range reqChan {
		var errStr string

		atomic.AddInt32(&activeWorkerCount, 1)

		// Run both of the error limiters simultaneously rather than in
		// sequence. Still need both before we continue.
		errorLimiter := make(chan error)
		rpsLimiter := make(chan error)
		go func() {
			err := errorRateLimiter.Start(30 * time.Second)
			errorLimiter <- err
		}()
		go func() {
			err := rateLimiter.Start(30 * time.Second)
			rpsLimiter <- err
		}()
		eErr = <-errorLimiter
		rErr = <-rpsLimiter

		// Check the error limiter for timeouts
		if eErr != nil {
			err = eErr
			errStr = "error throttling"

			// If the rate limiter didn't timeout be sure to signal it that we
			// didn't do anything.
			if rErr == nil {
				rateLimiter.Finish(true)
			}
		}
		if rErr != nil {
			err = rErr
			if errStr == "" {
				errStr = "rate limiting"
			} else {
				errStr += " and rate limiting"
			}

			// If the error limiter didn't also timeout be sure to signal it that we
			// didn't do anything.
			if eErr == nil {
				errorRateLimiter.Finish(true)
			}
		}
		// We're left with a single err and errStr for returning an error to the client.
		if err != nil {
			log.Printf("Rate Limit Error: %s - %s", errStr, err)
			log.Printf("RPS Events: %d Outstanding: %d", rateLimiter.Count(), rateLimiter.Outstanding())
			log.Printf("Errors Events: %d Outstanding: %d", errorRateLimiter.Count(), errorRateLimiter.Outstanding())
			log.Printf("RPSLimit: %# v", pretty.Formatter(rateLimiter))
			log.Printf("ErrorLimit: %# v", pretty.Formatter(errorRateLimiter))

			req.apiResp = &apicache.Response{
				Data: apicache.SynthesizeAPIError(500,
					fmt.Sprintf("APIProxy Error: Proxy timeout due to %s.", errStr),
					5*time.Minute),
				Expires: time.Now().Add(5 * time.Minute),
				Error: apicache.APIError{500,
					fmt.Sprintf("APIProxy Error: Proxy timeout due to %s.", errStr)},
				HTTPCode: 504,
			}
			req.err = err
		} else {
			resp, err := req.apiReq.Do()
			req.apiResp = resp
			req.err = err
			if resp.Error.ErrorCode == 0 || resp.HTTPCode == 504 || resp.HTTPCode == 418 {
				// 418 means we are currently tempbanned from the API.
				// 504 means the API proxy had some kind of internal or network error.
				//
				// We do not treat these as an error for rate limiting because
				// the apicache library handles it for us, these requests are
				// not actually making it to the CCP API.

				// Finish, but skip recording the event in the rate limiter
				// when there is no error.
				errorRateLimiter.Finish(true)
			} else {
				errorRateLimiter.Finish(false)
			}
			rateLimiter.Finish(false)
		}

		req.worker = workerID
		req.respChan <- req
		atomic.AddInt32(&workCount[workerID], 1)
		atomic.AddInt32(&activeWorkerCount, -1)
	}
	atomic.AddInt32(&workerCount, -1)
}

var startWorkersOnce = &sync.Once{}

func startWorkers() {
	startWorkersOnce.Do(realStartWorkers)
}

func realStartWorkers() {
	log.Printf("Starting %d Workers...", conf.Workers)
	workChan = make(chan apiReq)
	workCount = make([]int32, conf.Workers+1)

	for i := 1; i <= conf.Workers; i++ {
		debugLog.Printf("Starting worker #%d.", i)
		go worker(workChan, i)
	}
}

func stopWorkers() {
	close(workChan)
	for atomic.LoadInt32(&workerCount) > 0 {
		time.Sleep(10 * time.Millisecond)
	}
	startWorkersOnce = &sync.Once{}
}

func PrintWorkerStats(w io.Writer) {
	active, loaded := GetWorkerStats()
	fmt.Fprintf(w, "%d workers idle, %d workers active.\n", loaded-active, active)

	rateCount := rateLimiter.Count()
	rateOutstanding := rateLimiter.Outstanding()

	errorCount := errorRateLimiter.Count()
	errorOutstanding := errorRateLimiter.Outstanding()

	fmt.Fprintf(w, "%d requests in the last second. %d requests outstanding.\n", rateCount, rateOutstanding)
	fmt.Fprintf(w, "%d errors over last %d seconds. %d errors outstanding.\n", errorCount, conf.ErrorPeriod, errorOutstanding)

	for i := int32(1); i <= atomic.LoadInt32(&workerCount); i++ {
		count := atomic.LoadInt32(&workCount[i])
		fmt.Fprintf(w, "   %d: %d\n", i, count)
	}
}

func GetWorkerStats() (int32, int32) {
	return atomic.LoadInt32(&activeWorkerCount), atomic.LoadInt32(&workerCount)
}
<commit_msg>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa I AM AN IDIOT.<commit_after>package main

import (
	"fmt"
	"io"
	"log"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/inominate/apicache"
	"github.com/kr/pretty"
)

var apiClient apicache.Client

// worker tracking
var activeWorkerCount, workerCount int32
var workCount []int32

type apiReq struct {
	apiReq  *apicache.Request
	apiResp *apicache.Response

	apiErr apicache.APIError

	expires time.Time

	worker   int
	httpCode int
	err      error
	respChan chan apiReq
}

// Channel for sending jobs to workers
var workChan chan apiReq

func APIReq(url string, params map[string]string) (*apicache.Response, error) {
	var errorStr string

	if atomic.LoadInt32(&workerCount) <= 0 {
		panic("No workers!")
	}

	// Build the request
	apireq := apicache.NewRequest(url)
	for k, v := range params {
		switch k {
		case "force":
			if v != "" {
				apireq.Force = true
			}
		default:
			apireq.Set(k, v)
		}
	}

	workerID := "C"
	// Don't send it to a worker if we can just yank it fromm the cache
	apiResp, err := apireq.GetCached()
	// 221's are the bane of my existence.
	if err != nil || apireq.Force || apiResp.Error.ErrorCode == 221 {
		for i := 0; i < conf.Retries; i++ {
			respChan := make(chan apiReq)
			req := apiReq{apiReq: apireq, respChan: respChan}
			workChan <- req

			resp := <-respChan
			close(respChan)

			apiResp = resp.apiResp
			err = resp.err
			workerID = fmt.Sprintf("%d", resp.worker)

			// Attempt to recover from server issues, invalidate flag means we
			// believe this is not a server failure.
			// 418 is the tempban code
			// 500/900 are panic codes
			if err == nil || apiResp.Invalidate || apiResp.HTTPCode == 418 || apiResp.HTTPCode == 500 || apiResp.HTTPCode == 900 {
				break
			}
			time.Sleep(2 * time.Second)
			apireq.Force = true
		}
	}
	var singleDebug bool
	if apiResp.Error.ErrorCode == 221 {
		log.Printf("Failed to recover from error 221.")
		singleDebug = true
	}

	// This is similar to the request log, but knows more about where it came from.
	if debug || singleDebug {
		if apiResp.Error.ErrorCode != 0 {
			errorStr = fmt.Sprintf(" Error %d: %s", apiResp.Error.ErrorCode, apiResp.Error.ErrorText)
		}
		logParams := ""
		var paramVal string
		for k, _ := range params {
			if conf.Logging.CensorLog && strings.ToLower(k) == "vcode" && len(params[k]) > 8 {
				paramVal = params[k][0:8] + "..."
			} else {
				paramVal = params[k]
			}
			logParams = fmt.Sprintf("%s&%s=%s", logParams, k, paramVal)
		}
		if logParams != "" {
			logParams = "?" + logParams[1:]
		}

		debugLog.Printf("w%s: %s%s HTTP: %d Expires: %s%s", workerID, url, logParams, apiResp.HTTPCode, apiResp.Expires.Format("2006-01-02 15:04:05"), errorStr)
	}
	return apiResp, err
}

func worker(reqChan chan apiReq, workerID int) {
	atomic.AddInt32(&workerCount, 1)
	var err error

	var eErr error
	var rErr error

	for req := range reqChan {
		var errStr string

		atomic.AddInt32(&activeWorkerCount, 1)

		// Run both of the error limiters simultaneously rather than in
		// sequence. Still need both before we continue.
		errorLimiter := make(chan error)
		rpsLimiter := make(chan error)
		go func() {
			err := errorRateLimiter.Start(30 * time.Second)
			errorLimiter <- err
		}()
		go func() {
			err := rateLimiter.Start(30 * time.Second)
			rpsLimiter <- err
		}()
		eErr = <-errorLimiter
		rErr = <-rpsLimiter

		// Check the error limiter for timeouts
		if eErr != nil {
			err = eErr
			errStr = "error throttling"

			// If the rate limiter didn't timeout be sure to signal it that we
			// didn't do anything.
			if rErr == nil {
				rateLimiter.Finish(true)
			}
		}
		if rErr != nil {
			err = rErr
			if errStr == "" {
				errStr = "rate limiting"
			} else {
				errStr += " and rate limiting"
			}

			// If the error limiter didn't also timeout be sure to signal it that we
			// didn't do anything.
			if eErr == nil {
				errorRateLimiter.Finish(true)
			}
		}
		// We're left with a single err and errStr for returning an error to the client.
		if err != nil {
			log.Printf("Rate Limit Error: %s - %s", errStr, err)
			log.Printf("RPS Events: %d Outstanding: %d", rateLimiter.Count(), rateLimiter.Outstanding())
			log.Printf("Errors Events: %d Outstanding: %d", errorRateLimiter.Count(), errorRateLimiter.Outstanding())
			log.Printf("RPSLimit: %# v", pretty.Formatter(rateLimiter))
			log.Printf("ErrorLimit: %# v", pretty.Formatter(errorRateLimiter))

			req.apiResp = &apicache.Response{
				Data: apicache.SynthesizeAPIError(500,
					fmt.Sprintf("APIProxy Error: Proxy timeout due to %s.", errStr),
					5*time.Minute),
				Expires: time.Now().Add(5 * time.Minute),
				Error: apicache.APIError{500,
					fmt.Sprintf("APIProxy Error: Proxy timeout due to %s.", errStr)},
				HTTPCode: 504,
			}
			req.err = err
		} else {
			resp, err := req.apiReq.Do()
			req.apiResp = resp
			req.err = err
			if resp.Error.ErrorCode == 0 || resp.HTTPCode == 504 || resp.HTTPCode == 418 {
				// 418 means we are currently tempbanned from the API.
				// 504 means the API proxy had some kind of internal or network error.
				//
				// We do not treat these as an error for rate limiting because
				// the apicache library handles it for us, these requests are
				// not actually making it to the CCP API.

				// Finish, but skip recording the event in the rate limiter
				// when there is no error.
				errorRateLimiter.Finish(true)
			} else {
				errorRateLimiter.Finish(false)
			}
			rateLimiter.Finish(false)
		}

		req.worker = workerID
		req.respChan <- req
		atomic.AddInt32(&workCount[workerID], 1)
		atomic.AddInt32(&activeWorkerCount, -1)
	}
	atomic.AddInt32(&workerCount, -1)
}

var startWorkersOnce = &sync.Once{}

func startWorkers() {
	startWorkersOnce.Do(realStartWorkers)
}

func realStartWorkers() {
	log.Printf("Starting %d Workers...", conf.Workers)
	workChan = make(chan apiReq)
	workCount = make([]int32, conf.Workers+1)

	for i := 1; i <= conf.Workers; i++ {
		debugLog.Printf("Starting worker #%d.", i)
		go worker(workChan, i)
	}
}

func stopWorkers() {
	close(workChan)
	for atomic.LoadInt32(&workerCount) > 0 {
		time.Sleep(10 * time.Millisecond)
	}
	startWorkersOnce = &sync.Once{}
}

func PrintWorkerStats(w io.Writer) {
	active, loaded := GetWorkerStats()
	fmt.Fprintf(w, "%d workers idle, %d workers active.\n", loaded-active, active)

	rateCount := rateLimiter.Count()
	rateOutstanding := rateLimiter.Outstanding()

	errorCount := errorRateLimiter.Count()
	errorOutstanding := errorRateLimiter.Outstanding()

	fmt.Fprintf(w, "%d requests in the last second. %d requests outstanding.\n", rateCount, rateOutstanding)
	fmt.Fprintf(w, "%d errors over last %d seconds. %d errors outstanding.\n", errorCount, conf.ErrorPeriod, errorOutstanding)

	for i := int32(1); i <= atomic.LoadInt32(&workerCount); i++ {
		count := atomic.LoadInt32(&workCount[i])
		fmt.Fprintf(w, "   %d: %d\n", i, count)
	}
}

func GetWorkerStats() (int32, int32) {
	return atomic.LoadInt32(&activeWorkerCount), atomic.LoadInt32(&workerCount)
}
<|endoftext|>


<commit_before>package siesta

import (
	"net/http"
	"path"
	"regexp"
	"strings"
)

var services map[string]*Service = make(map[string]*Service)

type Service struct {
	baseURI string

	pre  []contextHandler
	post []contextHandler

	handlers map[*regexp.Regexp]contextHandler

	routes map[string]*node
}

func NewService(baseURI string) *Service {
	if services[baseURI] != nil {
		panic("service already registered")
	}

	return &Service{
		baseURI:  strings.TrimRight(baseURI, "/"),
		handlers: make(map[*regexp.Regexp]contextHandler),
		routes:   map[string]*node{},
	}
}

func addToChain(f interface{}, chain []contextHandler) []contextHandler {
	m := toContextHandler(f)
	return append(chain, m)
}

func (s *Service) AddPre(f interface{}) {
	s.pre = addToChain(f, s.pre)
}

func (s *Service) AddPost(f interface{}) {
	s.post = addToChain(f, s.post)
}

// Service satisfies the http.Handler interface.
func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	s.ServeHTTPInContext(NewSiestaContext(), w, r)
}

func (s *Service) ServeHTTPInContext(c Context, w http.ResponseWriter, r *http.Request) {
	quit := false
	for _, m := range s.pre {
		if quit {
			return
		}

		m(c, w, r, func() {
			quit = true
		})
	}

	if !quit {
		r.URL.Path = strings.TrimRight(r.URL.Path, "/")
		handler, params, _ := s.routes[r.Method].getValue(r.URL.Path)

		if handler == nil {
			http.NotFoundHandler().ServeHTTP(w, r)
		} else {
			r.ParseForm()
			for _, p := range params {
				r.Form.Set(p.Key, p.Value)
			}

			handler(c, w, r, func() {
				quit = true
			})
		}

	}

	for _, m := range s.post {
		if quit {
			return
		}

		m(c, w, r, func() {
			quit = true
		})
	}
}

func (s *Service) Route(verb, uriPath, usage string, f interface{}) {
	handler := toContextHandler(f)

	if n := s.routes[verb]; n == nil {
		s.routes[verb] = &node{}
	}

	s.routes[verb].addRoute(path.Join(s.baseURI, uriPath), handler)
}

func (s *Service) Register() {
	http.Handle(s.baseURI, s)
	http.Handle(s.baseURI+"/", s)
}
<commit_msg>check if a tree exists<commit_after>package siesta

import (
	"net/http"
	"path"
	"regexp"
	"strings"
)

var services map[string]*Service = make(map[string]*Service)

type Service struct {
	baseURI string

	pre  []contextHandler
	post []contextHandler

	handlers map[*regexp.Regexp]contextHandler

	routes map[string]*node
}

func NewService(baseURI string) *Service {
	if services[baseURI] != nil {
		panic("service already registered")
	}

	return &Service{
		baseURI:  strings.TrimRight(baseURI, "/"),
		handlers: make(map[*regexp.Regexp]contextHandler),
		routes:   map[string]*node{},
	}
}

func addToChain(f interface{}, chain []contextHandler) []contextHandler {
	m := toContextHandler(f)
	return append(chain, m)
}

func (s *Service) AddPre(f interface{}) {
	s.pre = addToChain(f, s.pre)
}

func (s *Service) AddPost(f interface{}) {
	s.post = addToChain(f, s.post)
}

// Service satisfies the http.Handler interface.
func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	s.ServeHTTPInContext(NewSiestaContext(), w, r)
}

func (s *Service) ServeHTTPInContext(c Context, w http.ResponseWriter, r *http.Request) {
	quit := false
	for _, m := range s.pre {
		if quit {
			return
		}

		m(c, w, r, func() {
			quit = true
		})
	}

	if !quit {
		r.URL.Path = strings.TrimRight(r.URL.Path, "/")

		var (
			handler contextHandler
			params  routeParams
		)

		// Lookup the tree for this method
		routeNode, ok := s.routes[r.Method]

		if ok {
			handler, params, _ = routeNode.getValue(r.URL.Path)
		}

		if handler == nil {
			http.NotFoundHandler().ServeHTTP(w, r)
		} else {
			r.ParseForm()
			for _, p := range params {
				r.Form.Set(p.Key, p.Value)
			}

			handler(c, w, r, func() {
				quit = true
			})
		}

	}

	for _, m := range s.post {
		if quit {
			return
		}

		m(c, w, r, func() {
			quit = true
		})
	}
}

func (s *Service) Route(verb, uriPath, usage string, f interface{}) {
	handler := toContextHandler(f)

	if n := s.routes[verb]; n == nil {
		s.routes[verb] = &node{}
	}

	s.routes[verb].addRoute(path.Join(s.baseURI, uriPath), handler)
}

func (s *Service) Register() {
	http.Handle(s.baseURI, s)
	http.Handle(s.baseURI+"/", s)
}
<|endoftext|>


<commit_before>package revel

import (
	"fmt"
	"github.com/streadway/simpleuuid"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"
)

// A signed cookie (and thus limited to 4kb in size).
// Restriction: Keys may not have a colon in them.
type Session map[string]string

const (
	SESSION_ID_KEY = "_ID"
	TS_KEY         = "_TS"
)

// expireAfterDuration is the time to live, in seconds, of a session cookie.
// It may be specified in config as "session.expires". Values greater than 0
// set a persistent cookie with a time to live as specified, and the value 0
// sets a session cookie.
var expireAfterDuration time.Duration

func init() {
	// Set expireAfterDuration, default to 30 days if no value in config
	OnAppStart(func() {
		var err error
		if expiresString, ok := Config.String("session.expires"); !ok {
			expireAfterDuration = 30 * 24 * time.Hour
		} else if expiresString == "session" {
			expireAfterDuration = 0
		} else if expireAfterDuration, err = time.ParseDuration(expiresString); err != nil {
			panic(fmt.Errorf("session.expires invalid: %s", err))
		}
	})
}

// Id retrieves from the cookie or creates a time-based UUID identifying this
// session.
func (s Session) Id() string {
	if uuidStr, ok := s[SESSION_ID_KEY]; ok {
		return uuidStr
	}

	uuid, err := simpleuuid.NewTime(time.Now())
	if err != nil {
		panic(err) // I don't think this can actually happen.
	}
	s[SESSION_ID_KEY] = uuid.String()
	return s[SESSION_ID_KEY]
}

// getSessionExpiration return a time.Time with the session's expiration date.
// If previous session has set to "session", remain it
func (s Session) getSessionExpiration() time.Time {
	if expireAfterDuration == 0 || s[TS_KEY] == "session" {
		return time.Time{}
	}
	return time.Now().Add(expireAfterDuration)
}

// cookie returns an http.Cookie containing the signed session.
func (s Session) cookie() *http.Cookie {
	var sessionValue string
	ts := s.getSessionExpiration()
	s[TS_KEY] = getSessionExpirationCookie(ts)
	for key, value := range s {
		if strings.ContainsAny(key, ":\x00") {
			panic("Session keys may not have colons or null bytes")
		}
		if strings.Contains(value, "\x00") {
			panic("Session values may not have null bytes")
		}
		sessionValue += "\x00" + key + ":" + value + "\x00"
	}

	sessionData := url.QueryEscape(sessionValue)
	return &http.Cookie{
		Name:     CookiePrefix + "_SESSION",
		Value:    Sign(sessionData) + "-" + sessionData,
		Path:     "/",
		HttpOnly: CookieHttpOnly,
		Secure:   CookieSecure,
		Expires:  ts.UTC(),
	}
}

// sessionTimeoutExpiredOrMissing returns a boolean of whether the session
// cookie is either not present or present but beyond its time to live; i.e.,
// whether there is not a valid session.
func sessionTimeoutExpiredOrMissing(session Session) bool {
	if exp, present := session[TS_KEY]; !present {
		return true
	} else if exp == "session" {
		return false
	} else if expInt, _ := strconv.Atoi(exp); int64(expInt) < time.Now().Unix() {
		return true
	}
	return false
}

// getSessionFromCookie returns a Session struct pulled from the signed
// session cookie.
func getSessionFromCookie(cookie *http.Cookie) Session {
	session := make(Session)

	// Separate the data from the signature.
	hyphen := strings.Index(cookie.Value, "-")
	if hyphen == -1 || hyphen >= len(cookie.Value)-1 {
		return session
	}
	sig, data := cookie.Value[:hyphen], cookie.Value[hyphen+1:]

	// Verify the signature.
	if !Verify(data, sig) {
		INFO.Println("Session cookie signature failed")
		return session
	}

	ParseKeyValueCookie(data, func(key, val string) {
		session[key] = val
	})

	if sessionTimeoutExpiredOrMissing(session) {
		session = make(Session)
	}

	return session
}

// SessionFilter is a Revel Filter that retrieves and sets the session cookie.
// Within Revel, it is available as a Session attribute on Controller instances.
// The name of the Session cookie is set as CookiePrefix + "_SESSION".
func SessionFilter(c *Controller, fc []Filter) {
	c.Session = restoreSession(c.Request.Request)
	// Make session vars available in templates as {{.session.xyz}}
	c.RenderArgs["session"] = c.Session

	fc[0](c, fc[1:])

	// Store the session (and sign it).
	c.SetCookie(c.Session.cookie())
}

// restoreSession returns either the current session, retrieved from the
// session cookie, or a new session.
func restoreSession(req *http.Request) Session {
	cookie, err := req.Cookie(CookiePrefix + "_SESSION")
	if err != nil {
		return make(Session)
	} else {
		return getSessionFromCookie(cookie)
	}
}

// getSessionExpirationCookie retrieves the cookie's time to live as a
// string of either the number of seconds, for a persistent cookie, or
// "session".
func getSessionExpirationCookie(t time.Time) string {
	if t.IsZero() {
		return "session"
	}
	return strconv.FormatInt(t.Unix(), 10)
}

// set session expiration to "session"
func (s Session) SetSessionNoExpiration() {
	s[TS_KEY] = "session"
}

// reset the session expiration to default
func (s Session) SetSessionDefaultExpiration() {
	delete(TS_KEY)
}
<commit_msg>fixed missing map<commit_after>package revel

import (
	"fmt"
	"github.com/streadway/simpleuuid"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"
)

// A signed cookie (and thus limited to 4kb in size).
// Restriction: Keys may not have a colon in them.
type Session map[string]string

const (
	SESSION_ID_KEY = "_ID"
	TS_KEY         = "_TS"
)

// expireAfterDuration is the time to live, in seconds, of a session cookie.
// It may be specified in config as "session.expires". Values greater than 0
// set a persistent cookie with a time to live as specified, and the value 0
// sets a session cookie.
var expireAfterDuration time.Duration

func init() {
	// Set expireAfterDuration, default to 30 days if no value in config
	OnAppStart(func() {
		var err error
		if expiresString, ok := Config.String("session.expires"); !ok {
			expireAfterDuration = 30 * 24 * time.Hour
		} else if expiresString == "session" {
			expireAfterDuration = 0
		} else if expireAfterDuration, err = time.ParseDuration(expiresString); err != nil {
			panic(fmt.Errorf("session.expires invalid: %s", err))
		}
	})
}

// Id retrieves from the cookie or creates a time-based UUID identifying this
// session.
func (s Session) Id() string {
	if uuidStr, ok := s[SESSION_ID_KEY]; ok {
		return uuidStr
	}

	uuid, err := simpleuuid.NewTime(time.Now())
	if err != nil {
		panic(err) // I don't think this can actually happen.
	}
	s[SESSION_ID_KEY] = uuid.String()
	return s[SESSION_ID_KEY]
}

// getSessionExpiration return a time.Time with the session's expiration date.
// If previous session has set to "session", remain it
func (s Session) getSessionExpiration() time.Time {
	if expireAfterDuration == 0 || s[TS_KEY] == "session" {
		return time.Time{}
	}
	return time.Now().Add(expireAfterDuration)
}

// cookie returns an http.Cookie containing the signed session.
func (s Session) cookie() *http.Cookie {
	var sessionValue string
	ts := s.getSessionExpiration()
	s[TS_KEY] = getSessionExpirationCookie(ts)
	for key, value := range s {
		if strings.ContainsAny(key, ":\x00") {
			panic("Session keys may not have colons or null bytes")
		}
		if strings.Contains(value, "\x00") {
			panic("Session values may not have null bytes")
		}
		sessionValue += "\x00" + key + ":" + value + "\x00"
	}

	sessionData := url.QueryEscape(sessionValue)
	return &http.Cookie{
		Name:     CookiePrefix + "_SESSION",
		Value:    Sign(sessionData) + "-" + sessionData,
		Path:     "/",
		HttpOnly: CookieHttpOnly,
		Secure:   CookieSecure,
		Expires:  ts.UTC(),
	}
}

// sessionTimeoutExpiredOrMissing returns a boolean of whether the session
// cookie is either not present or present but beyond its time to live; i.e.,
// whether there is not a valid session.
func sessionTimeoutExpiredOrMissing(session Session) bool {
	if exp, present := session[TS_KEY]; !present {
		return true
	} else if exp == "session" {
		return false
	} else if expInt, _ := strconv.Atoi(exp); int64(expInt) < time.Now().Unix() {
		return true
	}
	return false
}

// getSessionFromCookie returns a Session struct pulled from the signed
// session cookie.
func getSessionFromCookie(cookie *http.Cookie) Session {
	session := make(Session)

	// Separate the data from the signature.
	hyphen := strings.Index(cookie.Value, "-")
	if hyphen == -1 || hyphen >= len(cookie.Value)-1 {
		return session
	}
	sig, data := cookie.Value[:hyphen], cookie.Value[hyphen+1:]

	// Verify the signature.
	if !Verify(data, sig) {
		INFO.Println("Session cookie signature failed")
		return session
	}

	ParseKeyValueCookie(data, func(key, val string) {
		session[key] = val
	})

	if sessionTimeoutExpiredOrMissing(session) {
		session = make(Session)
	}

	return session
}

// SessionFilter is a Revel Filter that retrieves and sets the session cookie.
// Within Revel, it is available as a Session attribute on Controller instances.
// The name of the Session cookie is set as CookiePrefix + "_SESSION".
func SessionFilter(c *Controller, fc []Filter) {
	c.Session = restoreSession(c.Request.Request)
	// Make session vars available in templates as {{.session.xyz}}
	c.RenderArgs["session"] = c.Session

	fc[0](c, fc[1:])

	// Store the session (and sign it).
	c.SetCookie(c.Session.cookie())
}

// restoreSession returns either the current session, retrieved from the
// session cookie, or a new session.
func restoreSession(req *http.Request) Session {
	cookie, err := req.Cookie(CookiePrefix + "_SESSION")
	if err != nil {
		return make(Session)
	} else {
		return getSessionFromCookie(cookie)
	}
}

// getSessionExpirationCookie retrieves the cookie's time to live as a
// string of either the number of seconds, for a persistent cookie, or
// "session".
func getSessionExpirationCookie(t time.Time) string {
	if t.IsZero() {
		return "session"
	}
	return strconv.FormatInt(t.Unix(), 10)
}

// set session expiration to "session"
func (s Session) SetSessionNoExpiration() {
	s[TS_KEY] = "session"
}

// reset the session expiration to default
func (s Session) SetSessionDefaultExpiration() {
	delete(s, TS_KEY)
}
<|endoftext|>


<commit_before>package main

import (
	"testing"
)

func TestParser(t *testing.T) {
	tb := ToolBox{}
	tb.init("/data")

	tb.parse_exec("ls")
	tb.parse_exec("mgo abc")
	tb.parse_exec("show(1,2,3)")
	tb.parse_exec("show(1,2)")
	tb.parse_exec("replay(1,2)")
	tb.parse_exec("replay(1,2,3)")
	t.Log(t)
}
<commit_msg>update<commit_after>package main

import (
	"testing"
)

func TestParser(t *testing.T) {
	tb := ToolBox{}
	tb.init("/data")

	tb.parse_exec("ls")
	tb.parse_exec("mgo abc")
	tb.parse_exec("show(1,2,3)")
	tb.parse_exec("show(1,2)")
	tb.parse_exec("replay(1,2)")
	tb.parse_exec("replay(1,2,3)")
	tb.parse_exec("replay(1,2,3")
	t.Log(t)
}
<|endoftext|>


<commit_before>// session.go - mixnet client session
// Copyright (C) 2018  David Stainton.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package client

import (
	"context"
	"encoding/hex"
	"errors"
	"fmt"
	mrand "math/rand"
	"sync"
	"sync/atomic"
	"time"

	"github.com/katzenpost/client/config"
	cConstants "github.com/katzenpost/client/constants"
	"github.com/katzenpost/client/internal/pkiclient"
	"github.com/katzenpost/client/utils"
	coreConstants "github.com/katzenpost/core/constants"
	"github.com/katzenpost/core/crypto/ecdh"
	"github.com/katzenpost/core/log"
	"github.com/katzenpost/core/pki"
	"github.com/katzenpost/core/sphinx"
	sConstants "github.com/katzenpost/core/sphinx/constants"
	"github.com/katzenpost/core/worker"
	"github.com/katzenpost/minclient"
	"gopkg.in/eapache/channels.v1"
	"gopkg.in/op/go-logging.v1"
)

// Session is the struct type that keeps state for a given session.
type Session struct {
	worker.Worker

	cfg       *config.Config
	pkiClient pki.Client
	minclient *minclient.Client
	log       *logging.Logger

	fatalErrCh chan error
	opCh       chan workerOp

	eventCh   channels.Channel
	EventSink chan Event

	linkKey   *ecdh.PrivateKey
	onlineAt  time.Time
	hasPKIDoc bool

	egressQueue EgressQueue
	rescheduler *rescheduler

	surbIDMap        sync.Map // [sConstants.SURBIDLength]byte -> *Message
	sentWaitChanMap  sync.Map // MessageID -> chan *Message
	replyWaitChanMap sync.Map // MessageID -> chan []byte

	decoyLoopTally uint64
}

// New establishes a session with provider using key.
// This method will block until session is connected to the Provider.
func NewSession(
	ctx context.Context,
	fatalErrCh chan error,
	logBackend *log.Backend,
	cfg *config.Config,
	linkKey *ecdh.PrivateKey) (*Session, error) {
	var err error

	// create a pkiclient for our own client lookups
	// AND create a pkiclient for minclient's use
	proxyCfg := cfg.UpstreamProxyConfig()
	pkiClient, err := cfg.NewPKIClient(logBackend, proxyCfg)
	if err != nil {
		return nil, err
	}

	// create a pkiclient for minclient's use
	pkiClient2, err := cfg.NewPKIClient(logBackend, proxyCfg)
	if err != nil {
		return nil, err
	}
	pkiCacheClient := pkiclient.New(pkiClient2)

	clientLog := logBackend.GetLogger(fmt.Sprintf("%s@%s_client", cfg.Account.User, cfg.Account.Provider))

	s := &Session{
		cfg:         cfg,
		linkKey:     linkKey,
		pkiClient:   pkiClient,
		log:         clientLog,
		fatalErrCh:  fatalErrCh,
		eventCh:     channels.NewInfiniteChannel(),
		EventSink:   make(chan Event),
		opCh:        make(chan workerOp, 8),
		egressQueue: new(Queue),
	}
	// Configure the rescheduler instance
	s.rescheduler = NewRescheduler(s)
	// Configure and bring up the minclient instance.
	clientCfg := &minclient.ClientConfig{
		User:                cfg.Account.User,
		Provider:            cfg.Account.Provider,
		ProviderKeyPin:      cfg.Account.ProviderKeyPin,
		LinkKey:             s.linkKey,
		LogBackend:          logBackend,
		PKIClient:           pkiCacheClient,
		OnConnFn:            s.onConnection,
		OnMessageFn:         s.onMessage,
		OnACKFn:             s.onACK,
		OnDocumentFn:        s.onDocument,
		DialContextFn:       proxyCfg.ToDialContext("authority"),
		PreferedTransports:  cfg.Debug.PreferedTransports,
		MessagePollInterval: time.Duration(cfg.Debug.PollingInterval) * time.Millisecond,
		EnableTimeSync:      false, // Be explicit about it.
	}

	s.Go(s.eventSinkWorker)
	s.Go(s.garbageCollectionWorker)

	s.minclient, err = minclient.New(clientCfg)
	if err != nil {
		return nil, err
	}

	// block until we get the first PKI document
	// and then set our timers accordingly
	err = s.awaitFirstPKIDoc(ctx)
	if err != nil {
		return nil, err
	}
	s.Go(s.worker)
	return s, nil
}

func (s *Session) eventSinkWorker() {
	for {
		select {
		case <-s.HaltCh():
			s.log.Debugf("Event sink worker terminating gracefully.")
			return
		case e := <-s.eventCh.Out():
			select {
			case s.EventSink <- e.(Event):
			case <-s.HaltCh():
				s.log.Debugf("Event sink worker terminating gracefully.")
				return
			}
		}
	}
}

func (s *Session) garbageCollectionWorker() {
	timer := time.NewTimer(cConstants.GarbageCollectionInterval)
	defer timer.Stop()
	for {
		select {
		case <-s.HaltCh():
			s.log.Debugf("Garbage collection worker terminating gracefully.")
			return
		case <-timer.C:
			s.garbageCollect()
			timer.Reset(cConstants.GarbageCollectionInterval)
		}
	}
}

func (s *Session) garbageCollect() {
	s.log.Debug("Running garbage collection process.")
	// [sConstants.SURBIDLength]byte -> *Message
	surbIDMapRange := func(rawSurbID, rawMessage interface{}) bool {
		surbID := rawSurbID.([sConstants.SURBIDLength]byte)
		message := rawMessage.(*Message)
		if time.Now().After(message.SentAt.Add(message.ReplyETA).Add(cConstants.RoundTripTimeSlop)) {
			s.log.Debug("Garbage collecting SURB ID Map entry for Message ID %x", message.ID)
			s.surbIDMap.Delete(surbID)
			s.eventCh.In() <- &MessageIDGarbageCollected{
				MessageID: message.ID,
			}
		}
		return true
	}
	s.surbIDMap.Range(surbIDMapRange)
}

func (s *Session) awaitFirstPKIDoc(ctx context.Context) error {
	for {
		var qo workerOp
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-s.HaltCh():
			s.log.Debugf("Await first pki doc worker terminating gracefully")
			return errors.New("terminating gracefully")
		case <-time.After(time.Duration(s.cfg.Debug.InitialMaxPKIRetrievalDelay) * time.Second):
			return errors.New("timeout failure awaiting first PKI document")
		case qo = <-s.opCh:
		}
		switch op := qo.(type) {
		case opNewDocument:
			// Determine if PKI doc is valid. If not then abort.
			err := s.isDocValid(op.doc)
			if err != nil {
				s.fatalErrCh <- fmt.Errorf("aborting, PKI doc is not valid for our decoy traffic use case: %v", err)
				return err
			}
			s.setPollIntervalFromDoc(op.doc)
			return nil
		default:
			continue
		}
	}
	// NOT REACHED
}

// GetService returns a randomly selected service
// matching the specified service name
func (s *Session) GetService(serviceName string) (*utils.ServiceDescriptor, error) {
	doc := s.minclient.CurrentDocument()
	if doc == nil {
		return nil, errors.New("pki doc is nil")
	}
	serviceDescriptors := utils.FindServices(serviceName, doc)
	if len(serviceDescriptors) == 0 {
		return nil, errors.New("error, GetService failure, service not found in pki doc")
	}
	return &serviceDescriptors[mrand.Intn(len(serviceDescriptors))], nil
}

// OnConnection will be called by the minclient api
// upon connection change status to the Provider
func (s *Session) onConnection(err error) {
	s.log.Debugf("onConnection %v", err)
	s.eventCh.In() <- &ConnectionStatusEvent{
		IsConnected: err == nil,
		Err:         err,
	}
	s.opCh <- opConnStatusChanged{
		isConnected: err == nil,
	}
}

// OnMessage will be called by the minclient api
// upon receiving a message
func (s *Session) onMessage(ciphertextBlock []byte) error {
	s.log.Debugf("OnMessage")
	return nil
}

func (s *Session) incrementDecoyLoopTally() {
	atomic.AddUint64(&s.decoyLoopTally, 1)
}

func (s *Session) decrementDecoyLoopTally() {
	atomic.AddUint64(&s.decoyLoopTally, ^uint64(0))
}

// OnACK is called by the minclient api when we receive a SURB reply message.
func (s *Session) onACK(surbID *[sConstants.SURBIDLength]byte, ciphertext []byte) error {
	idStr := fmt.Sprintf("[%v]", hex.EncodeToString(surbID[:]))
	s.log.Infof("OnACK with SURBID %s", idStr)

	rawMessage, ok := s.surbIDMap.Load(*surbID)
	if !ok {
		s.log.Debug("Strange, received reply with unexpected SURBID")
		return nil
	}
	s.surbIDMap.Delete(*surbID)
	msg := rawMessage.(*Message)
	s.rescheduler.timerQ.Remove(msg)
	plaintext, err := sphinx.DecryptSURBPayload(ciphertext, msg.Key)
	if err != nil {
		s.log.Infof("Discarding SURB Reply, decryption failure: %s", err)
		return nil
	}
	if len(plaintext) != coreConstants.ForwardPayloadLength {
		s.log.Warningf("Discarding SURB %v: Invalid payload size: %v", idStr, len(plaintext))
		return nil
	}
	if msg.WithSURB && msg.IsDecoy {
		s.decrementDecoyLoopTally()
		return nil
	}

	if msg.IsBlocking {
		replyWaitChanRaw, ok := s.replyWaitChanMap.Load(*msg.ID)
		if !ok {
			//XXX: this can happen if a SURB-ACK arrives after a call to BlockingSendUnreliableMessage has timed-out
			// because the session.surbIDMap has not been deleted or garbage collected
			s.log.Warningf("Discarding surb %v for blocking message %x : caller likely timed-out", idStr, msg.ID)
			return nil
		}
		replyWaitChan := replyWaitChanRaw.(chan []byte)
		// do not block the worker if the receiver timed out!
		select {
		case replyWaitChan <- plaintext[2:]:
		default:
			s.log.Warningf("Failed to respond to a blocking message")
			close(replyWaitChan)
		}
	} else {
		s.eventCh.In() <- &MessageReplyEvent{
			MessageID: msg.ID,
			Payload:   plaintext[2:],
			Err:       nil,
		}
	}
	return nil
}

func (s *Session) onDocument(doc *pki.Document) {
	s.log.Debugf("onDocument(): Epoch %v", doc.Epoch)
	s.hasPKIDoc = true
	s.opCh <- opNewDocument{
		doc: doc,
	}
	s.eventCh.In() <- &NewDocumentEvent{
		Document: doc,
	}
}

func (s *Session) CurrentDocument() *pki.Document {
	return s.minclient.CurrentDocument()
}

func (s *Session) GetReunionConfig() *config.Reunion {
	return s.cfg.Reunion
}

func (s *Session) GetPandaConfig() *config.Panda {
	return s.cfg.Panda
}

func (s *Session) Shutdown() {
	s.Halt()
	s.rescheduler.timerQ.Halt()
	s.minclient.Shutdown()
	s.minclient.Wait()
}
<commit_msg>only try to remove reliable messages from rtx queue<commit_after>// session.go - mixnet client session
// Copyright (C) 2018  David Stainton.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package client

import (
	"context"
	"encoding/hex"
	"errors"
	"fmt"
	mrand "math/rand"
	"sync"
	"sync/atomic"
	"time"

	"github.com/katzenpost/client/config"
	cConstants "github.com/katzenpost/client/constants"
	"github.com/katzenpost/client/internal/pkiclient"
	"github.com/katzenpost/client/utils"
	coreConstants "github.com/katzenpost/core/constants"
	"github.com/katzenpost/core/crypto/ecdh"
	"github.com/katzenpost/core/log"
	"github.com/katzenpost/core/pki"
	"github.com/katzenpost/core/sphinx"
	sConstants "github.com/katzenpost/core/sphinx/constants"
	"github.com/katzenpost/core/worker"
	"github.com/katzenpost/minclient"
	"gopkg.in/eapache/channels.v1"
	"gopkg.in/op/go-logging.v1"
)

// Session is the struct type that keeps state for a given session.
type Session struct {
	worker.Worker

	cfg       *config.Config
	pkiClient pki.Client
	minclient *minclient.Client
	log       *logging.Logger

	fatalErrCh chan error
	opCh       chan workerOp

	eventCh   channels.Channel
	EventSink chan Event

	linkKey   *ecdh.PrivateKey
	onlineAt  time.Time
	hasPKIDoc bool

	egressQueue EgressQueue
	rescheduler *rescheduler

	surbIDMap        sync.Map // [sConstants.SURBIDLength]byte -> *Message
	sentWaitChanMap  sync.Map // MessageID -> chan *Message
	replyWaitChanMap sync.Map // MessageID -> chan []byte

	decoyLoopTally uint64
}

// New establishes a session with provider using key.
// This method will block until session is connected to the Provider.
func NewSession(
	ctx context.Context,
	fatalErrCh chan error,
	logBackend *log.Backend,
	cfg *config.Config,
	linkKey *ecdh.PrivateKey) (*Session, error) {
	var err error

	// create a pkiclient for our own client lookups
	// AND create a pkiclient for minclient's use
	proxyCfg := cfg.UpstreamProxyConfig()
	pkiClient, err := cfg.NewPKIClient(logBackend, proxyCfg)
	if err != nil {
		return nil, err
	}

	// create a pkiclient for minclient's use
	pkiClient2, err := cfg.NewPKIClient(logBackend, proxyCfg)
	if err != nil {
		return nil, err
	}
	pkiCacheClient := pkiclient.New(pkiClient2)

	clientLog := logBackend.GetLogger(fmt.Sprintf("%s@%s_client", cfg.Account.User, cfg.Account.Provider))

	s := &Session{
		cfg:         cfg,
		linkKey:     linkKey,
		pkiClient:   pkiClient,
		log:         clientLog,
		fatalErrCh:  fatalErrCh,
		eventCh:     channels.NewInfiniteChannel(),
		EventSink:   make(chan Event),
		opCh:        make(chan workerOp, 8),
		egressQueue: new(Queue),
	}
	// Configure the rescheduler instance
	s.rescheduler = NewRescheduler(s)
	// Configure and bring up the minclient instance.
	clientCfg := &minclient.ClientConfig{
		User:                cfg.Account.User,
		Provider:            cfg.Account.Provider,
		ProviderKeyPin:      cfg.Account.ProviderKeyPin,
		LinkKey:             s.linkKey,
		LogBackend:          logBackend,
		PKIClient:           pkiCacheClient,
		OnConnFn:            s.onConnection,
		OnMessageFn:         s.onMessage,
		OnACKFn:             s.onACK,
		OnDocumentFn:        s.onDocument,
		DialContextFn:       proxyCfg.ToDialContext("authority"),
		PreferedTransports:  cfg.Debug.PreferedTransports,
		MessagePollInterval: time.Duration(cfg.Debug.PollingInterval) * time.Millisecond,
		EnableTimeSync:      false, // Be explicit about it.
	}

	s.Go(s.eventSinkWorker)
	s.Go(s.garbageCollectionWorker)

	s.minclient, err = minclient.New(clientCfg)
	if err != nil {
		return nil, err
	}

	// block until we get the first PKI document
	// and then set our timers accordingly
	err = s.awaitFirstPKIDoc(ctx)
	if err != nil {
		return nil, err
	}
	s.Go(s.worker)
	return s, nil
}

func (s *Session) eventSinkWorker() {
	for {
		select {
		case <-s.HaltCh():
			s.log.Debugf("Event sink worker terminating gracefully.")
			return
		case e := <-s.eventCh.Out():
			select {
			case s.EventSink <- e.(Event):
			case <-s.HaltCh():
				s.log.Debugf("Event sink worker terminating gracefully.")
				return
			}
		}
	}
}

func (s *Session) garbageCollectionWorker() {
	timer := time.NewTimer(cConstants.GarbageCollectionInterval)
	defer timer.Stop()
	for {
		select {
		case <-s.HaltCh():
			s.log.Debugf("Garbage collection worker terminating gracefully.")
			return
		case <-timer.C:
			s.garbageCollect()
			timer.Reset(cConstants.GarbageCollectionInterval)
		}
	}
}

func (s *Session) garbageCollect() {
	s.log.Debug("Running garbage collection process.")
	// [sConstants.SURBIDLength]byte -> *Message
	surbIDMapRange := func(rawSurbID, rawMessage interface{}) bool {
		surbID := rawSurbID.([sConstants.SURBIDLength]byte)
		message := rawMessage.(*Message)
		if time.Now().After(message.SentAt.Add(message.ReplyETA).Add(cConstants.RoundTripTimeSlop)) {
			s.log.Debug("Garbage collecting SURB ID Map entry for Message ID %x", message.ID)
			s.surbIDMap.Delete(surbID)
			s.eventCh.In() <- &MessageIDGarbageCollected{
				MessageID: message.ID,
			}
		}
		return true
	}
	s.surbIDMap.Range(surbIDMapRange)
}

func (s *Session) awaitFirstPKIDoc(ctx context.Context) error {
	for {
		var qo workerOp
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-s.HaltCh():
			s.log.Debugf("Await first pki doc worker terminating gracefully")
			return errors.New("terminating gracefully")
		case <-time.After(time.Duration(s.cfg.Debug.InitialMaxPKIRetrievalDelay) * time.Second):
			return errors.New("timeout failure awaiting first PKI document")
		case qo = <-s.opCh:
		}
		switch op := qo.(type) {
		case opNewDocument:
			// Determine if PKI doc is valid. If not then abort.
			err := s.isDocValid(op.doc)
			if err != nil {
				s.fatalErrCh <- fmt.Errorf("aborting, PKI doc is not valid for our decoy traffic use case: %v", err)
				return err
			}
			s.setPollIntervalFromDoc(op.doc)
			return nil
		default:
			continue
		}
	}
	// NOT REACHED
}

// GetService returns a randomly selected service
// matching the specified service name
func (s *Session) GetService(serviceName string) (*utils.ServiceDescriptor, error) {
	doc := s.minclient.CurrentDocument()
	if doc == nil {
		return nil, errors.New("pki doc is nil")
	}
	serviceDescriptors := utils.FindServices(serviceName, doc)
	if len(serviceDescriptors) == 0 {
		return nil, errors.New("error, GetService failure, service not found in pki doc")
	}
	return &serviceDescriptors[mrand.Intn(len(serviceDescriptors))], nil
}

// OnConnection will be called by the minclient api
// upon connection change status to the Provider
func (s *Session) onConnection(err error) {
	s.log.Debugf("onConnection %v", err)
	s.eventCh.In() <- &ConnectionStatusEvent{
		IsConnected: err == nil,
		Err:         err,
	}
	s.opCh <- opConnStatusChanged{
		isConnected: err == nil,
	}
}

// OnMessage will be called by the minclient api
// upon receiving a message
func (s *Session) onMessage(ciphertextBlock []byte) error {
	s.log.Debugf("OnMessage")
	return nil
}

func (s *Session) incrementDecoyLoopTally() {
	atomic.AddUint64(&s.decoyLoopTally, 1)
}

func (s *Session) decrementDecoyLoopTally() {
	atomic.AddUint64(&s.decoyLoopTally, ^uint64(0))
}

// OnACK is called by the minclient api when we receive a SURB reply message.
func (s *Session) onACK(surbID *[sConstants.SURBIDLength]byte, ciphertext []byte) error {
	idStr := fmt.Sprintf("[%v]", hex.EncodeToString(surbID[:]))
	s.log.Infof("OnACK with SURBID %s", idStr)

	rawMessage, ok := s.surbIDMap.Load(*surbID)
	if !ok {
		s.log.Debug("Strange, received reply with unexpected SURBID")
		return nil
	}
	s.surbIDMap.Delete(*surbID)
	msg := rawMessage.(*Message)
	plaintext, err := sphinx.DecryptSURBPayload(ciphertext, msg.Key)
	if err != nil {
		s.log.Infof("Discarding SURB Reply, decryption failure: %s", err)
		return nil
	}
	if len(plaintext) != coreConstants.ForwardPayloadLength {
		s.log.Warningf("Discarding SURB %v: Invalid payload size: %v", idStr, len(plaintext))
		return nil
	}
	if msg.WithSURB && msg.IsDecoy {
		s.decrementDecoyLoopTally()
		return nil
	}
	if msg.Reliable {
		err := s.rescheduler.timerQ.Remove(msg)
		if err != nil {
			s.fatalErrCh <- fmt.Errorf("Failed removing reliable message from retransmit queue")
		}
	}
	if msg.IsBlocking {
		replyWaitChanRaw, ok := s.replyWaitChanMap.Load(*msg.ID)
		if !ok {
			//XXX: this can happen if a SURB-ACK arrives after a call to BlockingSendUnreliableMessage has timed-out
			// because the session.surbIDMap has not been deleted or garbage collected
			s.log.Warningf("Discarding surb %v for blocking message %x : caller likely timed-out", idStr, msg.ID)
			return nil
		}
		replyWaitChan := replyWaitChanRaw.(chan []byte)
		// do not block the worker if the receiver timed out!
		select {
		case replyWaitChan <- plaintext[2:]:
		default:
			s.log.Warningf("Failed to respond to a blocking message")
			close(replyWaitChan)
		}
	} else {
		s.eventCh.In() <- &MessageReplyEvent{
			MessageID: msg.ID,
			Payload:   plaintext[2:],
			Err:       nil,
		}
	}
	return nil
}

func (s *Session) onDocument(doc *pki.Document) {
	s.log.Debugf("onDocument(): Epoch %v", doc.Epoch)
	s.hasPKIDoc = true
	s.opCh <- opNewDocument{
		doc: doc,
	}
	s.eventCh.In() <- &NewDocumentEvent{
		Document: doc,
	}
}

func (s *Session) CurrentDocument() *pki.Document {
	return s.minclient.CurrentDocument()
}

func (s *Session) GetReunionConfig() *config.Reunion {
	return s.cfg.Reunion
}

func (s *Session) GetPandaConfig() *config.Panda {
	return s.cfg.Panda
}

func (s *Session) Shutdown() {
	s.Halt()
	s.rescheduler.timerQ.Halt()
	s.minclient.Shutdown()
	s.minclient.Wait()
}
<|endoftext|>