<commit_before>package fancy
/*
Copyright (c) 2009 Helmar Wodtke

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

import (
  "io";
  "os";
)

type Reader interface {
  ReadAt(buf []byte, pos int64) (n int, err os.Error);
  Read(b []byte) (n int, err os.Error);
  Slice(n int) []byte;
  Seek(off int64, whence int) (ret int64, err os.Error);
  ReadByte() (c byte, err os.Error);
  UnreadByte() os.Error;
  Size() int64;
}

// ------------------------------------------------------------------

const (
  _SECTOR_SIZE  = 512;
  _SECTOR_COUNT = 32;
)

type SecReaderT struct {
  cache     map[int64][]byte;
  age       map[int64]int;
  ticker    int;
  pos, size int64;
  f         io.ReaderAt;
}

func min(a, b int64) int64 {
  if a < b {
    return a
  }
  return b;
}

func (sr *SecReaderT) access(pos int64) (sl []byte, p int) {
  p = int(pos % _SECTOR_SIZE);
  pos /= _SECTOR_SIZE;
  if s, ok := sr.cache[pos]; ok {
    if sr.age[pos] != sr.ticker {
      sr.ticker++;
      sr.age[pos] = sr.ticker;
    }
    return s, p;
  }
  if len(sr.cache) >= _SECTOR_COUNT {
    a := sr.ticker;
    old := int64(0);
    for k := range sr.age {
      if sr.age[k] < a {
        old = k;
        a = sr.age[k];
      }
    }
    sr.cache[old] = nil, false;
    sr.age[old] = 0, false;
  }
  sr.ticker++;
  sl = make([]byte, min(sr.size-pos*_SECTOR_SIZE, _SECTOR_SIZE));
  sr.f.ReadAt(sl, pos*_SECTOR_SIZE);
  sr.cache[pos] = sl;
  sr.age[pos] = sr.ticker;
  return;
}

func (sr *SecReaderT) ReadAt(buf []byte, pos int64) (n int, err os.Error) {
  if pos >= sr.size {
    return 0, os.EOF
  }
  b, p := sr.access(pos);
  for ; p < _SECTOR_SIZE && n < len(buf); p++ {
    buf[n] = b[p];
    n++;
  }
  if secs := (len(buf) - n) / _SECTOR_SIZE; secs > 0 {
    sr.f.ReadAt(buf[n:n+secs*_SECTOR_SIZE], pos+int64(n));
    n += secs * _SECTOR_SIZE;
  }
  if len(buf)-n > 0 {
    b, p = sr.access(pos + int64(n));
    for ; n < len(buf); p++ {
      buf[n] = b[p];
      n++;
    }
  }
  if pos+int64(n) >= sr.size {
    n -= int(pos + int64(n) - sr.size)
  }
  return;
}

func (sr *SecReaderT) Read(b []byte) (n int, err os.Error) {
  n, err = sr.ReadAt(b, sr.pos);
  sr.pos += int64(n);
  return;
}

func (sr *SecReaderT) Seek(off int64, whence int) (ret int64, err os.Error) {
  ret = sr.pos;
  switch whence {
  case 0:
    sr.pos = 0
  case 2:
    sr.pos = sr.size
  }
  sr.pos += off;
  return;
}

func (sr *SecReaderT) ReadByte() (c byte, err os.Error) {
  if sr.pos < sr.size {
    b, p := sr.access(sr.pos);
    c = b[p];
    sr.pos++;
  } else {
    err = os.EOF
  }
  return;
}

func (sr *SecReaderT) UnreadByte() os.Error {
  sr.pos--;
  return nil;
}

func (sr *SecReaderT) Size() int64 { return sr.size }

func (sr *SecReaderT) Slice(n int) []byte {
  r := make([]byte, n);
  sr.Read(r);
  return r;
}

func SecReader(f io.ReaderAt, size int64) Reader {
  sr := new(SecReaderT);
  sr.f = f;
  sr.size = size;
  sr.cache = make(map[int64][]byte);
  sr.age = make(map[int64]int);
  return sr;
}

// ------------------------------------------------------------------

type SliceReaderT struct {
  bin []byte;
  pos int64;
}

func (sl *SliceReaderT) ReadAt(b []byte, off int64) (n int, err os.Error) {
  for n := 0; n < len(b); n++ {
    if off >= int64(len(sl.bin)) {
      if n > 0 {
        break
      }
      return n, os.EOF;
    }
    b[n] = sl.bin[off];
    off++;
  }
  return len(b), nil;
}

func (sl *SliceReaderT) Read(b []byte) (n int, err os.Error) {
  n, err = sl.ReadAt(b, sl.pos);
  sl.pos += int64(n);
  return;
}

func (sl *SliceReaderT) Seek(off int64, whence int) (ret int64, err os.Error) {
  ret = sl.pos;
  switch whence {
  case 0:
    sl.pos = 0
  case 2:
    sl.pos = int64(len(sl.bin))
  }
  sl.pos += off;
  return;
}

func (sl *SliceReaderT) Size() int64 { return int64(len(sl.bin)) }

func (sl *SliceReaderT) ReadByte() (c byte, err os.Error) {
  if sl.pos < int64(len(sl.bin)) {
    c = sl.bin[sl.pos];
    sl.pos++;
  } else {
    err = os.EOF
  }
  return;
}

func (sl *SliceReaderT) UnreadByte() os.Error {
  sl.pos--;
  return nil;
}

func (sl *SliceReaderT) Slice(n int) []byte {
  sl.pos += int64(n);
  return sl.bin[sl.pos-int64(n) : sl.pos];
}

func SliceReader(bin []byte) Reader {
  r := new(SliceReaderT);
  r.bin = bin;
  return r;
}

// ------------------------------------------------------------------

func ReadAndClose(f io.ReadCloser, err os.Error) []byte {
  if err != nil {
    return []byte{}
  }
  r, _ := io.ReadAll(f);
  f.Close();
  return r;
}

func FileReader(fn string) Reader {
  dir, err := os.Stat(fn);
  if err != nil {
    return nil
  }
  fil, err := os.Open(fn, os.O_RDONLY, -1);
  if err != nil {
    return nil
  }
  return SecReader(fil, int64(dir.Size));
}

func ReadAll(f io.Reader) []byte {
  r, _ := io.ReadAll(f);
  return r;
}
<commit_msg>Experiment.<commit_after>package fancy
/*
Copyright (c) 2009 Helmar Wodtke

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

import (
  "io";
  "bufio";
  "os";
)

type Reader interface {
  ReadAt(buf []byte, pos int64) (n int, err os.Error);
  Read(b []byte) (n int, err os.Error);
  Slice(n int) []byte;
  Seek(off int64, whence int) (ret int64, err os.Error);
  ReadByte() (c byte, err os.Error);
  UnreadByte() os.Error;
  Size() int64;
}

// ------------------------------------------------------------------

const (
  _SECTOR_SIZE  = 512;
  _SECTOR_COUNT = 32;
)

type SecReaderT struct {
  cache     map[int64][]byte;
  age       map[int64]int;
  ticker    int;
  pos, size int64;
  f         io.ReaderAt;
}

func min(a, b int64) int64 {
  if a < b {
    return a
  }
  return b;
}

func (sr *SecReaderT) access(pos int64) (sl []byte, p int) {
  p = int(pos % _SECTOR_SIZE);
  pos /= _SECTOR_SIZE;
  if s, ok := sr.cache[pos]; ok {
    if sr.age[pos] != sr.ticker {
      sr.ticker++;
      sr.age[pos] = sr.ticker;
    }
    return s, p;
  }
  if len(sr.cache) >= _SECTOR_COUNT {
    a := sr.ticker;
    old := int64(0);
    for k := range sr.age {
      if sr.age[k] < a {
        old = k;
        a = sr.age[k];
      }
    }
    sr.cache[old] = nil, false;
    sr.age[old] = 0, false;
  }
  sr.ticker++;
  sl = make([]byte, min(sr.size-pos*_SECTOR_SIZE, _SECTOR_SIZE));
  sr.f.ReadAt(sl, pos*_SECTOR_SIZE);
  sr.cache[pos] = sl;
  sr.age[pos] = sr.ticker;
  return;
}

func (sr *SecReaderT) ReadAt(buf []byte, pos int64) (n int, err os.Error) {
  if pos >= sr.size {
    return 0, os.EOF
  }
  b, p := sr.access(pos);
  for ; p < _SECTOR_SIZE && n < len(buf); p++ {
    buf[n] = b[p];
    n++;
  }
  if secs := (len(buf) - n) / _SECTOR_SIZE; secs > 0 {
    sr.f.ReadAt(buf[n:n+secs*_SECTOR_SIZE], pos+int64(n));
    n += secs * _SECTOR_SIZE;
  }
  if len(buf)-n > 0 {
    b, p = sr.access(pos + int64(n));
    for ; n < len(buf); p++ {
      buf[n] = b[p];
      n++;
    }
  }
  if pos+int64(n) >= sr.size {
    n -= int(pos + int64(n) - sr.size)
  }
  return;
}

func (sr *SecReaderT) Read(b []byte) (n int, err os.Error) {
  n, err = sr.ReadAt(b, sr.pos);
  sr.pos += int64(n);
  return;
}

func (sr *SecReaderT) Seek(off int64, whence int) (ret int64, err os.Error) {
  ret = sr.pos;
  switch whence {
  case 0:
    sr.pos = 0
  case 2:
    sr.pos = sr.size
  }
  sr.pos += off;
  return;
}

func (sr *SecReaderT) ReadByte() (c byte, err os.Error) {
  if sr.pos < sr.size {
    b, p := sr.access(sr.pos);
    c = b[p];
    sr.pos++;
  } else {
    err = os.EOF
  }
  return;
}

func (sr *SecReaderT) UnreadByte() os.Error {
  sr.pos--;
  return nil;
}

func (sr *SecReaderT) Size() int64 { return sr.size }

func (sr *SecReaderT) Slice(n int) []byte {
  r := make([]byte, n);
  sr.Read(r);
  return r;
}

// grmpf: Next is for AUTOGENERATE!
// The thing here is only (!) for convenience.
func (sr *SecReaderT) ReadBytes(delim byte) ([]byte, os.Error) {
  return bufio.NewReader(sr).ReadBytes(delim);
}

func SecReader(f io.ReaderAt, size int64) Reader {
  sr := new(SecReaderT);
  sr.f = f;
  sr.size = size;
  sr.cache = make(map[int64][]byte);
  sr.age = make(map[int64]int);
  return sr;
}

// ------------------------------------------------------------------

type SliceReaderT struct {
  bin []byte;
  pos int64;
}

func (sl *SliceReaderT) ReadAt(b []byte, off int64) (n int, err os.Error) {
  for n := 0; n < len(b); n++ {
    if off >= int64(len(sl.bin)) {
      if n > 0 {
        break
      }
      return n, os.EOF;
    }
    b[n] = sl.bin[off];
    off++;
  }
  return len(b), nil;
}

func (sl *SliceReaderT) Read(b []byte) (n int, err os.Error) {
  n, err = sl.ReadAt(b, sl.pos);
  sl.pos += int64(n);
  return;
}

func (sl *SliceReaderT) Seek(off int64, whence int) (ret int64, err os.Error) {
  ret = sl.pos;
  switch whence {
  case 0:
    sl.pos = 0
  case 2:
    sl.pos = int64(len(sl.bin))
  }
  sl.pos += off;
  return;
}

func (sl *SliceReaderT) Size() int64 { return int64(len(sl.bin)) }

func (sl *SliceReaderT) ReadByte() (c byte, err os.Error) {
  if sl.pos < int64(len(sl.bin)) {
    c = sl.bin[sl.pos];
    sl.pos++;
  } else {
    err = os.EOF
  }
  return;
}

func (sl *SliceReaderT) UnreadByte() os.Error {
  sl.pos--;
  return nil;
}

func (sl *SliceReaderT) Slice(n int) []byte {
  sl.pos += int64(n);
  return sl.bin[sl.pos-int64(n) : sl.pos];
}

// grmpf: Next is for AUTOGENERATE!
// The thing here is only (!) for convenience.
func (sl *SliceReaderT) ReadBytes(delim byte) ([]byte, os.Error) {
  return bufio.NewReader(sl).ReadBytes(delim);
}

func SliceReader(bin []byte) Reader {
  r := new(SliceReaderT);
  r.bin = bin;
  return r;
}

// ------------------------------------------------------------------

func ReadAndClose(f io.ReadCloser, err os.Error) []byte {
  if err != nil {
    return []byte{}
  }
  r, _ := io.ReadAll(f);
  f.Close();
  return r;
}

func FileReader(fn string) Reader {
  dir, err := os.Stat(fn);
  if err != nil {
    return nil
  }
  fil, err := os.Open(fn, os.O_RDONLY, -1);
  if err != nil {
    return nil
  }
  return SecReader(fil, int64(dir.Size));
}

func ReadAll(f io.Reader) []byte {
  r, _ := io.ReadAll(f);
  return r;
}

// ------------------------------------------------------------------
<|endoftext|>


<commit_before>package fdate

type FDate struct {
}

type Month int

const (
	Vendémiaire = 1 + iota
	Brumaire
	Frimaire
	Nivôse
	Pluviôse
	Ventôse
	Germinal
	Floréal
	Prairial
	Messidor
	Thermidor
	Fructidor
)

var months = [...]string {
	"Vendémiaire",
	"Brumaire",
	"Frimaire",
	"Nivôse",
	"Pluviôse",
	"Ventôse",
	"Germinal",
	"Floréal",
	"Prairial",
	"Messidor",
	"Thermidor",
	"Fructidor",
}

type Weekday int

const (
	primidi = iota
	duodi
	tridi
	quartidi
	quintidi
	sextidi
	septidi
	octidi
	nonidi
	décadi
)

var days = [...]string {
	"primidi",
	"duodi",
	"tridi",
	"quartidi",
	"quintidi",
	"sextidi",
	"septidi",
	"octidi",
	"nonidi",
	"décadi",
}
<commit_msg>date struct updated<commit_after>package fdate

type FDate struct {
	Month   string
	Weekday string
	DayName string // TODO: import these
	Year    RomanNumeral
}

type Month int

const (
	Vendémiaire = 1 + iota
	Brumaire
	Frimaire
	Nivôse
	Pluviôse
	Ventôse
	Germinal
	Floréal
	Prairial
	Messidor
	Thermidor
	Fructidor
)

var months = [...]string {
	"Vendémiaire",
	"Brumaire",
	"Frimaire",
	"Nivôse",
	"Pluviôse",
	"Ventôse",
	"Germinal",
	"Floréal",
	"Prairial",
	"Messidor",
	"Thermidor",
	"Fructidor",
}

type Weekday int

const (
	primidi = iota
	duodi
	tridi
	quartidi
	quintidi
	sextidi
	septidi
	octidi
	nonidi
	décadi
)

var days = [...]string {
	"primidi",
	"duodi",
	"tridi",
	"quartidi",
	"quintidi",
	"sextidi",
	"septidi",
	"octidi",
	"nonidi",
	"décadi",
}
<|endoftext|>


<commit_before>package main

import "fmt"

func main() {
	fmt.Println("Simmer down...")

	// Parse arguments here

	// Read source strings from a file
	//	f, err := os.Open("resources/test_strings.txt")
	//	if err != nil {
	//		fmt.Println("Oops")
	//		os.Exit(1)
	//	}
}
<commit_msg>for source string, find best match among target strings<commit_after>package main

import (
	"bufio"
	"fmt"
	"os"

	"github.com/xrash/smetrics"
)

// TODO Refactor big hairy main - Apply boy scout rule
func main() {
	fmt.Println("Simmer down...")
	boostThreshold := 0.7 // boostThreshold = minimum score for a string that gets boosted. This value was set to 0.7 in Winkler's papers.
	prefixSize := 4       // prefixSize = size of the initial prefix considered. This value was set to 4 in Winkler's papers.
	verbose := true

	// TODO Parse arguments here

	//Read source strings from a file
	source, err := os.Open("resources/test_strings.txt")
	exitOnError(err, "Oops cannot find source")

	target, err := os.Open("resources/ref_strings.txt")
	exitOnError(err, "Oops cannot find target")

	defer target.Close()
	targetScanner := bufio.NewScanner(target)

	//Get unique target strings
	//Use a map to get the set of unique target strings
	targetStrings := make(map[string]bool)
	for targetScanner.Scan() {
		targetStr := targetScanner.Text()
		if verbose {
			if targetStrings[targetStr] {
				fmt.Print(targetStr, " is already in targetStrings map\n") // debug
			} else {
				fmt.Print("Adding ", targetStr, " to targetStrings map\n") // debug
			}
		}
		targetStrings[targetStr] = true
	}

	if err := targetScanner.Err(); err != nil {
		// TODO do sth with error
	}

	defer source.Close()
	sourceScanner := bufio.NewScanner(source)

	bestMatches := [][]string{}
	for sourceScanner.Scan() {
		sourceStr := sourceScanner.Text()
		bestMatchDist := 0.0
		bestMatchStr := ""
		for targetStr := range targetStrings {
			dist := smetrics.JaroWinkler(sourceStr, targetStr, boostThreshold, prefixSize)
			if dist > bestMatchDist {
				bestMatchDist = dist
				bestMatchStr = targetStr
			}
			if verbose {
				fmt.Print(sourceStr, " ", targetStr, " ", dist, "\n")
			}
		}
		bestMatches = append(bestMatches, []string{sourceStr, bestMatchStr})
		if verbose {
			fmt.Print("best match ", bestMatchStr, "\n")
		}
	}

	if err := sourceScanner.Err(); err != nil {
		// TODO do sth with error
	}

	//At this point we have the best matches
	// TODO write out to a file
	for i := range bestMatches {
		fmt.Print(bestMatches[i][0], ",", bestMatches[i][1], "\n")
	}

}

func exitOnError(e error, msg string) {
	if e != nil {
		fmt.Println(msg)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// +build ignore

/* Copyright 2018 The Bazel Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// update_proto_csv reads the contents of the @go_googleapis repository
// and updates the proto.csv file. This must be done manually.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strings"
)

var (
	progName           = filepath.Base(os.Args[0])
	protoCsvPath       = flag.String("proto_csv", "", "Path to proto.csv to update")
	googleapisRootPath = flag.String("go_googleapis", "", "Path to @go_googleapis repository root directory")
)

var prefix = `# This file lists special protos that Gazelle knows how to import. This is used to generate
# code for proto and Go resolvers.
#
# Generated by internal/language/proto/gen/update_proto_csv.go
# Do not edit directly.
#
# proto name,proto label,go import path,go proto label
google/protobuf/any.proto,@com_google_protobuf//:any_proto,github.com/golang/protobuf/ptypes/any,@io_bazel_rules_go//proto/wkt:any_go_proto
google/protobuf/api.proto,@com_google_protobuf//:api_proto,google.golang.org/genproto/protobuf/api,@io_bazel_rules_go//proto/wkt:api_go_proto
google/protobuf/compiler/plugin.proto,@com_google_protobuf//:compiler_plugin_proto,github.com/golang/protobuf/protoc-gen-go/plugin,@io_bazel_rules_go//proto/wkt:compiler_plugin_go_proto
google/protobuf/descriptor.proto,@com_google_protobuf//:descriptor_proto,github.com/golang/protobuf/protoc-gen-go/descriptor,@io_bazel_rules_go//proto/wkt:descriptor_go_proto
google/protobuf/duration.proto,@com_google_protobuf//:duration_proto,github.com/golang/protobuf/ptypes/duration,@io_bazel_rules_go//proto/wkt:duration_go_proto
google/protobuf/empty.proto,@com_google_protobuf//:empty_proto,github.com/golang/protobuf/ptypes/empty,@io_bazel_rules_go//proto/wkt:empty_go_proto
google/protobuf/field_mask.proto,@com_google_protobuf//:field_mask_proto,google.golang.org/genproto/protobuf/field_mask,@io_bazel_rules_go//proto/wkt:field_mask_go_proto
google/protobuf/source_context.proto,@com_google_protobuf//:source_context_proto,google.golang.org/genproto/protobuf/source_context,@io_bazel_rules_go//proto/wkt:source_context_go_proto
google/protobuf/struct.proto,@com_google_protobuf//:struct_proto,github.com/golang/protobuf/ptypes/struct,@io_bazel_rules_go//proto/wkt:struct_go_proto
google/protobuf/timestamp.proto,@com_google_protobuf//:timestamp_proto,github.com/golang/protobuf/ptypes/timestamp,@io_bazel_rules_go//proto/wkt:timestamp_go_proto
google/protobuf/type.proto,@com_google_protobuf//:type_proto,google.golang.org/genproto/protobuf/ptype,@io_bazel_rules_go//proto/wkt:type_go_proto
google/protobuf/wrappers.proto,@com_google_protobuf//:wrappers_proto,github.com/golang/protobuf/ptypes/wrappers,@io_bazel_rules_go//proto/wkt:wrappers_go_proto
`

func main() {
	log.SetPrefix(progName + ": ")
	log.SetFlags(0)
	flag.Parse()

	if *protoCsvPath == "" {
		log.Fatal("-proto must be set")
	}
	if *googleapisRootPath == "" {
		log.Fatal("-go_googleapis must be set")
	}

	protoContent := &bytes.Buffer{}
	protoContent.WriteString(prefix)

	err := filepath.Walk(*googleapisRootPath, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !strings.HasSuffix(path, ".proto") {
			return nil
		}
		relPath, err := filepath.Rel(*googleapisRootPath, path)
		if err != nil || strings.HasPrefix(relPath, "..") {
			log.Panicf("file %q not in googleapisRootPath %q", path, *googleapisRootPath)
		}
		relPath = filepath.ToSlash(relPath)

		if strings.HasPrefix(relPath, "google/api/experimental/") {
			// Special case: these protos need to be built together with protos in
			// google/api. They have the same 'option go_package'. The proto_library
			// rule requires them to be in the same Bazel package, so we don't
			// create a build file in experimental.
			packagePath := "google.golang.org/genproto/googleapis/api"
			protoLabel, goLabel := protoLabels("google/api/x", "api")
			fmt.Fprintf(protoContent, "%s,%s,%s,%s\n", relPath, protoLabel, packagePath, goLabel)
			return nil
		}

		packagePath, packageName, err := loadGoPackage(path, relPath)
		if err != nil {
			log.Print(err)
			return nil
		}

		protoLabel, goLabel := protoLabels(relPath, packageName)

		fmt.Fprintf(protoContent, "%s,%s,%s,%s\n", relPath, protoLabel, packagePath, goLabel)
		return nil
	})
	if err != nil {
		log.Fatal(err)
	}

	if err := ioutil.WriteFile(*protoCsvPath, protoContent.Bytes(), 0666); err != nil {
		log.Fatal(err)
	}
}

var goPackageRx = regexp.MustCompile(`option go_package = "([^"]*)"`)

func loadGoPackage(protoPath, relPath string) (packagePath, packageName string, err error) {
	data, err := ioutil.ReadFile(protoPath)
	if err != nil {
		return "", "", err
	}
	m := goPackageRx.FindSubmatch(data)
	if m == nil {
		return "", "", fmt.Errorf("%s: does not contain 'option go_package'", protoPath)
	}
	opt := string(m[1])
	if i := strings.LastIndexByte(opt, ';'); i >= 0 {
		return opt[:i], opt[i+1:], nil
	}
	if i := strings.LastIndexByte(opt, '/'); i >= 0 {
		return opt, opt[i+1:], nil
	}
	return opt, opt, nil
}

func protoLabels(relPath, packageName string) (protoLabel, goLabel string) {
	dir := path.Dir(relPath)
	protoLabel = fmt.Sprintf("@go_googleapis//%s:%s_proto", dir, packageName)
	goLabel = fmt.Sprintf("@go_googleapis//%s:%s_go_proto", dir, packageName)
	return protoLabel, goLabel
}
<commit_msg>Add ability to update proto.csv by results of the bazel query. (#426)<commit_after>// +build ignore

/* Copyright 2018 The Bazel Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// update_proto_csv reads the contents of the @go_googleapis repository
// and updates the proto.csv file. This must be done manually.

package main

import (
	"bytes"
	"encoding/xml"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
)

var (
	progName                    = filepath.Base(os.Args[0])
	protoCsvPath                = flag.String("proto_csv", "", "Path to proto.csv to update")
	goGoogleapisRootPath        = flag.String("go_googleapis", "", "Path to @go_googleapis repository root directory")
	comGoogleGoogleapisRootPath = flag.String("com_google_googleapis", "", "Path to @com_google_googleapis repository root directory")
	bazelPath                   = flag.String("bazel", "bazel", "Path to bazel executable")
)

var prefix = `# This file lists special protos that Gazelle knows how to import. This is used to generate
# code for proto and Go resolvers.
#
# Generated by internal/language/proto/gen/update_proto_csv.go
# Do not edit directly.
#
# proto name,proto label,go import path,go proto label
google/protobuf/any.proto,@com_google_protobuf//:any_proto,github.com/golang/protobuf/ptypes/any,@io_bazel_rules_go//proto/wkt:any_go_proto
google/protobuf/api.proto,@com_google_protobuf//:api_proto,google.golang.org/genproto/protobuf/api,@io_bazel_rules_go//proto/wkt:api_go_proto
google/protobuf/compiler/plugin.proto,@com_google_protobuf//:compiler_plugin_proto,github.com/golang/protobuf/protoc-gen-go/plugin,@io_bazel_rules_go//proto/wkt:compiler_plugin_go_proto
google/protobuf/descriptor.proto,@com_google_protobuf//:descriptor_proto,github.com/golang/protobuf/protoc-gen-go/descriptor,@io_bazel_rules_go//proto/wkt:descriptor_go_proto
google/protobuf/duration.proto,@com_google_protobuf//:duration_proto,github.com/golang/protobuf/ptypes/duration,@io_bazel_rules_go//proto/wkt:duration_go_proto
google/protobuf/empty.proto,@com_google_protobuf//:empty_proto,github.com/golang/protobuf/ptypes/empty,@io_bazel_rules_go//proto/wkt:empty_go_proto
google/protobuf/field_mask.proto,@com_google_protobuf//:field_mask_proto,google.golang.org/genproto/protobuf/field_mask,@io_bazel_rules_go//proto/wkt:field_mask_go_proto
google/protobuf/source_context.proto,@com_google_protobuf//:source_context_proto,google.golang.org/genproto/protobuf/source_context,@io_bazel_rules_go//proto/wkt:source_context_go_proto
google/protobuf/struct.proto,@com_google_protobuf//:struct_proto,github.com/golang/protobuf/ptypes/struct,@io_bazel_rules_go//proto/wkt:struct_go_proto
google/protobuf/timestamp.proto,@com_google_protobuf//:timestamp_proto,github.com/golang/protobuf/ptypes/timestamp,@io_bazel_rules_go//proto/wkt:timestamp_go_proto
google/protobuf/type.proto,@com_google_protobuf//:type_proto,google.golang.org/genproto/protobuf/ptype,@io_bazel_rules_go//proto/wkt:type_go_proto
google/protobuf/wrappers.proto,@com_google_protobuf//:wrappers_proto,github.com/golang/protobuf/ptypes/wrappers,@io_bazel_rules_go//proto/wkt:wrappers_go_proto
`

func main() {
	log.SetPrefix(progName + ": ")
	log.SetFlags(0)
	flag.Parse()

	if *protoCsvPath == "" {
		log.Fatal("-proto_csv must be set")
	}

	protoContent := &bytes.Buffer{}
	protoContent.WriteString(prefix)

	if *goGoogleapisRootPath != "" {
		if err := generateFromPath(protoContent, *goGoogleapisRootPath); err != nil {
			log.Fatal(err)
		}
	} else if *comGoogleGoogleapisRootPath != "" {
		if err := generateFromQuery(protoContent, *bazelPath, *comGoogleGoogleapisRootPath); err != nil {
			log.Fatal(err)
		}
	} else {
		log.Fatal("either -com_google_googleapis or -go_googleapis must be set")
	}

	if err := ioutil.WriteFile(*protoCsvPath, protoContent.Bytes(), 0666); err != nil {
		log.Fatal(err)
	}
}

//
// Process -com_google_googleapis case
//
const repoPrefix = "@com_google_googleapis"

var bazelQuery = []string{
	`query`,
	`kind("proto_library rule", //google/...)`,
	`--output`,
	`xml`,
	`--experimental_enable_repo_mapping`,
}

// Represents the relevant attributes of go_proto_library rules.
// The "proto" field is used to map to a proto_library rule; it represents either the value of a
// "proto" attr or an item in "protos" attr.
type goProtoLibraryInfo struct {
	name, importpath, proto string
}

type Query struct {
	XMLName xml.Name `xml:"query"`
	Version string   `xml:"version,attr"`
	Rules   []Rule   `xml:"rule"`
}

type Rule struct {
	XMLName xml.Name `xml:"rule"`
	Class   string   `xml:"class,attr"`
	Name    string   `xml:"name,attr"`
	Strings []String `xml:"string"`
	Labels  []Label  `xml:"label"`
	Lists   []List   `xml:"list"`
}

type List struct {
	XMLName xml.Name `xml:"list"`
	Name    string   `xml:"name,attr"`
	Labels  []Label  `xml:"label"`
}

type Label struct {
	XMLName xml.Name `xml:"label"`
	Name    string   `xml:"name,attr"`
	Value   string   `xml:"value,attr"`
}

type String struct {
	XMLName xml.Name `xml:"string"`
	Name    string   `xml:"name,attr"`
	Value   string   `xml:"value,attr"`
}

func generateFromQuery(w io.Writer, bazelPath, workspacePath string) error {
	xmlData, err := runBazelQuery(bazelPath, workspacePath)
	if err != nil {
		return err
	}

	query, err := readQueryXml(xmlData)
	if err != nil {
		return err
	}

	relPathList, err := processQuery(query)
	for _, v := range relPathList {
		if _, err = fmt.Fprintln(w, v); err != nil {
			return err
		}
	}

	return nil
}

func runBazelQuery(bazelPath, workspacePath string) ([]byte, error) {
	cmd := exec.Command(bazelPath, bazelQuery...)
	cmd.Dir = workspacePath
	return cmd.Output()
}

func readQueryXml(xmlData []byte) (Query, error) {
	if bytes.HasPrefix(xmlData, []byte(`<?xml version="1.1"`)) {
		copy(xmlData, []byte(`<?xml version="1.0"`))
	}
	query := Query{}
	err := xml.Unmarshal(xmlData, &query)
	return query, err
}

func processQuery(query Query) ([]string, error) {
	protoLabelMap := make(map[string]goProtoLibraryInfo)
	for _, rule := range query.Rules {
		if "go_proto_library" == rule.Class {
			m, err := processGoProtoLibraryRule(rule)
			if err != nil {
				continue
			}
			for k, v := range m {
				protoLabelMap[k] = v
			}
		}
	}

	var relPathList []string
	for _, rule := range query.Rules {
		if "proto_library" != rule.Class {
			continue
		}
		if protoLibraryInfo, present := protoLabelMap[rule.Name]; present {
			paths, err := processProtoLibraryRule(rule, protoLibraryInfo)
			if err != nil {
				continue
			}
			relPathList = append(relPathList, paths...)
		}
	}

	sort.Strings(relPathList)
	return relPathList, nil
}

func processGoProtoLibraryRule(rule Rule) (map[string]goProtoLibraryInfo, error) {
	protoLabelMap := make(map[string]goProtoLibraryInfo)

	info := goProtoLibraryInfo{name: repoPrefix + rule.Name}
	for _, str := range rule.Strings {
		if "importpath" == str.Name {
			info.importpath = str.Value
			break
		}
	}

	if info.importpath == "" {
		// should never happen
		return protoLabelMap, fmt.Errorf("go_proto_library does not have 'importpath' argument")
	}

	// "proto" argument case (single value)
	protoLabel := ""
	for _, label := range rule.Labels {
		if "proto" == label.Name {
			protoLabel = label.Value
			break
		}
	}

	if protoLabel != "" {
		info.proto = repoPrefix + protoLabel
		protoLabelMap[protoLabel] = info
		return protoLabelMap, nil
	}

	// "protos" argument case (multiple values)
	for _, list := range rule.Lists {
		if "protos" == list.Name {
			for _, label := range list.Labels {
				info.proto = repoPrefix + label.Value
				protoLabelMap[label.Value] = info
			}
		}
	}

	return protoLabelMap, nil
}

func processProtoLibraryRule(rule Rule, info goProtoLibraryInfo) ([]string, error) {
	extraSuffix := "_with_info"

	var relPathList []string

	for _, list := range rule.Lists {
		if "srcs" != list.Name {
			continue
		}
		for _, label := range list.Labels {
			relPath := label.Value
			if strings.HasSuffix(relPath, extraSuffix) {
				relPath = relPath[:-len(extraSuffix)]
			}
			relPath = strings.Replace(relPath, ":", "/", 1)
			relPath = strings.Replace(relPath, "//", "", 1)
			relPathList = append(relPathList, strings.Join([]string{relPath, info.proto, info.importpath, info.name}, ","))
		}
	}

	return relPathList, nil
}

//
// Process -go_googleapis case
//
func generateFromPath(w io.Writer, rootPath string) error {
	return filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !strings.HasSuffix(path, ".proto") {
			return nil
		}
		relPath, err := filepath.Rel(rootPath, path)
		if err != nil || strings.HasPrefix(relPath, "..") {
			log.Panicf("file %q not in repository rootPath %q", path, rootPath)
		}
		relPath = filepath.ToSlash(relPath)

		if strings.HasPrefix(relPath, "google/api/experimental/") {
			// Special case: these protos need to be built together with protos in
			// google/api. They have the same 'option go_package'. The proto_library
			// rule requires them to be in the same Bazel package, so we don't
			// create a build file in experimental.
			packagePath := "google.golang.org/genproto/googleapis/api"
			protoLabel, goLabel := protoLabels("google/api/x", "api")
			fmt.Fprintf(w, "%s,%s,%s,%s\n", relPath, protoLabel, packagePath, goLabel)
			return nil
		}

		packagePath, packageName, err := loadGoPackage(path)
		if err != nil {
			log.Print(err)
			return nil
		}

		protoLabel, goLabel := protoLabels(relPath, packageName)

		fmt.Fprintf(w, "%s,%s,%s,%s\n", relPath, protoLabel, packagePath, goLabel)
		return nil
	})
}

var goPackageRx = regexp.MustCompile(`option go_package = "([^"]*)"`)

func loadGoPackage(protoPath string) (packagePath, packageName string, err error) {
	data, err := ioutil.ReadFile(protoPath)
	if err != nil {
		return "", "", err
	}
	m := goPackageRx.FindSubmatch(data)
	if m == nil {
		return "", "", fmt.Errorf("%s: does not contain 'option go_package'", protoPath)
	}
	opt := string(m[1])
	if i := strings.LastIndexByte(opt, ';'); i >= 0 {
		return opt[:i], opt[i+1:], nil
	}
	if i := strings.LastIndexByte(opt, '/'); i >= 0 {
		return opt, opt[i+1:], nil
	}
	return opt, opt, nil
}

func protoLabels(relPath, packageName string) (protoLabel, goLabel string) {
	dir := path.Dir(relPath)
	protoLabel = fmt.Sprintf("@go_googleapis//%s:%s_proto", dir, packageName)
	goLabel = fmt.Sprintf("@go_googleapis//%s:%s_go_proto", dir, packageName)
	return protoLabel, goLabel
}
<|endoftext|>


<commit_before>package option

type Option struct {
	All              bool     `short:"a" long:"all" description:"Search all packages"`
	Directory        string   `short:"d" long:"directory" description:"Specify root directory"`
	Update           bool     `short:"u" long:"update" description:"With update database"`
	Port             int      `short:"p" long:"port" description:"Use PORT (Default: 9292)"`
	ExpandPath       bool     `long:"expand-path" description:"Display absolute path"`
	NoColor          bool     `long:"nocolor" description:"Don't print color codes in results (Disabled by default)"`
	NoGroup          bool     `long:"nogroup" description:"Don't print file name at header (Disabled by default)"`
	FilesWithMatches bool     `short:"l" long:"files-with-matches" description:"Only print filenames that contain matches"`
	VcsIgnore        []string `long:"vcs-ignore" description:"VCS ignore files (Default: .gitignore, .hgignore)"`
	Ignore           []string `long:"ignore" description:"Ignore files/directories matching pattern"`
	IgnoreCase       bool     `short:"i" long:"ignore-case" description:"Match case insensitively"`
	SmartCase        bool     `short:"S" long:"smart-case" description:"Match case insensitively unless PATTERN contains uppercase characters"`
	FilesWithRegexp  string   `short:"g" description:"Print filenames matching PATTERN"`
	FileSearchRegexp string   `short:"G" long:"file-search-regexp" description:"PATTERN Limit search to filenames matching PATTERN"`
	Depth            int      `long:"depth" default:"25" default-mask:"-" description:"Search up to NUM derectories deep (Default: 25)"`
	Follow           bool     `short:"f" long:"follow" description:"Follow symlinks"`
	After            int      `short:"A" long:"after" description:"Print lines after match"`
	Before           int      `short:"B" long:"before" description:"Print lines before match"`
	Context          int      `short:"C" long:"context" description:"Print lines before and after match"`
	OutputEncode     []string `short:"o" long:"output-encode" description:"Specify output encoding (none, jis, sjis, euc)"`
	Proc             int      // Number of goroutine. Not user option.
	Version          bool     `long:"version" description:"Show version"`
}

func (self *Option) VcsIgnores() []string {
	if len(self.VcsIgnore) == 0 {
		self.VcsIgnore = []string{".gitignore", ".hgignore"}
	}
	return self.VcsIgnore
}
<commit_msg>Delete the -p option<commit_after>package option

type Option struct {
	All              bool     `short:"a" long:"all" description:"Search all packages"`
	Directory        string   `short:"d" long:"directory" description:"Specify root directory"`
	Update           bool     `short:"u" long:"update" description:"With update database"`
	Port             int      `long:"port" description:"Use PORT (Default: 9292)"`
	ExpandPath       bool     `long:"expand-path" description:"Display absolute path"`
	NoColor          bool     `long:"nocolor" description:"Don't print color codes in results (Disabled by default)"`
	NoGroup          bool     `long:"nogroup" description:"Don't print file name at header (Disabled by default)"`
	FilesWithMatches bool     `short:"l" long:"files-with-matches" description:"Only print filenames that contain matches"`
	VcsIgnore        []string `long:"vcs-ignore" description:"VCS ignore files (Default: .gitignore, .hgignore)"`
	Ignore           []string `long:"ignore" description:"Ignore files/directories matching pattern"`
	IgnoreCase       bool     `short:"i" long:"ignore-case" description:"Match case insensitively"`
	SmartCase        bool     `short:"S" long:"smart-case" description:"Match case insensitively unless PATTERN contains uppercase characters"`
	FilesWithRegexp  string   `short:"g" description:"Print filenames matching PATTERN"`
	FileSearchRegexp string   `short:"G" long:"file-search-regexp" description:"PATTERN Limit search to filenames matching PATTERN"`
	Depth            int      `long:"depth" default:"25" default-mask:"-" description:"Search up to NUM derectories deep (Default: 25)"`
	Follow           bool     `short:"f" long:"follow" description:"Follow symlinks"`
	After            int      `short:"A" long:"after" description:"Print lines after match"`
	Before           int      `short:"B" long:"before" description:"Print lines before match"`
	Context          int      `short:"C" long:"context" description:"Print lines before and after match"`
	OutputEncode     []string `short:"o" long:"output-encode" description:"Specify output encoding (none, jis, sjis, euc)"`
	Proc             int      // Number of goroutine. Not user option.
	Version          bool     `long:"version" description:"Show version"`
}

func (self *Option) VcsIgnores() []string {
	if len(self.VcsIgnore) == 0 {
		self.VcsIgnore = []string{".gitignore", ".hgignore"}
	}
	return self.VcsIgnore
}
<|endoftext|>


<commit_before>package framework

import (
	"reflect"
	"sync/atomic"
	"testing"
	"time"

	"github.com/hashicorp/vault/logical"
)

func BenchmarkBackendRoute(b *testing.B) {
	patterns := []string{
		"foo",
		"bar/(?P<name>.+?)",
		"baz/(?P<name>what)",
		`aws/policy/(?P<policy>\w)`,
		`aws/(?P<policy>\w)`,
	}

	backend := &Backend{Paths: make([]*Path, 0, len(patterns))}
	for _, p := range patterns {
		backend.Paths = append(backend.Paths, &Path{Pattern: p})
	}

	// Warm any caches
	backend.Route("aws/policy/foo")

	// Reset the timer since we did a lot above
	b.ResetTimer()

	// Run through and route. We do a sanity check of the return value
	for i := 0; i < b.N; i++ {
		if p := backend.Route("aws/policy/foo"); p == nil {
			b.Fatal("p should not be nil")
		}
	}
}

func TestBackend_impl(t *testing.T) {
	var _ logical.Backend = new(Backend)
}

func TestBackendHandleRequest(t *testing.T) {
	callback := func(req *logical.Request, data *FieldData) (*logical.Response, error) {
		return &logical.Response{
			Data: map[string]interface{}{
				"value": data.Get("value"),
			},
		}, nil
	}

	b := &Backend{
		Paths: []*Path{
			&Path{
				Pattern: "foo/bar",
				Fields: map[string]*FieldSchema{
					"value": &FieldSchema{Type: TypeInt},
				},
				Callbacks: map[logical.Operation]OperationFunc{
					logical.ReadOperation: callback,
				},
			},
		},
	}

	resp, err := b.HandleRequest(&logical.Request{
		Operation: logical.ReadOperation,
		Path:      "foo/bar",
		Data:      map[string]interface{}{"value": "42"},
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	if resp.Data["value"] != 42 {
		t.Fatalf("bad: %#v", resp)
	}
}

func TestBackendHandleRequest_404(t *testing.T) {
	callback := func(req *logical.Request, data *FieldData) (*logical.Response, error) {
		return &logical.Response{
			Data: map[string]interface{}{
				"value": data.Get("value"),
			},
		}, nil
	}

	b := &Backend{
		Paths: []*Path{
			&Path{
				Pattern: `foo/bar`,
				Fields: map[string]*FieldSchema{
					"value": &FieldSchema{Type: TypeInt},
				},
				Callbacks: map[logical.Operation]OperationFunc{
					logical.ReadOperation: callback,
				},
			},
		},
	}

	_, err := b.HandleRequest(&logical.Request{
		Operation: logical.ReadOperation,
		Path:      "foo/baz",
		Data:      map[string]interface{}{"value": "84"},
	})
	if err != logical.ErrUnsupportedPath {
		t.Fatalf("err: %s", err)
	}
}

func TestBackendHandleRequest_help(t *testing.T) {
	b := &Backend{
		Paths: []*Path{
			&Path{
				Pattern: "foo/bar",
				Fields: map[string]*FieldSchema{
					"value": &FieldSchema{Type: TypeInt},
				},
				HelpSynopsis:    "foo",
				HelpDescription: "bar",
			},
		},
	}

	resp, err := b.HandleRequest(&logical.Request{
		Operation: logical.HelpOperation,
		Path:      "foo/bar",
		Data:      map[string]interface{}{"value": "42"},
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	if resp.Data["help"] == nil {
		t.Fatalf("bad: %#v", resp)
	}
}

func TestBackendHandleRequest_rollback(t *testing.T) {
	var called uint32
	callback := func(kind string, data interface{}) bool {
		if data == "foo" {
			atomic.AddUint32(&called, 1)
		}

		return true
	}

	b := &Backend{
		Rollback:       callback,
		RollbackMinAge: 1 * time.Millisecond,
	}

	storage := new(logical.InmemStorage)
	if _, err := PutWAL(storage, "kind", "foo"); err != nil {
		t.Fatalf("err: %s", err)
	}

	time.Sleep(10 * time.Millisecond)

	_, err := b.HandleRequest(&logical.Request{
		Operation: logical.RollbackOperation,
		Path:      "",
		Storage:   storage,
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	if v := atomic.LoadUint32(&called); v != 1 {
		t.Fatalf("bad: %#v", v)
	}
}

func TestBackendHandleRequest_unsupportedOperation(t *testing.T) {
	callback := func(req *logical.Request, data *FieldData) (*logical.Response, error) {
		return &logical.Response{
			Data: map[string]interface{}{
				"value": data.Get("value"),
			},
		}, nil
	}

	b := &Backend{
		Paths: []*Path{
			&Path{
				Pattern: `foo/bar`,
				Fields: map[string]*FieldSchema{
					"value": &FieldSchema{Type: TypeInt},
				},
				Callbacks: map[logical.Operation]OperationFunc{
					logical.ReadOperation: callback,
				},
			},
		},
	}

	_, err := b.HandleRequest(&logical.Request{
		Operation: logical.WriteOperation,
		Path:      "foo/bar",
		Data:      map[string]interface{}{"value": "84"},
	})
	if err != logical.ErrUnsupportedOperation {
		t.Fatalf("err: %s", err)
	}
}

func TestBackendHandleRequest_urlPriority(t *testing.T) {
	callback := func(req *logical.Request, data *FieldData) (*logical.Response, error) {
		return &logical.Response{
			Data: map[string]interface{}{
				"value": data.Get("value"),
			},
		}, nil
	}

	b := &Backend{
		Paths: []*Path{
			&Path{
				Pattern: `foo/(?P<value>\d+)`,
				Fields: map[string]*FieldSchema{
					"value": &FieldSchema{Type: TypeInt},
				},
				Callbacks: map[logical.Operation]OperationFunc{
					logical.ReadOperation: callback,
				},
			},
		},
	}

	resp, err := b.HandleRequest(&logical.Request{
		Operation: logical.ReadOperation,
		Path:      "foo/42",
		Data:      map[string]interface{}{"value": "84"},
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	if resp.Data["value"] != 42 {
		t.Fatalf("bad: %#v", resp)
	}
}

func TestBackendRoute(t *testing.T) {
	cases := map[string]struct {
		Patterns []string
		Path     string
		Match    string
	}{
		"no match": {
			[]string{"foo"},
			"bar",
			"",
		},

		"exact": {
			[]string{"foo"},
			"foo",
			"foo",
		},

		"regexp": {
			[]string{"fo+"},
			"foo",
			"fo+",
		},
	}

	for n, tc := range cases {
		paths := make([]*Path, len(tc.Patterns))
		for i, pattern := range tc.Patterns {
			paths[i] = &Path{Pattern: pattern}
		}

		b := &Backend{Paths: paths}
		result := b.Route(tc.Path)
		match := ""
		if result != nil {
			match = result.Pattern
		}

		if match != tc.Match {
			t.Fatalf("bad: %s\n\nExpected: %s\nGot: %s",
				n, tc.Match, match)
		}
	}
}

func TestFieldSchemaDefaultOrZero(t *testing.T) {
	cases := map[string]struct {
		Schema *FieldSchema
		Value  interface{}
	}{
		"default set": {
			&FieldSchema{Type: TypeString, Default: "foo"},
			"foo",
		},

		"default not set": {
			&FieldSchema{Type: TypeString},
			"",
		},
	}

	for name, tc := range cases {
		actual := tc.Schema.DefaultOrZero()
		if !reflect.DeepEqual(actual, tc.Value) {
			t.Fatalf("bad: %s\n\nExpected: %#v\nGot: %#v",
				name, tc.Value, actual)
		}
	}
}
<commit_msg>logical/framework: test for minimum age<commit_after>package framework

import (
	"reflect"
	"sync/atomic"
	"testing"
	"time"

	"github.com/hashicorp/vault/logical"
)

func BenchmarkBackendRoute(b *testing.B) {
	patterns := []string{
		"foo",
		"bar/(?P<name>.+?)",
		"baz/(?P<name>what)",
		`aws/policy/(?P<policy>\w)`,
		`aws/(?P<policy>\w)`,
	}

	backend := &Backend{Paths: make([]*Path, 0, len(patterns))}
	for _, p := range patterns {
		backend.Paths = append(backend.Paths, &Path{Pattern: p})
	}

	// Warm any caches
	backend.Route("aws/policy/foo")

	// Reset the timer since we did a lot above
	b.ResetTimer()

	// Run through and route. We do a sanity check of the return value
	for i := 0; i < b.N; i++ {
		if p := backend.Route("aws/policy/foo"); p == nil {
			b.Fatal("p should not be nil")
		}
	}
}

func TestBackend_impl(t *testing.T) {
	var _ logical.Backend = new(Backend)
}

func TestBackendHandleRequest(t *testing.T) {
	callback := func(req *logical.Request, data *FieldData) (*logical.Response, error) {
		return &logical.Response{
			Data: map[string]interface{}{
				"value": data.Get("value"),
			},
		}, nil
	}

	b := &Backend{
		Paths: []*Path{
			&Path{
				Pattern: "foo/bar",
				Fields: map[string]*FieldSchema{
					"value": &FieldSchema{Type: TypeInt},
				},
				Callbacks: map[logical.Operation]OperationFunc{
					logical.ReadOperation: callback,
				},
			},
		},
	}

	resp, err := b.HandleRequest(&logical.Request{
		Operation: logical.ReadOperation,
		Path:      "foo/bar",
		Data:      map[string]interface{}{"value": "42"},
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	if resp.Data["value"] != 42 {
		t.Fatalf("bad: %#v", resp)
	}
}

func TestBackendHandleRequest_404(t *testing.T) {
	callback := func(req *logical.Request, data *FieldData) (*logical.Response, error) {
		return &logical.Response{
			Data: map[string]interface{}{
				"value": data.Get("value"),
			},
		}, nil
	}

	b := &Backend{
		Paths: []*Path{
			&Path{
				Pattern: `foo/bar`,
				Fields: map[string]*FieldSchema{
					"value": &FieldSchema{Type: TypeInt},
				},
				Callbacks: map[logical.Operation]OperationFunc{
					logical.ReadOperation: callback,
				},
			},
		},
	}

	_, err := b.HandleRequest(&logical.Request{
		Operation: logical.ReadOperation,
		Path:      "foo/baz",
		Data:      map[string]interface{}{"value": "84"},
	})
	if err != logical.ErrUnsupportedPath {
		t.Fatalf("err: %s", err)
	}
}

func TestBackendHandleRequest_help(t *testing.T) {
	b := &Backend{
		Paths: []*Path{
			&Path{
				Pattern: "foo/bar",
				Fields: map[string]*FieldSchema{
					"value": &FieldSchema{Type: TypeInt},
				},
				HelpSynopsis:    "foo",
				HelpDescription: "bar",
			},
		},
	}

	resp, err := b.HandleRequest(&logical.Request{
		Operation: logical.HelpOperation,
		Path:      "foo/bar",
		Data:      map[string]interface{}{"value": "42"},
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	if resp.Data["help"] == nil {
		t.Fatalf("bad: %#v", resp)
	}
}

func TestBackendHandleRequest_rollback(t *testing.T) {
	var called uint32
	callback := func(kind string, data interface{}) bool {
		if data == "foo" {
			atomic.AddUint32(&called, 1)
		}

		return true
	}

	b := &Backend{
		Rollback:       callback,
		RollbackMinAge: 1 * time.Millisecond,
	}

	storage := new(logical.InmemStorage)
	if _, err := PutWAL(storage, "kind", "foo"); err != nil {
		t.Fatalf("err: %s", err)
	}

	time.Sleep(10 * time.Millisecond)

	_, err := b.HandleRequest(&logical.Request{
		Operation: logical.RollbackOperation,
		Path:      "",
		Storage:   storage,
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	if v := atomic.LoadUint32(&called); v != 1 {
		t.Fatalf("bad: %#v", v)
	}
}

func TestBackendHandleRequest_rollbackMinAge(t *testing.T) {
	var called uint32
	callback := func(kind string, data interface{}) bool {
		if data == "foo" {
			atomic.AddUint32(&called, 1)
		}

		return true
	}

	b := &Backend{
		Rollback:       callback,
		RollbackMinAge: 5 * time.Second,
	}

	storage := new(logical.InmemStorage)
	if _, err := PutWAL(storage, "kind", "foo"); err != nil {
		t.Fatalf("err: %s", err)
	}

	_, err := b.HandleRequest(&logical.Request{
		Operation: logical.RollbackOperation,
		Path:      "",
		Storage:   storage,
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	if v := atomic.LoadUint32(&called); v != 0 {
		t.Fatalf("bad: %#v", v)
	}
}

func TestBackendHandleRequest_unsupportedOperation(t *testing.T) {
	callback := func(req *logical.Request, data *FieldData) (*logical.Response, error) {
		return &logical.Response{
			Data: map[string]interface{}{
				"value": data.Get("value"),
			},
		}, nil
	}

	b := &Backend{
		Paths: []*Path{
			&Path{
				Pattern: `foo/bar`,
				Fields: map[string]*FieldSchema{
					"value": &FieldSchema{Type: TypeInt},
				},
				Callbacks: map[logical.Operation]OperationFunc{
					logical.ReadOperation: callback,
				},
			},
		},
	}

	_, err := b.HandleRequest(&logical.Request{
		Operation: logical.WriteOperation,
		Path:      "foo/bar",
		Data:      map[string]interface{}{"value": "84"},
	})
	if err != logical.ErrUnsupportedOperation {
		t.Fatalf("err: %s", err)
	}
}

func TestBackendHandleRequest_urlPriority(t *testing.T) {
	callback := func(req *logical.Request, data *FieldData) (*logical.Response, error) {
		return &logical.Response{
			Data: map[string]interface{}{
				"value": data.Get("value"),
			},
		}, nil
	}

	b := &Backend{
		Paths: []*Path{
			&Path{
				Pattern: `foo/(?P<value>\d+)`,
				Fields: map[string]*FieldSchema{
					"value": &FieldSchema{Type: TypeInt},
				},
				Callbacks: map[logical.Operation]OperationFunc{
					logical.ReadOperation: callback,
				},
			},
		},
	}

	resp, err := b.HandleRequest(&logical.Request{
		Operation: logical.ReadOperation,
		Path:      "foo/42",
		Data:      map[string]interface{}{"value": "84"},
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	if resp.Data["value"] != 42 {
		t.Fatalf("bad: %#v", resp)
	}
}

func TestBackendRoute(t *testing.T) {
	cases := map[string]struct {
		Patterns []string
		Path     string
		Match    string
	}{
		"no match": {
			[]string{"foo"},
			"bar",
			"",
		},

		"exact": {
			[]string{"foo"},
			"foo",
			"foo",
		},

		"regexp": {
			[]string{"fo+"},
			"foo",
			"fo+",
		},
	}

	for n, tc := range cases {
		paths := make([]*Path, len(tc.Patterns))
		for i, pattern := range tc.Patterns {
			paths[i] = &Path{Pattern: pattern}
		}

		b := &Backend{Paths: paths}
		result := b.Route(tc.Path)
		match := ""
		if result != nil {
			match = result.Pattern
		}

		if match != tc.Match {
			t.Fatalf("bad: %s\n\nExpected: %s\nGot: %s",
				n, tc.Match, match)
		}
	}
}

func TestFieldSchemaDefaultOrZero(t *testing.T) {
	cases := map[string]struct {
		Schema *FieldSchema
		Value  interface{}
	}{
		"default set": {
			&FieldSchema{Type: TypeString, Default: "foo"},
			"foo",
		},

		"default not set": {
			&FieldSchema{Type: TypeString},
			"",
		},
	}

	for name, tc := range cases {
		actual := tc.Schema.DefaultOrZero()
		if !reflect.DeepEqual(actual, tc.Value) {
			t.Fatalf("bad: %s\n\nExpected: %#v\nGot: %#v",
				name, tc.Value, actual)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Albert Nigmatzianov. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package applescript

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
)

const (
	script = `
on run argv
	set commandType to first item of argv as string
	if (commandType is equal to "add_track_to_playlist") then
		add_track_to_playlist(second item of argv, third item of argv)
	end if
	if (commandType is equal to "list_of_playlists") then
		list_of_playlists()
	end if
end run

on add_track_to_playlist(trackPath, playlistName)
	tell application "iTunes"
		add (trackPath as POSIX file) to playlist playlistName
	end tell
end add_track_to_playlist

on list_of_playlists()
	tell application "iTunes"
		get name of playlists
	end tell
end list_of_playlists
`
)

var scriptFile *os.File

func AddTrackToPlaylist(trackPath, playlistName string) error {
	_, err := executeOSAScript("add_track_to_playlist", trackPath, playlistName)
	return err
}

func ListOfPlaylists() (string, error) {
	return executeOSAScript("list_of_playlists")
}

// executeOSAScript executes AppleScript script with args and returns output and error.
func executeOSAScript(args ...string) (string, error) {
	if scriptFile == nil {
		var err error
		scriptFile, err = ioutil.TempFile("", "")
		if err != nil {
			return "", fmt.Errorf("couldn't create osascript file: %v", err)
		}
		if _, err = scriptFile.Write([]byte(script)); err != nil {
			return "", fmt.Errorf("couldn't write script to file: %v", err)
		}
	}

	args = append([]string{scriptFile.Name()}, args...)
	jww.INFO.Println("Executing osascript with args :", args)
	out, err := exec.Command("osascript", args...).Output()
	return string(out), err
}
<commit_msg>applescript: Import jww<commit_after>// Copyright 2016 Albert Nigmatzianov. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package applescript

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"

	jww "github.com/spf13/jWalterWeatherman"
)

const (
	script = `
on run argv
	set commandType to first item of argv as string
	if (commandType is equal to "add_track_to_playlist") then
		add_track_to_playlist(second item of argv, third item of argv)
	end if
	if (commandType is equal to "list_of_playlists") then
		list_of_playlists()
	end if
end run

on add_track_to_playlist(trackPath, playlistName)
	tell application "iTunes"
		add (trackPath as POSIX file) to playlist playlistName
	end tell
end add_track_to_playlist

on list_of_playlists()
	tell application "iTunes"
		get name of playlists
	end tell
end list_of_playlists
`
)

var scriptFile *os.File

func AddTrackToPlaylist(trackPath, playlistName string) error {
	_, err := executeOSAScript("add_track_to_playlist", trackPath, playlistName)
	return err
}

func ListOfPlaylists() (string, error) {
	return executeOSAScript("list_of_playlists")
}

// executeOSAScript executes AppleScript script with args and returns output and error.
func executeOSAScript(args ...string) (string, error) {
	if scriptFile == nil {
		var err error
		scriptFile, err = ioutil.TempFile("", "")
		if err != nil {
			return "", fmt.Errorf("couldn't create osascript file: %v", err)
		}
		if _, err = scriptFile.Write([]byte(script)); err != nil {
			return "", fmt.Errorf("couldn't write script to file: %v", err)
		}
	}

	args = append([]string{scriptFile.Name()}, args...)
	jww.INFO.Println("Executing osascript with args :", args)
	out, err := exec.Command("osascript", args...).Output()
	return string(out), err
}
<|endoftext|>


<commit_before>// Package application allows the creation of Application struct.
// There's only one Application per main().
package application

import (
	"fmt"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/carbocation/interpose"
	"github.com/gorilla/sessions"
	_ "github.com/lib/pq"
	_ "github.com/mattes/migrate/driver/postgres"
	"github.com/mattes/migrate/migrate"
	gocache "github.com/patrickmn/go-cache"
	"github.com/rcrowley/go-metrics"
	"gopkg.in/tylerb/graceful.v1"

	"github.com/resourced/resourced-master/config"
	"github.com/resourced/resourced-master/mailer"
	"github.com/resourced/resourced-master/messagebus"
	"github.com/resourced/resourced-master/middlewares"
)

// New is the constructor for Application struct.
func New(configDir string) (*Application, error) {
	generalConfig, err := config.NewGeneralConfig(configDir)
	if err != nil {
		return nil, err
	}

	dbConfig, err := config.NewDBConfig(generalConfig)
	if err != nil {
		return nil, err
	}

	hostname, err := os.Hostname()
	if err != nil {
		return nil, err
	}

	app := &Application{}
	app.Hostname = hostname
	app.GeneralConfig = generalConfig
	app.DBConfig = dbConfig
	app.cookieStore = sessions.NewCookieStore([]byte(app.GeneralConfig.CookieSecret))
	app.Mailers = make(map[string]*mailer.Mailer)
	app.HandlerInstruments = app.NewHandlerInstruments()
	app.LatencyGauges = make(map[string]metrics.Gauge)
	app.MetricsRegistry = app.NewMetricsRegistry(app.HandlerInstruments, app.LatencyGauges)
	app.Peers = gocache.New(1*time.Minute, 10*time.Minute)
	app.RefetchChecksChan = make(chan bool)

	if app.GeneralConfig.Email != nil {
		mailer, err := mailer.New(app.GeneralConfig.Email)
		if err != nil {
			return nil, err
		}
		app.Mailers["GeneralConfig"] = mailer
	}

	if app.GeneralConfig.Checks.Email != nil {
		mailer, err := mailer.New(app.GeneralConfig.Checks.Email)
		if err != nil {
			return nil, err
		}
		app.Mailers["GeneralConfig.Checks"] = mailer
	}

	// Create MessageBus
	bus, err := app.NewMessageBus(app.GeneralConfig)
	if err != nil {
		return nil, err
	}
	app.MessageBus = bus

	return app, err
}

// Application is the application object that runs HTTP server.
type Application struct {
	Hostname           string
	GeneralConfig      config.GeneralConfig
	DBConfig           *config.DBConfig
	cookieStore        *sessions.CookieStore
	Mailers            map[string]*mailer.Mailer
	HandlerInstruments map[string]chan int64
	LatencyGauges      map[string]metrics.Gauge
	MetricsRegistry    metrics.Registry
	MessageBus         *messagebus.MessageBus
	Peers              *gocache.Cache
	RefetchChecksChan  chan bool
}

func (app *Application) FullAddr() string {
	addr := app.GeneralConfig.Addr
	if strings.HasPrefix(addr, ":") {
		addr = app.Hostname + addr
	}
	if strings.HasPrefix(addr, "localhost") {
		addr = strings.Replace(addr, "localhost", app.Hostname, 1)
	}
	if strings.HasPrefix(addr, "127.0.0.1") {
		addr = strings.Replace(addr, "127.0.0.1", app.Hostname, 1)
	}
	if strings.HasPrefix(addr, "0.0.0.0") {
		addr = strings.Replace(addr, "0.0.0.0", app.Hostname, 1)
	}

	return addr
}

// MiddlewareStruct configures all the middlewares that are in-use for all request handlers.
func (app *Application) MiddlewareStruct() (*interpose.Middleware, error) {
	middle := interpose.New()
	middle.Use(middlewares.SetAddr(app.GeneralConfig.Addr))
	middle.Use(middlewares.SetVIPAddr(app.GeneralConfig.VIPAddr))
	middle.Use(middlewares.SetVIPProtocol(app.GeneralConfig.VIPProtocol))
	middle.Use(middlewares.SetDBs(app.DBConfig))
	middle.Use(middlewares.SetCookieStore(app.cookieStore))
	middle.Use(middlewares.SetMailers(app.Mailers))
	middle.Use(middlewares.SetMessageBus(app.MessageBus))

	middle.UseHandler(app.mux())

	return middle, nil
}

// NewHTTPServer returns an instance of HTTP server.
func (app *Application) NewHTTPServer() (*graceful.Server, error) {
	// Create HTTP middlewares
	middle, err := app.MiddlewareStruct()
	if err != nil {
		return nil, err
	}

	requestTimeout, err := time.ParseDuration(app.GeneralConfig.RequestShutdownTimeout)
	if err != nil {
		return nil, err
	}

	// Create HTTP server
	srv := &graceful.Server{
		Timeout: requestTimeout,
		Server:  &http.Server{Addr: app.GeneralConfig.Addr, Handler: middle},
	}

	return srv, nil
}

// MigrateUpAll runs all migration files to be up-to-date.
func (app *Application) MigrateUpAll() error {
	errs, ok := migrate.UpSync(app.GeneralConfig.DSN, "./migrations/core")
	if !ok {
		return fmt.Errorf("DSN: %v, Error: %v", app.GeneralConfig.DSN, errs[0])
	}

	errs, ok = migrate.UpSync(app.GeneralConfig.Hosts.DSN, "./migrations/hosts")
	if !ok {
		return fmt.Errorf("DSN: %v, Error: %v", app.GeneralConfig.Hosts.DSN, errs[0])
	}

	for _, dsn := range app.GeneralConfig.Hosts.DSNByClusterID {
		errs, ok = migrate.UpSync(dsn, "./migrations/hosts")
		if !ok {
			return fmt.Errorf("DSN: %v, Error: %v", dsn, errs[0])
		}
	}

	errs, ok = migrate.UpSync(app.GeneralConfig.Checks.DSN, "./migrations/ts-checks")
	if !ok {
		return fmt.Errorf("DSN: %v, Error: %v", app.GeneralConfig.Checks.DSN, errs[0])
	}

	for _, dsn := range app.GeneralConfig.Checks.DSNByClusterID {
		errs, ok = migrate.UpSync(dsn, "./migrations/ts-checks")
		if !ok {
			return fmt.Errorf("DSN: %v, Error: %v", dsn, errs[0])
		}
	}

	errs, ok = migrate.UpSync(app.GeneralConfig.Events.DSN, "./migrations/ts-events")
	if !ok {
		return fmt.Errorf("DSN: %v, Error: %v", app.GeneralConfig.Events.DSN, errs[0])
	}

	for _, dsn := range app.GeneralConfig.Events.DSNByClusterID {
		errs, ok = migrate.UpSync(dsn, "./migrations/ts-events")
		if !ok {
			return fmt.Errorf("DSN: %v, Error: %v", dsn, errs[0])
		}
	}

	errs, ok = migrate.UpSync(app.GeneralConfig.ExecutorLogs.DSN, "./migrations/ts-executor-logs")
	if !ok {
		return fmt.Errorf("DSN: %v, Error: %v", app.GeneralConfig.ExecutorLogs.DSN, errs[0])
	}

	for _, dsn := range app.GeneralConfig.ExecutorLogs.DSNByClusterID {
		errs, ok = migrate.UpSync(dsn, "./migrations/ts-executor-logs")
		if !ok {
			return fmt.Errorf("DSN: %v, Error: %v", dsn, errs[0])
		}
	}

	errs, ok = migrate.UpSync(app.GeneralConfig.Logs.DSN, "./migrations/ts-logs")
	if !ok {
		return fmt.Errorf("DSN: %v, Error: %v", app.GeneralConfig.Logs.DSN, errs[0])
	}

	for _, dsn := range app.GeneralConfig.Logs.DSNByClusterID {
		errs, ok = migrate.UpSync(dsn, "./migrations/ts-logs")
		if !ok {
			return fmt.Errorf("DSN: %v, Error: %v", dsn, errs[0])
		}
	}

	errs, ok = migrate.UpSync(app.GeneralConfig.Metrics.DSN, "./migrations/ts-metrics")
	if !ok {
		return fmt.Errorf("DSN: %v, Error: %v", app.GeneralConfig.Metrics.DSN, errs[0])
	}

	for _, dsn := range app.GeneralConfig.Metrics.DSNByClusterID {
		errs, ok = migrate.UpSync(dsn, "./migrations/ts-metrics")
		if !ok {
			return fmt.Errorf("DSN: %v, Error: %v", dsn, errs[0])
		}
	}

	errs, ok = migrate.UpSync(app.GeneralConfig.MetricsAggr15m.DSN, "./migrations/ts-metrics")
	if !ok {
		return fmt.Errorf("DSN: %v, Error: %v", app.GeneralConfig.MetricsAggr15m.DSN, errs[0])
	}

	for _, dsn := range app.GeneralConfig.MetricsAggr15m.DSNByClusterID {
		errs, ok = migrate.UpSync(dsn, "./migrations/ts-metrics")
		if !ok {
			return fmt.Errorf("DSN: %v, Error: %v", dsn, errs[0])
		}
	}

	return nil
}
<commit_msg>Create a dedicated stdout/stderr loggers.<commit_after>// Package application allows the creation of Application struct.
// There's only one Application per main().
package application

import (
	"fmt"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/carbocation/interpose"
	"github.com/gorilla/sessions"
	_ "github.com/lib/pq"
	_ "github.com/mattes/migrate/driver/postgres"
	"github.com/mattes/migrate/migrate"
	gocache "github.com/patrickmn/go-cache"
	"github.com/rcrowley/go-metrics"
	"gopkg.in/tylerb/graceful.v1"

	"github.com/resourced/resourced-master/config"
	"github.com/resourced/resourced-master/mailer"
	"github.com/resourced/resourced-master/messagebus"
	"github.com/resourced/resourced-master/middlewares"
)

// New is the constructor for Application struct.
func New(configDir string) (*Application, error) {
	generalConfig, err := config.NewGeneralConfig(configDir)
	if err != nil {
		return nil, err
	}

	dbConfig, err := config.NewDBConfig(generalConfig)
	if err != nil {
		return nil, err
	}

	hostname, err := os.Hostname()
	if err != nil {
		return nil, err
	}

	app := &Application{}
	app.Hostname = hostname
	app.GeneralConfig = generalConfig
	app.DBConfig = dbConfig
	app.cookieStore = sessions.NewCookieStore([]byte(app.GeneralConfig.CookieSecret))
	app.Mailers = make(map[string]*mailer.Mailer)
	app.HandlerInstruments = app.NewHandlerInstruments()
	app.LatencyGauges = make(map[string]metrics.Gauge)
	app.MetricsRegistry = app.NewMetricsRegistry(app.HandlerInstruments, app.LatencyGauges)
	app.Peers = gocache.New(1*time.Minute, 10*time.Minute)
	app.RefetchChecksChan = make(chan bool)

	if app.GeneralConfig.Email != nil {
		mailer, err := mailer.New(app.GeneralConfig.Email)
		if err != nil {
			return nil, err
		}
		app.Mailers["GeneralConfig"] = mailer
	}

	if app.GeneralConfig.Checks.Email != nil {
		mailer, err := mailer.New(app.GeneralConfig.Checks.Email)
		if err != nil {
			return nil, err
		}
		app.Mailers["GeneralConfig.Checks"] = mailer
	}

	// Create MessageBus
	bus, err := app.NewMessageBus(app.GeneralConfig)
	if err != nil {
		return nil, err
	}
	app.MessageBus = bus

	// Setup loggers
	app.StdOutLogger = logrus.New()
	app.StdOutLogger.Out = os.Stdout

	app.StdErrLogger = logrus.New()
	app.StdErrLogger.Out = os.Stderr

	return app, err
}

// Application is the application object that runs HTTP server.
type Application struct {
	Hostname           string
	GeneralConfig      config.GeneralConfig
	DBConfig           *config.DBConfig
	cookieStore        *sessions.CookieStore
	Mailers            map[string]*mailer.Mailer
	HandlerInstruments map[string]chan int64
	LatencyGauges      map[string]metrics.Gauge
	MetricsRegistry    metrics.Registry
	MessageBus         *messagebus.MessageBus
	Peers              *gocache.Cache
	RefetchChecksChan  chan bool
	StdOutLogger       *logrus.Logger
	StdErrLogger       *logrus.Logger
}

func (app *Application) FullAddr() string {
	addr := app.GeneralConfig.Addr
	if strings.HasPrefix(addr, ":") {
		addr = app.Hostname + addr
	}
	if strings.HasPrefix(addr, "localhost") {
		addr = strings.Replace(addr, "localhost", app.Hostname, 1)
	}
	if strings.HasPrefix(addr, "127.0.0.1") {
		addr = strings.Replace(addr, "127.0.0.1", app.Hostname, 1)
	}
	if strings.HasPrefix(addr, "0.0.0.0") {
		addr = strings.Replace(addr, "0.0.0.0", app.Hostname, 1)
	}

	return addr
}

// MiddlewareStruct configures all the middlewares that are in-use for all request handlers.
func (app *Application) MiddlewareStruct() (*interpose.Middleware, error) {
	middle := interpose.New()
	middle.Use(middlewares.SetAddr(app.GeneralConfig.Addr))
	middle.Use(middlewares.SetVIPAddr(app.GeneralConfig.VIPAddr))
	middle.Use(middlewares.SetVIPProtocol(app.GeneralConfig.VIPProtocol))
	middle.Use(middlewares.SetDBs(app.DBConfig))
	middle.Use(middlewares.SetCookieStore(app.cookieStore))
	middle.Use(middlewares.SetMailers(app.Mailers))
	middle.Use(middlewares.SetMessageBus(app.MessageBus))

	middle.UseHandler(app.mux())

	return middle, nil
}

// NewHTTPServer returns an instance of HTTP server.
func (app *Application) NewHTTPServer() (*graceful.Server, error) {
	// Create HTTP middlewares
	middle, err := app.MiddlewareStruct()
	if err != nil {
		return nil, err
	}

	requestTimeout, err := time.ParseDuration(app.GeneralConfig.RequestShutdownTimeout)
	if err != nil {
		return nil, err
	}

	// Create HTTP server
	srv := &graceful.Server{
		Timeout: requestTimeout,
		Server:  &http.Server{Addr: app.GeneralConfig.Addr, Handler: middle},
	}

	return srv, nil
}

// MigrateUpAll runs all migration files to be up-to-date.
func (app *Application) MigrateUpAll() error {
	errs, ok := migrate.UpSync(app.GeneralConfig.DSN, "./migrations/core")
	if !ok {
		return fmt.Errorf("DSN: %v, Error: %v", app.GeneralConfig.DSN, errs[0])
	}

	errs, ok = migrate.UpSync(app.GeneralConfig.Hosts.DSN, "./migrations/hosts")
	if !ok {
		return fmt.Errorf("DSN: %v, Error: %v", app.GeneralConfig.Hosts.DSN, errs[0])
	}

	for _, dsn := range app.GeneralConfig.Hosts.DSNByClusterID {
		errs, ok = migrate.UpSync(dsn, "./migrations/hosts")
		if !ok {
			return fmt.Errorf("DSN: %v, Error: %v", dsn, errs[0])
		}
	}

	errs, ok = migrate.UpSync(app.GeneralConfig.Checks.DSN, "./migrations/ts-checks")
	if !ok {
		return fmt.Errorf("DSN: %v, Error: %v", app.GeneralConfig.Checks.DSN, errs[0])
	}

	for _, dsn := range app.GeneralConfig.Checks.DSNByClusterID {
		errs, ok = migrate.UpSync(dsn, "./migrations/ts-checks")
		if !ok {
			return fmt.Errorf("DSN: %v, Error: %v", dsn, errs[0])
		}
	}

	errs, ok = migrate.UpSync(app.GeneralConfig.Events.DSN, "./migrations/ts-events")
	if !ok {
		return fmt.Errorf("DSN: %v, Error: %v", app.GeneralConfig.Events.DSN, errs[0])
	}

	for _, dsn := range app.GeneralConfig.Events.DSNByClusterID {
		errs, ok = migrate.UpSync(dsn, "./migrations/ts-events")
		if !ok {
			return fmt.Errorf("DSN: %v, Error: %v", dsn, errs[0])
		}
	}

	errs, ok = migrate.UpSync(app.GeneralConfig.ExecutorLogs.DSN, "./migrations/ts-executor-logs")
	if !ok {
		return fmt.Errorf("DSN: %v, Error: %v", app.GeneralConfig.ExecutorLogs.DSN, errs[0])
	}

	for _, dsn := range app.GeneralConfig.ExecutorLogs.DSNByClusterID {
		errs, ok = migrate.UpSync(dsn, "./migrations/ts-executor-logs")
		if !ok {
			return fmt.Errorf("DSN: %v, Error: %v", dsn, errs[0])
		}
	}

	errs, ok = migrate.UpSync(app.GeneralConfig.Logs.DSN, "./migrations/ts-logs")
	if !ok {
		return fmt.Errorf("DSN: %v, Error: %v", app.GeneralConfig.Logs.DSN, errs[0])
	}

	for _, dsn := range app.GeneralConfig.Logs.DSNByClusterID {
		errs, ok = migrate.UpSync(dsn, "./migrations/ts-logs")
		if !ok {
			return fmt.Errorf("DSN: %v, Error: %v", dsn, errs[0])
		}
	}

	errs, ok = migrate.UpSync(app.GeneralConfig.Metrics.DSN, "./migrations/ts-metrics")
	if !ok {
		return fmt.Errorf("DSN: %v, Error: %v", app.GeneralConfig.Metrics.DSN, errs[0])
	}

	for _, dsn := range app.GeneralConfig.Metrics.DSNByClusterID {
		errs, ok = migrate.UpSync(dsn, "./migrations/ts-metrics")
		if !ok {
			return fmt.Errorf("DSN: %v, Error: %v", dsn, errs[0])
		}
	}

	errs, ok = migrate.UpSync(app.GeneralConfig.MetricsAggr15m.DSN, "./migrations/ts-metrics")
	if !ok {
		return fmt.Errorf("DSN: %v, Error: %v", app.GeneralConfig.MetricsAggr15m.DSN, errs[0])
	}

	for _, dsn := range app.GeneralConfig.MetricsAggr15m.DSNByClusterID {
		errs, ok = migrate.UpSync(dsn, "./migrations/ts-metrics")
		if !ok {
			return fmt.Errorf("DSN: %v, Error: %v", dsn, errs[0])
		}
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Code generated by client-gen. DO NOT EDIT.

/*
Package applyconfigurations provides typesafe go representations of the apply
configurations that are used to constructs Server-side Apply requests.

Basics

The Apply functions in the typed client (see the k8s.io/client-go/kubernetes/typed packages) offer
a direct and typesafe way of calling Server-side Apply. Each Apply function takes an "apply
configuration" type as an argument, which is a structured representation of an Apply request. For
example:

    import (
         ...
         v1ac "k8s.io/client-go/applyconfigurations/autoscaling/v1"
    )
    hpaApplyConfig := v1ac.HorizontalPodAutoscaler(autoscalerName, ns).
         WithSpec(v1ac.HorizontalPodAutoscalerSpec().
                  WithMinReplicas(0)
         )
    return hpav1client.Apply(ctx, hpaApplyConfig, metav1.ApplyOptions{FieldManager: "mycontroller", Force: true})

Note in this example that HorizontalPodAutoscaler is imported from an "applyconfigurations"
package. Each "apply configuration" type represents the same Kubernetes object kind as the
corresponding go struct, but where all fields are pointers to make them optional, allowing apply
requests to be accurately represented. For example, this when the apply configuration in the above
example is marshalled to YAML, it produces:

    apiVersion: autoscaling/v1
    kind: HorizontalPodAutoscaler
    metadata:
        name: myHPA
        namespace: myNamespace
    spec:
        minReplicas: 0

To understand why this is needed, the above YAML cannot be produced by the
v1.HorizontalPodAutoscaler go struct. Take for example:

    hpa := v1.HorizontalPodAutoscaler{
         TypeMeta: metav1.TypeMeta{
                  APIVersion: "autoscaling/v1",
                  Kind:       "HorizontalPodAutoscaler",
         },
         ObjectMeta: ObjectMeta{
                  Namespace: ns,
                  Name:      autoscalerName,
         },
         Spec: v1.HorizontalPodAutoscalerSpec{
                  MinReplicas: pointer.Int32Ptr(0),
         },
    }

The above code attempts to declare the same apply configuration as shown in the previous examples,
but when marshalled to YAML, produces:

    kind: HorizontalPodAutoscaler
    apiVersion: autoscaling/v1
    metadata:
      name: myHPA
      namespace: myNamespace
      creationTimestamp: null
    spec:
      scaleTargetRef:
        kind: ""
        name: ""
      minReplicas: 0
      maxReplicas: 0

Which, among other things, contains spec.maxReplicas set to 0. This is almost certainly not what
the caller intended (the intended apply configuration says nothing about the maxReplicas field),
and could have serious consequences on a production system: it directs the autoscaler to downscale
to zero pods. The problem here originates from the fact that the go structs contain required fields
that are zero valued if not set explicitly. The go structs work as intended for create and update
operations, but are fundamentally incompatible with apply, which is why we have introduced the
generated "apply configuration" types.

The "apply configurations" also have convenience With<FieldName> functions that make it easier to
build apply requests. This allows developers to set fields without having to deal with the fact that
all the fields in the "apply configuration" types are pointers, and are inconvenient to set using
go. For example "MinReplicas: &0" is not legal go code, so without the With functions, developers
would work around this problem by using a library, .e.g. "MinReplicas: pointer.Int32Ptr(0)", but
string enumerations like corev1.Protocol are still a problem since they cannot be supported by a
general purpose library. In addition to the convenience, the With functions also isolate
developers from the underlying representation, which makes it safer for the underlying
representation to be changed to support additional features in the future.

Controller Support

The new client-go support makes it much easier to use Server-side Apply in controllers, by either of two mechanisms.

When authoring new controllers to use Server-side Apply, a good approach is to have the controller
recreate the apply configuration for an object each time it reconciles that object.  This ensures
that the controller fully reconciles all the fields that it is responsible for. Controllers
typically should unconditionally set all the fields they own by setting "Force: true" in the
ApplyOptions. Controllers must also provide a FieldManager name that is unique to the
reconciliation loop that apply is called from.

When upgrading existing controllers to use Server-side Apply the same approach often works
well--migrate the controllers to recreate the apply configuration each time it reconciles any
object. Unfortunately, the controller might have multiple code paths that update different parts of
an object depending on various conditions. Migrating a controller like this to Server-side Apply can
be risky because if the controller forgets to include any fields in an apply configuration that is
included in a previous apply request, a field can be accidentlly deleted. To ease this type of
migration, client-go apply support provides a way to replace any controller reconciliation code that
performs a "read/modify-in-place/update" (or patch) workflow with a "extract/modify-in-place/apply"
workflow. Here's an example of the new workflow:

    fieldMgr := "my-field-manager"
    deploymentClient := clientset.AppsV1().Deployments("default")
    // read, could also be read from a shared informer
    deployment, err := deploymentClient.Get(ctx, "example-deployment", metav1.GetOptions{})
    if err != nil {
      // handle error
    }
    // extract
    deploymentApplyConfig, err := appsv1ac.ExtractDeployment(deployment, fieldMgr)
    if err != nil {
      // handle error
    }
    // modify-in-place
    deploymentApplyConfig.Spec.Template.Spec.WithContainers(corev1ac.Container().
	WithName("modify-slice").
	WithImage("nginx:1.14.2"),
    )
    // apply
    applied, err := deploymentClient.Apply(ctx, extractedDeployment, metav1.ApplyOptions{FieldManager: fieldMgr})
*/
package applyconfigurations
<commit_msg>Split documentation according to both mechanisms available for migration<commit_after>/*
Copyright The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Code generated by applyconfiguration-gen. DO NOT EDIT.

/*
Package applyconfigurations provides typesafe go representations of the apply
configurations that are used to constructs Server-side Apply requests.

Basics

The Apply functions in the typed client (see the k8s.io/client-go/kubernetes/typed packages) offer
a direct and typesafe way of calling Server-side Apply. Each Apply function takes an "apply
configuration" type as an argument, which is a structured representation of an Apply request. For
example:

    import (
         ...
         v1ac "k8s.io/client-go/applyconfigurations/autoscaling/v1"
    )
    hpaApplyConfig := v1ac.HorizontalPodAutoscaler(autoscalerName, ns).
         WithSpec(v1ac.HorizontalPodAutoscalerSpec().
                  WithMinReplicas(0)
         )
    return hpav1client.Apply(ctx, hpaApplyConfig, metav1.ApplyOptions{FieldManager: "mycontroller", Force: true})

Note in this example that HorizontalPodAutoscaler is imported from an "applyconfigurations"
package. Each "apply configuration" type represents the same Kubernetes object kind as the
corresponding go struct, but where all fields are pointers to make them optional, allowing apply
requests to be accurately represented. For example, this when the apply configuration in the above
example is marshalled to YAML, it produces:

    apiVersion: autoscaling/v1
    kind: HorizontalPodAutoscaler
    metadata:
        name: myHPA
        namespace: myNamespace
    spec:
        minReplicas: 0

To understand why this is needed, the above YAML cannot be produced by the
v1.HorizontalPodAutoscaler go struct. Take for example:

    hpa := v1.HorizontalPodAutoscaler{
         TypeMeta: metav1.TypeMeta{
                  APIVersion: "autoscaling/v1",
                  Kind:       "HorizontalPodAutoscaler",
         },
         ObjectMeta: ObjectMeta{
                  Namespace: ns,
                  Name:      autoscalerName,
         },
         Spec: v1.HorizontalPodAutoscalerSpec{
                  MinReplicas: pointer.Int32Ptr(0),
         },
    }

The above code attempts to declare the same apply configuration as shown in the previous examples,
but when marshalled to YAML, produces:

    kind: HorizontalPodAutoscaler
    apiVersion: autoscaling/v1
    metadata:
      name: myHPA
      namespace: myNamespace
      creationTimestamp: null
    spec:
      scaleTargetRef:
        kind: ""
        name: ""
      minReplicas: 0
      maxReplicas: 0

Which, among other things, contains spec.maxReplicas set to 0. This is almost certainly not what
the caller intended (the intended apply configuration says nothing about the maxReplicas field),
and could have serious consequences on a production system: it directs the autoscaler to downscale
to zero pods. The problem here originates from the fact that the go structs contain required fields
that are zero valued if not set explicitly. The go structs work as intended for create and update
operations, but are fundamentally incompatible with apply, which is why we have introduced the
generated "apply configuration" types.

The "apply configurations" also have convenience With<FieldName> functions that make it easier to
build apply requests. This allows developers to set fields without having to deal with the fact that
all the fields in the "apply configuration" types are pointers, and are inconvenient to set using
go. For example "MinReplicas: &0" is not legal go code, so without the With functions, developers
would work around this problem by using a library, .e.g. "MinReplicas: pointer.Int32Ptr(0)", but
string enumerations like corev1.Protocol are still a problem since they cannot be supported by a
general purpose library. In addition to the convenience, the With functions also isolate
developers from the underlying representation, which makes it safer for the underlying
representation to be changed to support additional features in the future.

Controller Support

The new client-go support makes it much easier to use Server-side Apply in controllers, by either of
two mechanisms.

Mechanism 1:

When authoring new controllers to use Server-side Apply, a good approach is to have the controller
recreate the apply configuration for an object each time it reconciles that object.  This ensures
that the controller fully reconciles all the fields that it is responsible for. Controllers
typically should unconditionally set all the fields they own by setting "Force: true" in the
ApplyOptions. Controllers must also provide a FieldManager name that is unique to the
reconciliation loop that apply is called from.

When upgrading existing controllers to use Server-side Apply the same approach often works
well--migrate the controllers to recreate the apply configuration each time it reconciles any
object. For cases where this does not work well, see Mechanism 2.

Mechanism 2:

When upgrading existing controllers to use Server-side Apply, the controller might have multiple
code paths that update different parts of an object depending on various conditions. Migrating a
controller like this to Server-side Apply can be risky because if the controller forgets to include
any fields in an apply configuration that is included in a previous apply request, a field can be
accidentally deleted. For such cases, an alternative to mechanism 1 is to replace any controller
reconciliation code that performs a "read/modify-in-place/update" (or patch) workflow with a
"extract/modify-in-place/apply" workflow. Here's an example of the new workflow:

    fieldMgr := "my-field-manager"
    deploymentClient := clientset.AppsV1().Deployments("default")
    // read, could also be read from a shared informer
    deployment, err := deploymentClient.Get(ctx, "example-deployment", metav1.GetOptions{})
    if err != nil {
      // handle error
    }
    // extract
    deploymentApplyConfig, err := appsv1ac.ExtractDeployment(deployment, fieldMgr)
    if err != nil {
      // handle error
    }
    // modify-in-place
    deploymentApplyConfig.Spec.Template.Spec.WithContainers(corev1ac.Container().
	WithName("modify-slice").
	WithImage("nginx:1.14.2"),
    )
    // apply
    applied, err := deploymentClient.Apply(ctx, extractedDeployment, metav1.ApplyOptions{FieldManager: fieldMgr})
*/
package applyconfigurations
<|endoftext|>


<commit_before>package main

import (
	"testing"
	"time"
)

func TestParseDuration(t *testing.T) {
	tests := []struct {
		in string
		// out or fail are mutually exclusive
		out  time.Duration
		fail bool
	}{
		{in: "10ms", out: 10 * time.Millisecond},
		{in: "3s", out: 3 * time.Second},
		{in: "60m", out: 60 * time.Minute},
		{in: "12h", out: 12 * time.Hour},
		{in: "7d", out: 168 * time.Hour},

		{in: "", fail: true},
		{in: "0", fail: true},
		{in: "7ns", fail: true},
		{in: "14mo", fail: true},
		{in: "3.5h", fail: true},
		{in: "106752d", fail: true}, // overflow
	}
	for _, tc := range tests {
		got, err := parseDuration(tc.in)
		if !tc.fail && err != nil {
			t.Errorf("parseDuration(%q) unexpectedly failed: %v", tc.in, err)
			continue
		}
		if tc.fail && err == nil {
			t.Errorf("parseDuration(%q) did not fail", tc.in)
			continue
		}
		if tc.fail {
			continue
		}
		if got != tc.out {
			t.Errorf("parseDuration(%q) = %v, want %v", tc.in, got, tc.out)
		}
	}
}
<commit_msg>all: ensure all files have licenses, add license test<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"testing"
	"time"
)

func TestParseDuration(t *testing.T) {
	tests := []struct {
		in string
		// out or fail are mutually exclusive
		out  time.Duration
		fail bool
	}{
		{in: "10ms", out: 10 * time.Millisecond},
		{in: "3s", out: 3 * time.Second},
		{in: "60m", out: 60 * time.Minute},
		{in: "12h", out: 12 * time.Hour},
		{in: "7d", out: 168 * time.Hour},

		{in: "", fail: true},
		{in: "0", fail: true},
		{in: "7ns", fail: true},
		{in: "14mo", fail: true},
		{in: "3.5h", fail: true},
		{in: "106752d", fail: true}, // overflow
	}
	for _, tc := range tests {
		got, err := parseDuration(tc.in)
		if !tc.fail && err != nil {
			t.Errorf("parseDuration(%q) unexpectedly failed: %v", tc.in, err)
			continue
		}
		if tc.fail && err == nil {
			t.Errorf("parseDuration(%q) did not fail", tc.in)
			continue
		}
		if tc.fail {
			continue
		}
		if got != tc.out {
			t.Errorf("parseDuration(%q) = %v, want %v", tc.in, got, tc.out)
		}
	}
}
<|endoftext|>


<commit_before>package server

import (
	"io"
	"log"
	"net/http"

	"github.com/gorilla/mux"
	"github.com/unrolled/render"
	"github.com/wanelo/image-server/core"
	mantajob "github.com/wanelo/image-server/job/manta"
	"github.com/wanelo/image-server/uploader/manta/client"
)

func CreateBatchHandler(w http.ResponseWriter, req *http.Request, sc *core.ServerConfiguration) {
	vars := mux.Vars(req)
	namespace := vars["namespace"]

	r := render.New(render.Options{
		IndentJSON: true,
	})

	job, err := mantajob.CreateJob(sc.Outputs, sc.RemoteBasePath, namespace, req.Body)
	if err != nil {
		errorHandlerJSON(err, w, r, http.StatusInternalServerError)
		return
	}

	json := map[string]string{
		"job_id": job.JobID,
	}

	r.JSON(w, http.StatusOK, json)

	go job.AddInputs()
}

func BatchHandler(w http.ResponseWriter, req *http.Request, sc *core.ServerConfiguration) {
	vars := mux.Vars(req)
	uuid := vars["uuid"]

	mantaClient := client.DefaultClient()
	job, err := mantaClient.GetJob(uuid)

	if err != nil {
		log.Println(err)
		errorHandler(err, w, req, 500)
		return
	}

	if job.State == "done" {
		result, err := getJobOutput(uuid, mantaClient)
		if err != nil {
			log.Println(err)
			errorHandler(err, w, req, 500)
			return
		}

		w.WriteHeader(200)
		io.Copy(w, result)
	} else {
		// if not complete return 202
		w.WriteHeader(202)
	}
}

func getJobOutput(uuid string, mantaClient *client.Client) (io.Reader, error) {
	output, err := mantaClient.GetJobOutput(uuid)
	if err != nil {
		return nil, err
	}

	result, err := mantaClient.GetObject(output)
	if err != nil {
		return nil, err
	}

	return result, nil
}
<commit_msg>Return job status for batch jobs in progress<commit_after>package server

import (
	"io"
	"log"
	"net/http"

	"github.com/gorilla/mux"
	"github.com/unrolled/render"
	"github.com/wanelo/image-server/core"
	mantajob "github.com/wanelo/image-server/job/manta"
	"github.com/wanelo/image-server/uploader/manta/client"
)

func CreateBatchHandler(w http.ResponseWriter, req *http.Request, sc *core.ServerConfiguration) {
	vars := mux.Vars(req)
	namespace := vars["namespace"]

	r := render.New(render.Options{
		IndentJSON: true,
	})

	job, err := mantajob.CreateJob(sc.Outputs, sc.RemoteBasePath, namespace, req.Body)
	if err != nil {
		errorHandlerJSON(err, w, r, http.StatusInternalServerError)
		return
	}

	json := map[string]string{
		"job_id": job.JobID,
	}

	r.JSON(w, http.StatusOK, json)

	go job.AddInputs()
}

func BatchHandler(w http.ResponseWriter, req *http.Request, sc *core.ServerConfiguration) {
	vars := mux.Vars(req)
	uuid := vars["uuid"]

	mantaClient := client.DefaultClient()
	job, err := mantaClient.GetJob(uuid)

	if err != nil {
		log.Println(err)
		errorHandler(err, w, req, 500)
		return
	}

	if job.State == "done" {
		result, err := getJobOutput(uuid, mantaClient)
		if err != nil {
			log.Println(err)
			errorHandler(err, w, req, 500)
			return
		}

		w.WriteHeader(200)
		io.Copy(w, result)
	} else {
		// If not complete, return job details and 202
		r := render.New(render.Options{
			IndentJSON: true,
		})

		r.JSON(w, 202, job)
	}
}

func getJobOutput(uuid string, mantaClient *client.Client) (io.Reader, error) {
	output, err := mantaClient.GetJobOutput(uuid)
	if err != nil {
		return nil, err
	}

	result, err := mantaClient.GetObject(output)
	if err != nil {
		return nil, err
	}

	return result, nil
}
<|endoftext|>


<commit_before>package appstore

import (
	"errors"
	"reflect"
	"testing"
	"time"
)

func TestHandleError(t *testing.T) {
	var expected, actual error

	// status 0
	expected = nil
	actual = HandleError(0)
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("got %v\nwant %v", actual, expected)
	}

	// status 21000
	expected = errors.New("The App Store could not read the JSON object you provided.")
	actual = HandleError(21000)
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("got %v\nwant %v", actual, expected)
	}

	// status 21002
	expected = errors.New("The data in the receipt-data property was malformed or missing.")
	actual = HandleError(21002)
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("got %v\nwant %v", actual, expected)
	}

	// status 21003
	expected = errors.New("The receipt could not be authenticated.")
	actual = HandleError(21003)
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("got %v\nwant %v", actual, expected)
	}

	// status 21005
	expected = errors.New("The receipt server is not currently available.")
	actual = HandleError(21005)
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("got %v\nwant %v", actual, expected)
	}

	// status 21007
	expected = errors.New("This receipt is from the test environment, but it was sent to the production environment for verification. Send it to the test environment instead.")
	actual = HandleError(21007)
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("got %v\nwant %v", actual, expected)
	}

	// status 21008
	expected = errors.New("This receipt is from the production environment, but it was sent to the test environment for verification. Send it to the production environment instead.")
	actual = HandleError(21008)
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("got %v\nwant %v", actual, expected)
	}

	// status unkown
	expected = errors.New("An unknown error ocurred")
	actual = HandleError(100)
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("got %v\nwant %v", actual, expected)
	}
}

func TestNew(t *testing.T) {
	expected := Client{
		URL:     "https://sandbox.itunes.apple.com/verifyReceipt",
		TimeOut: time.Second * 5,
	}

	actual := New()
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("got %v\nwant %v", actual, expected)
	}
}

func TestNewWithConfig(t *testing.T) {
	config := Config{
		IsProduction: true,
		TimeOut:      time.Second * 2,
	}

	expected := Client{
		URL:     "https://buy.itunes.apple.com/verifyReceipt",
		TimeOut: time.Second * 2,
	}

	actual := NewWithConfig(config)
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("got %v\nwant %v", actual, expected)
	}
}

func TestVerify(t *testing.T) {
	client := New()

	expected := IAPResponse{
		Status: 21002,
	}
	req := IAPRequest{
		ReceiptData: "dummy data",
	}
	actual, _ := client.Verify(req)
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("got %v\nwant %v", actual, expected)
	}
}
<commit_msg>AppStore test coverage 100%<commit_after>package appstore

import (
	"errors"
	"fmt"
	"net/http"
	"net/http/httptest"
	"os"
	"reflect"
	"testing"
	"time"
)

func TestHandleError(t *testing.T) {
	var expected, actual error

	// status 0
	expected = nil
	actual = HandleError(0)
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("got %v\nwant %v", actual, expected)
	}

	// status 21000
	expected = errors.New("The App Store could not read the JSON object you provided.")
	actual = HandleError(21000)
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("got %v\nwant %v", actual, expected)
	}

	// status 21002
	expected = errors.New("The data in the receipt-data property was malformed or missing.")
	actual = HandleError(21002)
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("got %v\nwant %v", actual, expected)
	}

	// status 21003
	expected = errors.New("The receipt could not be authenticated.")
	actual = HandleError(21003)
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("got %v\nwant %v", actual, expected)
	}

	// status 21005
	expected = errors.New("The receipt server is not currently available.")
	actual = HandleError(21005)
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("got %v\nwant %v", actual, expected)
	}

	// status 21007
	expected = errors.New("This receipt is from the test environment, but it was sent to the production environment for verification. Send it to the test environment instead.")
	actual = HandleError(21007)
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("got %v\nwant %v", actual, expected)
	}

	// status 21008
	expected = errors.New("This receipt is from the production environment, but it was sent to the test environment for verification. Send it to the production environment instead.")
	actual = HandleError(21008)
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("got %v\nwant %v", actual, expected)
	}

	// status unkown
	expected = errors.New("An unknown error ocurred")
	actual = HandleError(100)
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("got %v\nwant %v", actual, expected)
	}
}

func TestNew(t *testing.T) {
	expected := Client{
		URL:     "https://sandbox.itunes.apple.com/verifyReceipt",
		TimeOut: time.Second * 5,
	}

	actual := New()
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("got %v\nwant %v", actual, expected)
	}
}

func TestNewWithEnvironment(t *testing.T) {
	expected := Client{
		URL:     "https://buy.itunes.apple.com/verifyReceipt",
		TimeOut: time.Second * 5,
	}

	os.Setenv("IAP_ENVIRONMENT", "production")
	actual := New()
	os.Clearenv()

	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("got %v\nwant %v", actual, expected)
	}
}

func TestNewWithConfig(t *testing.T) {
	config := Config{
		IsProduction: true,
		TimeOut:      time.Second * 2,
	}

	expected := Client{
		URL:     "https://buy.itunes.apple.com/verifyReceipt",
		TimeOut: time.Second * 2,
	}

	actual := NewWithConfig(config)
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("got %v\nwant %v", actual, expected)
	}
}

func TestNewWithConfigTimeout(t *testing.T) {
	config := Config{
		IsProduction: true,
	}

	expected := Client{
		URL:     "https://buy.itunes.apple.com/verifyReceipt",
		TimeOut: time.Second * 5,
	}

	actual := NewWithConfig(config)
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("got %v\nwant %v", actual, expected)
	}
}

func TestVerify(t *testing.T) {
	client := New()

	expected := IAPResponse{
		Status: 21002,
	}
	req := IAPRequest{
		ReceiptData: "dummy data",
	}
	actual, _ := client.Verify(req)
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("got %v\nwant %v", actual, expected)
	}
}

func TestVerifyErrors(t *testing.T) {
	server, client := testTools(199, "dummy response")
	defer server.Close()

	req := IAPRequest{
		ReceiptData: "dummy data",
	}

	expected := errors.New("An error occurred in IAP - code:199")
	_, actual := client.Verify(req)
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("got %v\nwant %v", actual, expected)
	}
}

func TestVerifyTimeout(t *testing.T) {
	// HTTP 100 is "continue" so it will time out
	server, client := testTools(100, "dummy response")
	defer server.Close()

	req := IAPRequest{
		ReceiptData: "dummy data",
	}

	expected := errors.New("")
	_, actual := client.Verify(req)
	if !reflect.DeepEqual(reflect.TypeOf(actual), reflect.TypeOf(expected)) {
		t.Errorf("got %v\nwant %v", actual, expected)
	}
}

func testTools(code int, body string) (*httptest.Server, *Client) {

	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(code)
		w.Header().Set("Content-Type", "application/json")
		fmt.Fprintln(w, body)
	}))

	client := &Client{URL: server.URL, TimeOut: time.Second * 2}
	return server, client
}
<|endoftext|>