<commit_before>package scipipe

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sync"
	"time"
)

// IP Is the base interface which all other IPs need to adhere to
type IP interface {
	ID() string
	Atomize()
	// ----------------------------------------
	// Some tentative additions:
	// ----------------------------------------
	// Digest() string
	// URL() string        // Example: file:///proj/cheminf/exp/20180101-logd/dat/rawdata.tsv
	// StagedPath() string // Example: /proj/cheminf/exp/20180101-logd/dat/rawdata.tsv
	// TempPath() string
	// EnsureStaged()
	// EnsureUnstaged()
	// ParamType() int // string / int8 / float64 / bool / date? / time?
	// Param() string
	// Key() string
	// Keys() map[string]string
	// AddKey(key string)
	// AddKeys(keys ...string)
	// AuditInfo()
	// SetAuditInfo()
	// AuditInfoFilePath() string
	// WriteAuditLogToFile()
	// Params() map[string]string
	// ----------------------------------------
	// Persistable() bool // Whether the content or data of the IP can be persisted, e.g. by being written to disk
	// Read() []byte
	// Write(data []byte)
	// OpenR() io.Reader // Return a reader interface to read content directly
	// OpenW() io.Writer // (Possibly relevant for object storage without staging)
	// OpenRW() io.ReadWriter
	// Close()
	// ----------------------------------------
}

// ------------------------------------------------------------------------
// BaseIP type
// ------------------------------------------------------------------------

// BaseIP contains foundational functionality which all IPs need to implement.
// It is meant to be embedded into other IP implementations.
type BaseIP struct {
	path      string
	id        string
	auditInfo *AuditInfo
}

// NewBaseIP creates a new BaseIP
func NewBaseIP(path string) *BaseIP {
	return &BaseIP{
		path: path,
		id:   randSeqLC(20),
	}
}

// ID returns a globally unique ID for the IP
func (ip *BaseIP) ID() string {
	return ip.id
}

// ------------------------------------------------------------------------
// FileIP type
// ------------------------------------------------------------------------

// FileIP (Short for "Information Packet" in Flow-Based Programming terminology)
// contains information and helper methods for a physical file on a normal disk.
type FileIP struct {
	*BaseIP
	buffer    *bytes.Buffer
	doStream  bool
	lock      *sync.Mutex
	SubStream *InPort
}

// NewFileIP creates a new FileIP
func NewFileIP(path string) *FileIP {
	ip := &FileIP{
		BaseIP:    NewBaseIP(path),
		lock:      &sync.Mutex{},
		SubStream: NewInPort("in_substream"),
	}
	if ip.Exists() {
		ip.AuditInfo() // This will populate the audit info from file
	}
	//Don't init buffer if not needed?
	//buf := make([]byte, 0, 128)
	//ip.buffer = bytes.NewBuffer(buf)
	return ip
}

// ------------------------------------------------------------------------
// Path stuff
// ------------------------------------------------------------------------

// Path returns the (final) path of the physical file
func (ip *FileIP) Path() string {
	return ip.path
}

// TempPath returns the temporary path of the physical file
func (ip *FileIP) TempPath() string {
	return ip.path + ".tmp"
}

// FifoPath returns the path to use when a FIFO file is used instead of a
// normal file
func (ip *FileIP) FifoPath() string {
	return ip.path + ".fifo"
}

// ------------------------------------------------------------------------
// Check-thing stuff
// ------------------------------------------------------------------------

// Size returns the size of an existing file, in bytes
func (ip *FileIP) Size() int64 {
	fi, err := os.Stat(ip.path)
	Check(err)
	return fi.Size()
}

// Exists checks if the file exists (at its final file name)
func (ip *FileIP) Exists() bool {
	exists := false
	ip.lock.Lock()
	if _, err := os.Stat(ip.Path()); err == nil {
		exists = true
	}
	ip.lock.Unlock()
	return exists
}

// TempFileExists checks if the temp-file exists
func (ip *FileIP) TempFileExists() bool {
	tempFileExists := false
	ip.lock.Lock()
	if _, err := os.Stat(ip.TempPath()); err == nil {
		tempFileExists = true
	}
	ip.lock.Unlock()
	return tempFileExists
}

// FifoFileExists checks if the FIFO-file (named pipe file) exists
func (ip *FileIP) FifoFileExists() bool {
	fifoFileExists := false
	ip.lock.Lock()
	if _, err := os.Stat(ip.FifoPath()); err == nil {
		fifoFileExists = true
	}
	ip.lock.Unlock()
	return fifoFileExists
}

// ------------------------------------------------------------------------
// Open file-stuff
// ------------------------------------------------------------------------

// Open opens the file and returns a file handle (*os.File)
func (ip *FileIP) Open() *os.File {
	f, err := os.Open(ip.Path())
	CheckWithMsg(err, "Could not open file: "+ip.Path())
	return f
}

// OpenTemp opens the temp file and returns a file handle (*os.File)
func (ip *FileIP) OpenTemp() *os.File {
	f, err := os.Open(ip.TempPath())
	CheckWithMsg(err, "Could not open temp file: "+ip.TempPath())
	return f
}

// OpenWriteTemp opens the file for writing, and returns a file handle (*os.File)
func (ip *FileIP) OpenWriteTemp() *os.File {
	ip.createDirs()
	f, err := os.Create(ip.TempPath())
	CheckWithMsg(err, "Could not open temp file for writing: "+ip.TempPath())
	return f
}

// ------------------------------------------------------------------------
// FIFO-specific stuff
// ------------------------------------------------------------------------

// CreateFifo creates a FIFO file for the FileIP
func (ip *FileIP) CreateFifo() {
	ip.createDirs()
	ip.lock.Lock()
	cmd := "mkfifo " + ip.FifoPath()
	Debug.Println("Now creating FIFO with command:", cmd)

	if _, err := os.Stat(ip.FifoPath()); err == nil {
		Warning.Println("FIFO already exists, so not creating a new one:", ip.FifoPath())
	} else {
		_, err := exec.Command("bash", "-c", cmd).Output()
		CheckWithMsg(err, "Could not execute command: "+cmd)
	}

	ip.lock.Unlock()
}

// RemoveFifo removes the FIFO file, if it exists
func (ip *FileIP) RemoveFifo() {
	// FIXME: Shouldn't we check first whether the fifo exists?
	ip.lock.Lock()
	output, err := exec.Command("bash", "-c", "rm "+ip.FifoPath()).Output()
	CheckWithMsg(err, "Could not delete fifo file: "+ip.FifoPath())
	Debug.Println("Removed FIFO output: ", output)
	ip.lock.Unlock()
}

// ------------------------------------------------------------------------
// Read/Write stuff
// ------------------------------------------------------------------------

// Read reads the whole content of the file and returns the content as a byte
// array
func (ip *FileIP) Read() []byte {
	dat, err := ioutil.ReadFile(ip.Path())
	CheckWithMsg(err, "Could not open file for reading: "+ip.Path())
	return dat
}

// Write writes a byte array ([]byte) to the file's temp file path
func (ip *FileIP) Write(dat []byte) {
	ip.createDirs()
	err := ioutil.WriteFile(ip.TempPath(), dat, 0644)
	CheckWithMsg(err, "Could not write to temp file: "+ip.TempPath())
}

const (
	sleepDurationSec = 1
)

// Atomize renames the temporary file name to the final file name, thus enabling
// to separate unfinished, and finished files
func (ip *FileIP) Atomize() {
	Debug.Println("FileIP: Atomizing", ip.TempPath(), "->", ip.Path())
	doneAtomizing := false
	for !doneAtomizing {
		if ip.TempFileExists() {
			ip.lock.Lock()
			err := os.Rename(ip.TempPath(), ip.path)
			CheckWithMsg(err, "Could not rename file: "+ip.TempPath())
			ip.lock.Unlock()
			doneAtomizing = true
			Debug.Println("FileIP: Done atomizing", ip.TempPath(), "->", ip.Path())
		} else {
			Debug.Printf("Sleeping for %d seconds before atomizing ...\n", sleepDurationSec)
			time.Sleep(time.Duration(sleepDurationSec) * time.Second)
		}
	}
}

// ------------------------------------------------------------------------
// Params and keys
// ------------------------------------------------------------------------

// Param returns the parameter named key, from the IPs audit info
func (ip *FileIP) Param(key string) string {
	val, ok := ip.AuditInfo().Params[key]
	if !ok {
		Failf("Could not find parameter %s in ip with path: %s\n", key, ip.Path())
	}
	return val
}

// ------------------------------------------------------------------------
// Keys stuff
// ------------------------------------------------------------------------

// Key returns the key value for the key with key k from the IPs audit info
// (don't confuse this with the keys of maps in go. Keys in this case is a
// SciPipe audit info concept)
func (ip *FileIP) Key(k string) string {
	v, ok := ip.AuditInfo().Keys[k]
	if !ok {
		Failf("Could not find key %s in ip with path: %s\n", k, ip.Path())
	}
	return v
}

// Keys returns the audit info's key values
func (ip *FileIP) Keys() map[string]string {
	return ip.AuditInfo().Keys
}

// AddKey adds the key k with value v
func (ip *FileIP) AddKey(k string, v string) {
	ai := ip.AuditInfo()
	if ai.Keys[k] != "" && ai.Keys[k] != v {
		Failf("Can not add value %s to existing key %s with different value %s\n", v, k, ai.Keys[k])
	}
	ai.Keys[k] = v
}

// AddKeys adds a map of keys to the IPs audit info
func (ip *FileIP) AddKeys(keys map[string]string) {
	for k, v := range keys {
		ip.AddKey(k, v)
	}
}

// ------------------------------------------------------------------------
// AuditInfo stuff
// ------------------------------------------------------------------------

// AuditFilePath returns the file path of the audit info file for the FileIP
func (ip *FileIP) AuditFilePath() string {
	return ip.Path() + ".audit.json"
}

// SetAuditInfo sets the AuditInfo struct for the FileIP
func (ip *FileIP) SetAuditInfo(ai *AuditInfo) {
	ip.lock.Lock()
	ip.auditInfo = ai
	ip.lock.Unlock()
}

// WriteAuditLogToFile writes the audit log to its designated file
func (ip *FileIP) WriteAuditLogToFile() {
	auditInfo := ip.AuditInfo()
	auditInfoJSON, jsonErr := json.MarshalIndent(auditInfo, "", "    ")
	CheckWithMsg(jsonErr, "Could not marshall JSON")
	ip.createDirs()
	writeErr := ioutil.WriteFile(ip.AuditFilePath(), auditInfoJSON, 0644)
	CheckWithMsg(writeErr, "Could not write audit file: "+ip.Path())
}

// AuditInfo returns the AuditInfo struct for the FileIP
func (ip *FileIP) AuditInfo() *AuditInfo {
	defer ip.lock.Unlock()
	ip.lock.Lock()
	if ip.auditInfo == nil {
		ip.auditInfo = NewAuditInfo()
		auditFileData, readFileErr := ioutil.ReadFile(ip.AuditFilePath())
		if readFileErr != nil {
			Warning.Println("Could not read audit file: " + ip.AuditFilePath() + " (" + readFileErr.Error() + ")")
		} else {
			unmarshalErr := json.Unmarshal(auditFileData, ip.auditInfo)
			CheckWithMsg(unmarshalErr, "Could not unmarshal audit log file content: "+ip.AuditFilePath())
		}
	}
	return ip.auditInfo
}

// ------------------------------------------------------------------------
// Extra convenience functions
// ------------------------------------------------------------------------

// UnMarshalJSON is a helper function to unmarshal the content of the IPs file
// to the interface v
func (ip *FileIP) UnMarshalJSON(v interface{}) {
	d := ip.Read()
	err := json.Unmarshal(d, v)
	CheckWithMsg(err, "Could not unmarshal content of file: "+ip.Path())
}

// ------------------------------------------------------------------------
// Helper functions
// ------------------------------------------------------------------------

// CreateDirs creates all directories needed to enable writing the IP to its
// path (or temporary-path, which will have the same directory)
func (ip *FileIP) createDirs() {
	dir := filepath.Dir(ip.Path())
	err := os.MkdirAll(dir, 0777)
	CheckWithMsg(err, "Could not create directory: "+dir)
}
<commit_msg>Output warnings on missing .tmp files, use an exponential back off to re-check three times, before failing<commit_after>package scipipe

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sync"
	"time"
)

// IP Is the base interface which all other IPs need to adhere to
type IP interface {
	ID() string
	Atomize()
	// ----------------------------------------
	// Some tentative additions:
	// ----------------------------------------
	// Digest() string
	// URL() string        // Example: file:///proj/cheminf/exp/20180101-logd/dat/rawdata.tsv
	// StagedPath() string // Example: /proj/cheminf/exp/20180101-logd/dat/rawdata.tsv
	// TempPath() string
	// EnsureStaged()
	// EnsureUnstaged()
	// ParamType() int // string / int8 / float64 / bool / date? / time?
	// Param() string
	// Key() string
	// Keys() map[string]string
	// AddKey(key string)
	// AddKeys(keys ...string)
	// AuditInfo()
	// SetAuditInfo()
	// AuditInfoFilePath() string
	// WriteAuditLogToFile()
	// Params() map[string]string
	// ----------------------------------------
	// Persistable() bool // Whether the content or data of the IP can be persisted, e.g. by being written to disk
	// Read() []byte
	// Write(data []byte)
	// OpenR() io.Reader // Return a reader interface to read content directly
	// OpenW() io.Writer // (Possibly relevant for object storage without staging)
	// OpenRW() io.ReadWriter
	// Close()
	// ----------------------------------------
}

// ------------------------------------------------------------------------
// BaseIP type
// ------------------------------------------------------------------------

// BaseIP contains foundational functionality which all IPs need to implement.
// It is meant to be embedded into other IP implementations.
type BaseIP struct {
	path      string
	id        string
	auditInfo *AuditInfo
}

// NewBaseIP creates a new BaseIP
func NewBaseIP(path string) *BaseIP {
	return &BaseIP{
		path: path,
		id:   randSeqLC(20),
	}
}

// ID returns a globally unique ID for the IP
func (ip *BaseIP) ID() string {
	return ip.id
}

// ------------------------------------------------------------------------
// FileIP type
// ------------------------------------------------------------------------

// FileIP (Short for "Information Packet" in Flow-Based Programming terminology)
// contains information and helper methods for a physical file on a normal disk.
type FileIP struct {
	*BaseIP
	buffer    *bytes.Buffer
	doStream  bool
	lock      *sync.Mutex
	SubStream *InPort
}

// NewFileIP creates a new FileIP
func NewFileIP(path string) *FileIP {
	ip := &FileIP{
		BaseIP:    NewBaseIP(path),
		lock:      &sync.Mutex{},
		SubStream: NewInPort("in_substream"),
	}
	if ip.Exists() {
		ip.AuditInfo() // This will populate the audit info from file
	}
	//Don't init buffer if not needed?
	//buf := make([]byte, 0, 128)
	//ip.buffer = bytes.NewBuffer(buf)
	return ip
}

// ------------------------------------------------------------------------
// Path stuff
// ------------------------------------------------------------------------

// Path returns the (final) path of the physical file
func (ip *FileIP) Path() string {
	return ip.path
}

// TempPath returns the temporary path of the physical file
func (ip *FileIP) TempPath() string {
	return ip.path + ".tmp"
}

// FifoPath returns the path to use when a FIFO file is used instead of a
// normal file
func (ip *FileIP) FifoPath() string {
	return ip.path + ".fifo"
}

// ------------------------------------------------------------------------
// Check-thing stuff
// ------------------------------------------------------------------------

// Size returns the size of an existing file, in bytes
func (ip *FileIP) Size() int64 {
	fi, err := os.Stat(ip.path)
	Check(err)
	return fi.Size()
}

// Exists checks if the file exists (at its final file name)
func (ip *FileIP) Exists() bool {
	exists := false
	ip.lock.Lock()
	if _, err := os.Stat(ip.Path()); err == nil {
		exists = true
	}
	ip.lock.Unlock()
	return exists
}

// TempFileExists checks if the temp-file exists
func (ip *FileIP) TempFileExists() bool {
	tempFileExists := false
	ip.lock.Lock()
	if _, err := os.Stat(ip.TempPath()); err == nil {
		tempFileExists = true
	}
	ip.lock.Unlock()
	return tempFileExists
}

// FifoFileExists checks if the FIFO-file (named pipe file) exists
func (ip *FileIP) FifoFileExists() bool {
	fifoFileExists := false
	ip.lock.Lock()
	if _, err := os.Stat(ip.FifoPath()); err == nil {
		fifoFileExists = true
	}
	ip.lock.Unlock()
	return fifoFileExists
}

// ------------------------------------------------------------------------
// Open file-stuff
// ------------------------------------------------------------------------

// Open opens the file and returns a file handle (*os.File)
func (ip *FileIP) Open() *os.File {
	f, err := os.Open(ip.Path())
	CheckWithMsg(err, "Could not open file: "+ip.Path())
	return f
}

// OpenTemp opens the temp file and returns a file handle (*os.File)
func (ip *FileIP) OpenTemp() *os.File {
	f, err := os.Open(ip.TempPath())
	CheckWithMsg(err, "Could not open temp file: "+ip.TempPath())
	return f
}

// OpenWriteTemp opens the file for writing, and returns a file handle (*os.File)
func (ip *FileIP) OpenWriteTemp() *os.File {
	ip.createDirs()
	f, err := os.Create(ip.TempPath())
	CheckWithMsg(err, "Could not open temp file for writing: "+ip.TempPath())
	return f
}

// ------------------------------------------------------------------------
// FIFO-specific stuff
// ------------------------------------------------------------------------

// CreateFifo creates a FIFO file for the FileIP
func (ip *FileIP) CreateFifo() {
	ip.createDirs()
	ip.lock.Lock()
	cmd := "mkfifo " + ip.FifoPath()
	Debug.Println("Now creating FIFO with command:", cmd)

	if _, err := os.Stat(ip.FifoPath()); err == nil {
		Warning.Println("FIFO already exists, so not creating a new one:", ip.FifoPath())
	} else {
		_, err := exec.Command("bash", "-c", cmd).Output()
		CheckWithMsg(err, "Could not execute command: "+cmd)
	}

	ip.lock.Unlock()
}

// RemoveFifo removes the FIFO file, if it exists
func (ip *FileIP) RemoveFifo() {
	// FIXME: Shouldn't we check first whether the fifo exists?
	ip.lock.Lock()
	output, err := exec.Command("bash", "-c", "rm "+ip.FifoPath()).Output()
	CheckWithMsg(err, "Could not delete fifo file: "+ip.FifoPath())
	Debug.Println("Removed FIFO output: ", output)
	ip.lock.Unlock()
}

// ------------------------------------------------------------------------
// Read/Write stuff
// ------------------------------------------------------------------------

// Read reads the whole content of the file and returns the content as a byte
// array
func (ip *FileIP) Read() []byte {
	dat, err := ioutil.ReadFile(ip.Path())
	CheckWithMsg(err, "Could not open file for reading: "+ip.Path())
	return dat
}

// Write writes a byte array ([]byte) to the file's temp file path
func (ip *FileIP) Write(dat []byte) {
	ip.createDirs()
	err := ioutil.WriteFile(ip.TempPath(), dat, 0644)
	CheckWithMsg(err, "Could not write to temp file: "+ip.TempPath())
}

var (
	sleepDurationSec = 1
)

const (
	maxTries      = 3
	backoffFactor = 4
)

// Atomize renames the temporary file name to the final file name, thus enabling
// to separate unfinished, and finished files
func (ip *FileIP) Atomize() {
	Debug.Println("FileIP: Atomizing", ip.TempPath(), "->", ip.Path())
	doneAtomizing := false
	tries := 0

	for !doneAtomizing {
		if ip.TempFileExists() {
			ip.lock.Lock()
			err := os.Rename(ip.TempPath(), ip.path)
			CheckWithMsg(err, "Could not rename file: "+ip.TempPath())
			ip.lock.Unlock()
			doneAtomizing = true
			Debug.Println("FileIP: Done atomizing", ip.TempPath(), "->", ip.Path())
		} else {
			if tries >= maxTries {
				Failf("Failed to find .tmp file after %d tries, so shutting down: %s\nNote: If this problem persists, it could be a problem with your workflow, that the configured output filename in scipipe doesn't match what is written by the tool.\n", maxTries, ip.TempPath())
			}
			Warning.Printf("Expected .tmp file missing: %s\nSleeping for %d seconds before checking again ...\n", ip.TempPath(), sleepDurationSec)
			time.Sleep(time.Duration(sleepDurationSec) * time.Second)
			sleepDurationSec *= backoffFactor
			tries += 1
		}
	}
}

// ------------------------------------------------------------------------
// Params and keys
// ------------------------------------------------------------------------

// Param returns the parameter named key, from the IPs audit info
func (ip *FileIP) Param(key string) string {
	val, ok := ip.AuditInfo().Params[key]
	if !ok {
		Failf("Could not find parameter %s in ip with path: %s\n", key, ip.Path())
	}
	return val
}

// ------------------------------------------------------------------------
// Keys stuff
// ------------------------------------------------------------------------

// Key returns the key value for the key with key k from the IPs audit info
// (don't confuse this with the keys of maps in go. Keys in this case is a
// SciPipe audit info concept)
func (ip *FileIP) Key(k string) string {
	v, ok := ip.AuditInfo().Keys[k]
	if !ok {
		Failf("Could not find key %s in ip with path: %s\n", k, ip.Path())
	}
	return v
}

// Keys returns the audit info's key values
func (ip *FileIP) Keys() map[string]string {
	return ip.AuditInfo().Keys
}

// AddKey adds the key k with value v
func (ip *FileIP) AddKey(k string, v string) {
	ai := ip.AuditInfo()
	if ai.Keys[k] != "" && ai.Keys[k] != v {
		Failf("Can not add value %s to existing key %s with different value %s\n", v, k, ai.Keys[k])
	}
	ai.Keys[k] = v
}

// AddKeys adds a map of keys to the IPs audit info
func (ip *FileIP) AddKeys(keys map[string]string) {
	for k, v := range keys {
		ip.AddKey(k, v)
	}
}

// ------------------------------------------------------------------------
// AuditInfo stuff
// ------------------------------------------------------------------------

// AuditFilePath returns the file path of the audit info file for the FileIP
func (ip *FileIP) AuditFilePath() string {
	return ip.Path() + ".audit.json"
}

// SetAuditInfo sets the AuditInfo struct for the FileIP
func (ip *FileIP) SetAuditInfo(ai *AuditInfo) {
	ip.lock.Lock()
	ip.auditInfo = ai
	ip.lock.Unlock()
}

// WriteAuditLogToFile writes the audit log to its designated file
func (ip *FileIP) WriteAuditLogToFile() {
	auditInfo := ip.AuditInfo()
	auditInfoJSON, jsonErr := json.MarshalIndent(auditInfo, "", "    ")
	CheckWithMsg(jsonErr, "Could not marshall JSON")
	ip.createDirs()
	writeErr := ioutil.WriteFile(ip.AuditFilePath(), auditInfoJSON, 0644)
	CheckWithMsg(writeErr, "Could not write audit file: "+ip.Path())
}

// AuditInfo returns the AuditInfo struct for the FileIP
func (ip *FileIP) AuditInfo() *AuditInfo {
	defer ip.lock.Unlock()
	ip.lock.Lock()
	if ip.auditInfo == nil {
		ip.auditInfo = NewAuditInfo()
		auditFileData, readFileErr := ioutil.ReadFile(ip.AuditFilePath())
		if readFileErr != nil {
			Warning.Println("Could not read audit file: " + ip.AuditFilePath() + " (" + readFileErr.Error() + ")")
		} else {
			unmarshalErr := json.Unmarshal(auditFileData, ip.auditInfo)
			CheckWithMsg(unmarshalErr, "Could not unmarshal audit log file content: "+ip.AuditFilePath())
		}
	}
	return ip.auditInfo
}

// ------------------------------------------------------------------------
// Extra convenience functions
// ------------------------------------------------------------------------

// UnMarshalJSON is a helper function to unmarshal the content of the IPs file
// to the interface v
func (ip *FileIP) UnMarshalJSON(v interface{}) {
	d := ip.Read()
	err := json.Unmarshal(d, v)
	CheckWithMsg(err, "Could not unmarshal content of file: "+ip.Path())
}

// ------------------------------------------------------------------------
// Helper functions
// ------------------------------------------------------------------------

// CreateDirs creates all directories needed to enable writing the IP to its
// path (or temporary-path, which will have the same directory)
func (ip *FileIP) createDirs() {
	dir := filepath.Dir(ip.Path())
	err := os.MkdirAll(dir, 0777)
	CheckWithMsg(err, "Could not create directory: "+dir)
}
<|endoftext|>


<commit_before>//
// ls.go
// Copyright(c)2014-2015 Google, Inc.
//
// This file is part of skicka.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package main

import (
	"fmt"
	"github.com/google/skicka/gdrive"
	"os"
	"path/filepath"
	"time"
)

func getPermissionsAsString(driveFile *gdrive.File) (string, error) {
	var str string
	if driveFile.IsFolder() {
		str = "d"
	} else {
		str = "-"
	}

	perm, err := getPermissions(driveFile)
	if err != nil {
		// No permissions are available if the file was uploaded via the
		// Drive Web page, for example.
		str += "?????????"
	} else {
		rwx := "rwx"
		for i := 0; i < 9; i++ {
			if perm&(1<<(8-uint(i))) != 0 {
				str += string(rwx[i%3])
			} else {
				str += "-"
			}
		}
	}
	return str, nil
}

func ls(args []string) int {
	// Parse command line arguments.
	long := false
	longlong := false
	recursive := false
	dirAsFile := false
	var argFilenames []string
	for _, arg := range args {
		if len(argFilenames) > 0 {
			// After the end of command-line arguments, all subsequent args
			// are treated as filenames, regardless of whether they match
			// any of the flags.
			argFilenames = append(argFilenames, arg)
		} else if arg == "-l" {
			long = true
		} else if arg == "-ll" {
			longlong = true
		} else if arg == "-r" {
			recursive = true
		} else if arg == "-d" {
			dirAsFile = true
		} else if len(arg) > 0 && arg[0] == '-' {
			fmt.Printf("Usage: skicka ls [-d,-l,-ll,-r] [drive_path ...]\n")
			fmt.Printf("Run \"skicka help\" for more detailed help text.\n")
			return 1
		} else {
			argFilenames = append(argFilenames, arg)
		}
	}

	if len(argFilenames) == 0 {
		argFilenames = append(argFilenames, string(os.PathSeparator))
	}

	errs := 0
	for index, drivePath := range argFilenames {
		drivePath = filepath.Clean(drivePath)

		if len(argFilenames) > 1 {
			fmt.Printf("%s:\n", drivePath)
		}

		// Get the files for the current path from Google Drive.
		files := gd.GetFiles(drivePath)
		if len(files) == 0 {
			fmt.Fprintf(os.Stderr, "skicka: %s: file not found\n", drivePath)
			errs++
			continue
		}

		if !files[0].IsFolder() || dirAsFile {
			// If the user specified a full path to a regular file on the
			// command line or gave the -d option, then don't try to list
			// the directory contents.
			for _, f := range files {
				lsFile(f, recursive, long, longlong)
			}
		} else {
			// Otherwise get either the files in the enclosing folder, or
			// all files under the folder, depending on whether the
			// recursive option was specified.
			includeBase := false
			var files []*gdrive.File
			var err error
			if recursive {
				files, err = gd.GetFilesUnderFolder(drivePath, includeBase)
			} else {
				files, err = gd.GetFilesInFolder(drivePath)
			}

			if err != nil {
				fmt.Fprintf(os.Stderr, "skicka: %s: %v\n", drivePath, err)
				errs++
				continue
			}

			for _, f := range files {
				lsFile(f, recursive, long, longlong)
			}

			if len(argFilenames) > 1 && index < len(argFilenames)-1 {
				fmt.Printf("\n")
			}
		}
	}
	return errs
}

// Produce listing output to stdout for a single file.
func lsFile(f *gdrive.File, recursive, long, longlong bool) {
	printFilename := f.Path
	if !recursive {
		printFilename = filepath.Base(printFilename)
	}
	if f.IsFolder() {
		printFilename += string(os.PathSeparator)
	}

	if !long && !longlong {
		fmt.Printf("%s\n", printFilename)
		return
	}

	synctime := f.ModTime
	permString, _ := getPermissionsAsString(f)
	if longlong {
		md5 := f.Md5
		if len(md5) != 32 {
			md5 = "--------------------------------"
		}
		fmt.Printf("%s  %s  %s  %s  %s\n", permString,
			fmtbytes(f.FileSize, true), md5, synctime.Format(time.ANSIC),
			printFilename)
		if debug {
			fmt.Printf("\t[ ")
			for _, prop := range f.Properties {
				fmt.Printf("%s: %s, ", prop.Key, prop.Value)
			}
			fmt.Printf("MimeType: %s, ", f.MimeType)
			fmt.Printf("id: %s ]\n", f.Id)
		}
	} else {
		fmt.Printf("%s  %s  %s  %s\n", permString, fmtbytes(f.FileSize, true),
			synctime.Format(time.ANSIC), printFilename)
	}
}
<commit_msg>Display local time.<commit_after>//
// ls.go
// Copyright(c)2014-2015 Google, Inc.
//
// This file is part of skicka.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package main

import (
	"fmt"
	"os"
	"path/filepath"
	"time"

	"github.com/google/skicka/gdrive"
)

func getPermissionsAsString(driveFile *gdrive.File) (string, error) {
	var str string
	if driveFile.IsFolder() {
		str = "d"
	} else {
		str = "-"
	}

	perm, err := getPermissions(driveFile)
	if err != nil {
		// No permissions are available if the file was uploaded via the
		// Drive Web page, for example.
		str += "?????????"
	} else {
		rwx := "rwx"
		for i := 0; i < 9; i++ {
			if perm&(1<<(8-uint(i))) != 0 {
				str += string(rwx[i%3])
			} else {
				str += "-"
			}
		}
	}
	return str, nil
}

func ls(args []string) int {
	// Parse command line arguments.
	long := false
	longlong := false
	recursive := false
	dirAsFile := false
	var argFilenames []string
	for _, arg := range args {
		if len(argFilenames) > 0 {
			// After the end of command-line arguments, all subsequent args
			// are treated as filenames, regardless of whether they match
			// any of the flags.
			argFilenames = append(argFilenames, arg)
		} else if arg == "-l" {
			long = true
		} else if arg == "-ll" {
			longlong = true
		} else if arg == "-r" {
			recursive = true
		} else if arg == "-d" {
			dirAsFile = true
		} else if len(arg) > 0 && arg[0] == '-' {
			fmt.Printf("Usage: skicka ls [-d,-l,-ll,-r] [drive_path ...]\n")
			fmt.Printf("Run \"skicka help\" for more detailed help text.\n")
			return 1
		} else {
			argFilenames = append(argFilenames, arg)
		}
	}

	if len(argFilenames) == 0 {
		argFilenames = append(argFilenames, string(os.PathSeparator))
	}

	errs := 0
	for index, drivePath := range argFilenames {
		drivePath = filepath.Clean(drivePath)

		if len(argFilenames) > 1 {
			fmt.Printf("%s:\n", drivePath)
		}

		// Get the files for the current path from Google Drive.
		files := gd.GetFiles(drivePath)
		if len(files) == 0 {
			fmt.Fprintf(os.Stderr, "skicka: %s: file not found\n", drivePath)
			errs++
			continue
		}

		if !files[0].IsFolder() || dirAsFile {
			// If the user specified a full path to a regular file on the
			// command line or gave the -d option, then don't try to list
			// the directory contents.
			for _, f := range files {
				lsFile(f, recursive, long, longlong)
			}
		} else {
			// Otherwise get either the files in the enclosing folder, or
			// all files under the folder, depending on whether the
			// recursive option was specified.
			includeBase := false
			var files []*gdrive.File
			var err error
			if recursive {
				files, err = gd.GetFilesUnderFolder(drivePath, includeBase)
			} else {
				files, err = gd.GetFilesInFolder(drivePath)
			}

			if err != nil {
				fmt.Fprintf(os.Stderr, "skicka: %s: %v\n", drivePath, err)
				errs++
				continue
			}

			for _, f := range files {
				lsFile(f, recursive, long, longlong)
			}

			if len(argFilenames) > 1 && index < len(argFilenames)-1 {
				fmt.Printf("\n")
			}
		}
	}
	return errs
}

// Produce listing output to stdout for a single file.
func lsFile(f *gdrive.File, recursive, long, longlong bool) {
	printFilename := f.Path
	if !recursive {
		printFilename = filepath.Base(printFilename)
	}
	if f.IsFolder() {
		printFilename += string(os.PathSeparator)
	}

	if !long && !longlong {
		fmt.Printf("%s\n", printFilename)
		return
	}

	synctime := f.ModTime.Local()
	permString, _ := getPermissionsAsString(f)
	if longlong {
		md5 := f.Md5
		if len(md5) != 32 {
			md5 = "--------------------------------"
		}
		fmt.Printf("%s  %s  %s  %s  %s\n", permString,
			fmtbytes(f.FileSize, true), md5, synctime.Format(time.ANSIC),
			printFilename)
		if debug {
			fmt.Printf("\t[ ")
			for _, prop := range f.Properties {
				fmt.Printf("%s: %s, ", prop.Key, prop.Value)
			}
			fmt.Printf("MimeType: %s, ", f.MimeType)
			fmt.Printf("id: %s ]\n", f.Id)
		}
	} else {
		fmt.Printf("%s  %s  %s  %s\n", permString, fmtbytes(f.FileSize, true),
			synctime.Format(time.ANSIC), printFilename)
	}
}
<|endoftext|>


<commit_before>package rwvfs

import (
	"errors"
	"fmt"
	"io"
	"os"
	pathpkg "path"
	"path/filepath"
	"strings"

	"golang.org/x/tools/godoc/vfs"
)

// OS returns an implementation of FileSystem reading from the tree rooted at
// root.
func OS(root string) FileSystem {
	return OSPerm(root, 0666, 0755)
}

func OSPerm(root string, filePerm, dirPerm os.FileMode) FileSystem {
	return osFS{
		root:       root,
		filePerm:   filePerm,
		dirPerm:    dirPerm,
		FileSystem: vfs.OS(root),
	}
}

type osFS struct {
	root     string
	filePerm os.FileMode
	dirPerm  os.FileMode
	vfs.FileSystem
}

// resolve is from golang.org/x/tools/godoc/vfs.
func (fs osFS) resolve(path string) string {
	// Clean the path so that it cannot possibly begin with ../.
	// If it did, the result of filepath.Join would be outside the
	// tree rooted at root.  We probably won't ever see a path
	// with .. in it, but be safe anyway.
	path = pathpkg.Clean("/" + path)

	return filepath.Join(string(fs.root), path)
}

func (fs osFS) ReadLink(name string) (string, error) {
	dst, err := os.Readlink(fs.resolve(name))
	if err != nil {
		return "", err
	}
	if !filepath.IsAbs(dst) {
		dst = filepath.Join(fs.resolve(name), dst)
	}
	if !strings.HasPrefix(dst, fs.root+string(os.PathSeparator)) && dst != fs.root {
		return dst, ErrOutsideRoot
	}
	return filepath.Rel(fs.root, dst)
}

func (fs osFS) Symlink(oldname, newname string) error {
	return os.Symlink(fs.resolve(oldname), fs.resolve(newname))
}

// ErrOutsideRoot occurs when a symlink refers to a path that is not in the current VFS.
var ErrOutsideRoot = errors.New("link destination is outside of filesystem")

// Create opens the file at path for writing, creating the file if it doesn't
// exist and truncating it otherwise.
func (fs osFS) Create(path string) (io.WriteCloser, error) {
	f, err := os.OpenFile(fs.resolve(path), os.O_RDWR|os.O_CREATE|os.O_TRUNC, fs.filePerm)
	if err != nil {
		return nil, err
	}

	fi, err := f.Stat()
	if err != nil {
		return nil, err
	}
	if fi.IsDir() {
		return nil, fmt.Errorf("Open: %s is a directory", path)
	}

	return f, nil
}

func (fs osFS) Mkdir(name string) error {
	return os.Mkdir(fs.resolve(name), fs.dirPerm)
}

func (fs osFS) Remove(name string) error {
	return os.Remove(fs.resolve(name))
}
<commit_msg>add option for making OS FS use MkdirAll (since creating parent dirs is tough when you use Sub, etc.)<commit_after>package rwvfs

import (
	"errors"
	"fmt"
	"io"
	"os"
	pathpkg "path"
	"path/filepath"
	"strings"

	"golang.org/x/tools/godoc/vfs"
)

// OS returns an implementation of FileSystem reading from the tree rooted at
// root.
func OS(root string) FileSystem {
	return OSPerm(root, 0666, 0755)
}

func OSPerm(root string, filePerm, dirPerm os.FileMode) FileSystem {
	return &osFS{
		root:       root,
		filePerm:   filePerm,
		dirPerm:    dirPerm,
		FileSystem: vfs.OS(root),
	}
}

type osFS struct {
	root     string
	filePerm os.FileMode
	dirPerm  os.FileMode
	parents  bool
	vfs.FileSystem
}

// CreateParentDirs makes this FS use os.MkdirAll instead of os.Mkdir,
// which means that parent dirs are created as needed and calling
// Mkdir on an existing dir does not return an error.
func (fs *osFS) CreateParentDirs(v bool) {
	fs.parents = v
}

// resolve is from golang.org/x/tools/godoc/vfs.
func (fs *osFS) resolve(path string) string {
	// Clean the path so that it cannot possibly begin with ../.
	// If it did, the result of filepath.Join would be outside the
	// tree rooted at root.  We probably won't ever see a path
	// with .. in it, but be safe anyway.
	path = pathpkg.Clean("/" + path)

	return filepath.Join(string(fs.root), path)
}

func (fs *osFS) ReadLink(name string) (string, error) {
	dst, err := os.Readlink(fs.resolve(name))
	if err != nil {
		return "", err
	}
	if !filepath.IsAbs(dst) {
		dst = filepath.Join(fs.resolve(name), dst)
	}
	if !strings.HasPrefix(dst, fs.root+string(os.PathSeparator)) && dst != fs.root {
		return dst, ErrOutsideRoot
	}
	return filepath.Rel(fs.root, dst)
}

func (fs *osFS) Symlink(oldname, newname string) error {
	return os.Symlink(fs.resolve(oldname), fs.resolve(newname))
}

// ErrOutsideRoot occurs when a symlink refers to a path that is not in the current VFS.
var ErrOutsideRoot = errors.New("link destination is outside of filesystem")

// Create opens the file at path for writing, creating the file if it doesn't
// exist and truncating it otherwise.
func (fs *osFS) Create(path string) (io.WriteCloser, error) {
	f, err := os.OpenFile(fs.resolve(path), os.O_RDWR|os.O_CREATE|os.O_TRUNC, fs.filePerm)
	if err != nil {
		return nil, err
	}

	fi, err := f.Stat()
	if err != nil {
		return nil, err
	}
	if fi.IsDir() {
		return nil, fmt.Errorf("Open: %s is a directory", path)
	}

	return f, nil
}

func (fs *osFS) Mkdir(name string) error {
	if fs.parents {
		return os.MkdirAll(fs.resolve(name), fs.dirPerm)
	}
	return os.Mkdir(fs.resolve(name), fs.dirPerm)
}

func (fs *osFS) Remove(name string) error {
	return os.Remove(fs.resolve(name))
}
<|endoftext|>


<commit_before>// Can be run as follow:
// go install github.com/easytzb/Algorithms/2; ../bin/2 insertion selection insertion_without_exchange insertion_with_sentinel shell 默认三种算法各执行1000，且slice长度为1000
// go install github.com/easytzb/Algorithms/2; ../bin/2 [1000 [1000]] improv_insertion insertion selection
// go install github.com/easytzb/Algorithms/2; ../bin/2 deck|dequeue
package main

import (
	"fmt"
	"math/rand"
	"os"
	"reflect"
	"runtime"
	"strconv"
	"time"
)

func sort(f func([]int), times int, length int) {
	t0 := time.Now()
	for i := 0; i < times; i++ {
		arr := rand.Perm(length)
		f(arr)
	}
	t1 := time.Now()
	fmt.Printf("N:%v\ttimes:%v\t%v\t%v\n", length, times, t1.Sub(t0), runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name())
}

func main() {
	funcs := map[string]func([]int){
		"insertion_without_exchange":               insertion_without_exchange,
		"insertion_without_exchange_with_sentinel": insertion_without_exchange_with_sentinel,
		"insertion":                                insertion,
		"insertion_with_sentinel":                  insertion_with_sentinel,
		"selection":                                selection,
		"bubble":                                   bubble,
		"shell":                                    shell}

	cardFuncs := map[string]func(){
		"deck":    deck,
		"dequeue": dequeue}

	len := len(os.Args)
	startIndex, times, sliceLen := 1, 1000, 1000

	if len < 2 {
		fmt.Println("see usage comments in file")
		os.Exit(2)
	}

	secondArg, err2 := strconv.Atoi(os.Args[1])
	if err2 == nil {
		startIndex = 2
		times = secondArg
		thirdArg, err3 := strconv.Atoi(os.Args[2])
		if err3 == nil {
			startIndex = 3
			sliceLen = thirdArg
		}
	}

	for i := startIndex; i < len; i++ {
		if fun, ok := funcs[os.Args[i]]; ok {
			sort(fun, times, sliceLen)
		} else if fun, ok := cardFuncs[os.Args[i]]; ok {
			fun()
		} else {
			fmt.Printf("Function %v don't exist.\n", os.Args[i])
		}
	}
}
<commit_msg>add cocktail sort<commit_after>// Can be run as follow:
// go install github.com/easytzb/Algorithms/2; ../bin/2 insertion selection insertion_without_exchange insertion_with_sentinel shell 默认三种算法各执行1000，且slice长度为1000
// go install github.com/easytzb/Algorithms/2; ../bin/2 [1000 [1000]] improv_insertion insertion selection
// go install github.com/easytzb/Algorithms/2; ../bin/2 deck|dequeue
package main

import (
	"fmt"
	"math/rand"
	"os"
	"reflect"
	"runtime"
	"strconv"
	"time"
)

func sort(f func([]int), times int, length int) {
	t0 := time.Now()
	for i := 0; i < times; i++ {
		arr := rand.Perm(length)
		f(arr)
	}
	t1 := time.Now()
	fmt.Printf("N:%v\ttimes:%v\t%v\t%v\n", length, times, t1.Sub(t0), runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name())
}

func main() {
	funcs := map[string]func([]int){
		"insertion_without_exchange":               insertion_without_exchange,
		"insertion_without_exchange_with_sentinel": insertion_without_exchange_with_sentinel,
		"insertion":                                insertion,
		"insertion_with_sentinel":                  insertion_with_sentinel,
		"selection":                                selection,
		"bubble":                                   bubble,
		"cocktail":                                 cocktail,
		"shell":                                    shell}

	cardFuncs := map[string]func(){
		"deck":    deck,
		"dequeue": dequeue}

	len := len(os.Args)
	startIndex, times, sliceLen := 1, 1000, 1000

	if len < 2 {
		fmt.Println("see usage comments in file")
		os.Exit(2)
	}

	secondArg, err2 := strconv.Atoi(os.Args[1])
	if err2 == nil {
		startIndex = 2
		times = secondArg
		thirdArg, err3 := strconv.Atoi(os.Args[2])
		if err3 == nil {
			startIndex = 3
			sliceLen = thirdArg
		}
	}

	for i := startIndex; i < len; i++ {
		if fun, ok := funcs[os.Args[i]]; ok {
			sort(fun, times, sliceLen)
		} else if fun, ok := cardFuncs[os.Args[i]]; ok {
			fun()
		} else {
			fmt.Printf("Function %v don't exist.\n", os.Args[i])
		}
	}
}
<|endoftext|>


<commit_before>/*** Copyright (c) 2016, University of Florida Research Foundation, Inc. ***
 *** For more information please refer to the LICENSE.md file            ***/

// Package gorods is a Golang binding for the iRods C API (iRods client library).
// GoRods uses cgo to call iRods client functions.
package gorods

// #include "wrapper.h"
import "C"

import (
	"fmt"
)

type AccessObject interface {
	GetName() string
	GetZone() string
}

type ACL struct {
	AccessObject AccessObject
	AccessLevel  int
	Type         int
}

type ACLs []*ACL

func (acl ACL) GetTypeString(typ int) string {
	switch typ {
	case UserType:
		return "u"
	case GroupType:
		return "g"
	case UnknownType:
		return "?"
	case Null:
		return "null"
	case Read:
		return "read"
	case Write:
		return "write"
	case Own:
		return "own"
	default:
		return ""
	}
}

func (acl ACL) String() string {
	typeString := acl.GetTypeString(acl.Type)

	return fmt.Sprintf("%v:%v#%v:%v", typeString, acl.AccessObject.GetName(), acl.AccessObject.GetZone(), acl.GetTypeString(acl.AccessLevel))
}
<commit_msg>#10 added casting functions to ACL<commit_after>/*** Copyright (c) 2016, University of Florida Research Foundation, Inc. ***
 *** For more information please refer to the LICENSE.md file            ***/

// Package gorods is a Golang binding for the iRods C API (iRods client library).
// GoRods uses cgo to call iRods client functions.
package gorods

// #include "wrapper.h"
import "C"

import (
	"fmt"
)

type AccessObject interface {
	GetName() string
	GetZone() string
}

type ACL struct {
	AccessObject AccessObject
	AccessLevel  int
	Type         int
}

type ACLs []*ACL

func (acl ACL) GetTypeString(typ int) string {
	switch typ {
	case UserType:
		return "u"
	case GroupType:
		return "g"
	case UnknownType:
		return "?"
	case Null:
		return "null"
	case Read:
		return "read"
	case Write:
		return "write"
	case Own:
		return "own"
	default:
		return ""
	}
}

func (acl *ACL) User() *User {
	if acl.Type == UserType {
		return acl.AccessObject.(*User)
	}

	return nil
}

func (acl *ACL) Group() *Group {
	if acl.Type == GroupType {
		return acl.AccessObject.(*Group)
	}

	return nil
}

func (acl *ACL) String() string {
	typeString := acl.GetTypeString(acl.Type)

	return fmt.Sprintf("%v:%v#%v:%v", typeString, acl.AccessObject.GetName(), acl.AccessObject.GetZone(), acl.GetTypeString(acl.AccessLevel))
}
<|endoftext|>


<commit_before>// Copyright 2012 Google, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package ahocorasick provides a simple implementation of Aho-Corasick
// string matching for go.
//
// This implementation allows you to very quickly match a number of
// strings against a stream of input, returning matched strings as they appear.
//
// Example:
//   aho := ahocorasick.NewAhoCorasick([]string{"abc", "abd", "abab", "blah"})
//   var input io.ByteReader = ...
//   for match := range aho.Match(input) {
//     fmt.Printf("Found match %q at index %d\n", match.Value, match.Index)
//   }
// There's also some helper functions so you don't have to convert to an
// io.ByteReader manually:
//   * MatchString - matches against a string
//   * MatchBytes  - matches against a []byte
//   * MatchReader - matches against an io.Reader
// Example:
//   conn := net.Dial("www.google.com:80", "tcp")
//   aho := ahocorasick.NewAhoCorasick([]string{"GET", "POST", "HTTP"})
//   for match := ahocorasick.MatchReader(conn, aho) {
//     fmt.Printf("Found match %q at index %d\n", match.Value, match.Index)
//   }
package ahocorasick

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"sort"
)

// acNode is a node within the Aho-Corasick parse tree.
type acNode struct {
	key          string
	suffix, dict *acNode
	transitions  transitionMap
	dictIndex    *int
}

// acTree stores the root node of the Aho-Corasick parse tree.
type acTree struct {
	root *acNode
}

// Match provides information on a single keyword within a byte-stream.
type Match struct {
	// The dictionary value found in the byte stream
	Value string
	// The start index of the value in the byte stream
	Index int
	// The index of this value in the dictionary used to create this AC matcher
	DictIndex int
}

// transition is used by transitionMap to store transitions within the AC tree.
type transition struct {
	b byte
	n *acNode
}

// transitionMap provides a sorted vector of transitions for nodes in the AC tree,
// with binary search to find the next node for a single byte.
// This seems to take about 50% more time than a [256]*acNode, but with a
// substantial size savings.
// Note:  This idea taken from http://cs.haifa.ac.il/~landau/gadi/shiri.pdf
type transitionMap []*transition

// Implements sort.Interface for our transitionMap
func (v transitionMap) Len() int {
	return len(v)
}
func (v transitionMap) Less(i, j int) bool {
	return v[i].b < v[j].b
}
func (v transitionMap) Swap(i, j int) {
	v[i], v[j] = v[j], v[i]
}

// add adds a transition to the transitionMap
func (v *transitionMap) add(b byte, n *acNode) {
	for _, e := range *v {
		if e.b == b {
			e.n = n
			return
		}
	}
	*v = append(*v, &transition{b, n})
}

// compile sorts our transition map so binary search will work
func (v transitionMap) compile() {
	sort.Sort(v)
}

// get returns the node we should transition to if we see the given byte,
// or nil if we have no viable transition.
func (v transitionMap) get(b byte) *acNode {
	top, bottom := len(v), 0
	for top > bottom {
		i := (top-bottom)/2 + bottom
		b2 := v[i].b
		if b2 > b {
			top = i
		} else if b2 < b {
			bottom = i + 1
		} else {
			return v[i].n
		}
	}
	return nil
}

// AhoCorasick provides Aho-Corsick matching for byte streams.
type AhoCorasick interface {
	// Match reads in bytes from its input and streams matches to the
	// channel it returns.  It closes the channel when it reaches the
	// EOF of the byte reader.
	//
	// Note: matches are returned in the order they're found, NOT the
	// order of their index.  So given the strings 'abcd' and 'b' in the
	// stream 'zabcdef', Match{'b', 2} will be returned BEFORE
	// Match{'abcd', 1}, since 'b' ends first.
	Match(io.ByteReader) chan Match
}

// MatchString is a helpful wrapper to look for matches in a string.
func MatchString(input string, aho AhoCorasick) chan Match {
	return MatchBytes([]byte(input), aho)
}

// MatchBytes is a helpful wrapper to look for matches in a []byte.
func MatchBytes(input []byte, aho AhoCorasick) chan Match {
	return aho.Match(bytes.NewBuffer(input))
}

// MatchReader is a helpful wrapper to look for matches in an io.Reader.
func MatchReader(input io.Reader, aho AhoCorasick) chan Match {
	return aho.Match(bufio.NewReader(input))
}

// NewAhoCorasick creates a new AhoCorasick string matcher that matches
// the given set of strings against byte streams.  Constructs the tree in O(n),
// where n is the total number of characters in all strings in the dictionary.
func NewAhoCorasick(dict []string) AhoCorasick {
	root := &acNode{
		key:         "",
		transitions: transitionMap{},
	}
	tree := &acTree{}
	tree.root = root
	substrs := map[string]*acNode{}
	substrs[""] = root
	// This first for loop constructs the Trie for all of our string elements
	// O(n) time, since the outer loop iterates over all strings, and the inner
	// loop iterates over each character, thus the combination loops once over
	// each character in all strings.
	for index, str := range dict {
		if str == "" {
			panic(errors.New("Can't look for empty string with Aho"))
		}
		from := root
		for j := 1; j <= len(str); j++ {
			substr := str[:j]
			to := substrs[substr]
			if to == nil {
				to = &acNode{
					key:         substr,
					transitions: transitionMap{},
				}
				substrs[substr] = to
			}
			if j == len(str) {
				di := index
				to.dictIndex = &di
			}
			b := str[j-1]
			from.transitions.add(b, to)
			from = to
		}
	}
	// This second for loop post-processes the Trie, making it Aho-Corasick
	// compatible.  It adds suffix and dictionary links to each internal node.
	// Again, this loop takes O(n) time, since the inner loops are done once per
	// character in a string, while the outer loop is done once per string.
	// There's a bunch of map lookups, but since they're hash maps, they're O(1)
	// apiece.
	for str, node := range substrs {
		// Compile all transitions, now that we've added everything.  This takes
		// O(c) time for each node, since the number of transitions is capped at
		// 256 (we're using bytes).
		node.transitions.compile()
		// All nodes WILL have a suffix link except root.  Nodes without other
		// suffix nodes will link to root.
		for i := 1; i <= len(str); i++ {
			suffix := str[i:]
			if sNode, ok := substrs[suffix]; ok {
				node.suffix = sNode
				break
			}
		}
		for i := 1; i < len(str); i++ {
			suffix := str[i:]
			if sNode, ok := substrs[suffix]; ok && sNode.dictIndex != nil {
				node.dict = sNode
				break
			}
		}
	}
	return tree
}

// Print prints out a node in an easy-to-read manner, for debugging.
func (n *acNode) Print() {
	fmt.Printf("Node %q dictIndex:%v\n", n.key, n.dictIndex)
	for _, t := range n.transitions {
		fmt.Printf("  transition: %q -> %q\n", t.b, t.n.key)
	}
	if n.suffix != nil {
		fmt.Printf("  suffix: %q\n", n.suffix.key)
	}
	if n.dict != nil {
		fmt.Printf("  dict: %q\n", n.dict.key)
	}
}

// Match implements the AhoCorasick.Match interface, matching input from
// an io.ByteReader against its internal tree and streaming matces out through
// its returned channel.  The internal tree is not modified during this,
// so code should be fully concurrency-safe.
func (t *acTree) Match(r io.ByteReader) chan Match {
	// We add a little extra space so we don't need to block immediately if we
	// find a string.  For inputs with lots of matches, this seems to have a good
	// affect on speed (~30-35% speedup).  In BenchmarkMatchingManyMatches
	// in aho_test.go, this brings processing time down from 485ns -> 320ns on
	// a 2.53GHz Intel Core i5.  Without matches, we regularly get 20-30ns per
	// character, so many matches still slows us down substantially.
	c := make(chan Match, 20)
	go t.match(r, c)
	return c
}

// match implements the internal logic for running a single match, reading
// in each byte of the ByteReader and traversing the internal AC tree to look
// for matches.
func (t *acTree) match(reader io.ByteReader, output chan Match) {
	current := t.root
	index := 0
	for b, err := reader.ReadByte(); err != io.EOF; b, err = reader.ReadByte() {
		index++
		for current != nil {
			if to := current.transitions.get(b); to != nil {
				current = to
				if current.dictIndex != nil {
					// We've hit a node whose substring is in our dictionary, so output its key
					output <- Match{current.key, index - len(current.key), *current.dictIndex}
				}
				// If this node links to others in the dictionary, output their keys
				for dict := current.dict; dict != nil; dict = dict.dict {
					output <- Match{dict.key, index - len(dict.key), *dict.dictIndex}
				}
				// We've found a transition in the graph, so we're done
				break
			}
			// No transition found yet, so check our suffix
			current = current.suffix
		}
		if current == nil {
			current = t.root
		}
	}
	close(output)
}
<commit_msg>Fix up docs a bit.<commit_after>// Copyright 2012 Google, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package ahocorasick provides a simple implementation of Aho-Corasick
// string matching for go.
//
// This implementation allows you to very quickly match a number of
// strings against a stream of input, returning matched strings as they appear.
//
// Example:
//   aho := ahocorasick.NewAhoCorasick([]string{"abc", "abd", "abab", "blah"})
//   var input io.ByteReader = ...
//   for match := range aho.Match(input) {
//     fmt.Printf("Found match %q at index %d\n", match.Value, match.Index)
//   }
// There's also some helper functions so you don't have to convert to an
// io.ByteReader manually:
//   * MatchString - matches against a string
//   * MatchBytes  - matches against a []byte
//   * MatchReader - matches against an io.Reader
// Example:
//   aho := ahocorasick.NewAhoCorasick([]string{"abc", "abd", "abab", "blah"})
//   for match := range ahocorasick.MatchReader(os.Stdin, aho) {
//     fmt.Printf("Found match %q at index %d\n", match.Value, match.Index)
//   }
package ahocorasick

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"sort"
)

// acNode is a node within the Aho-Corasick parse tree.
type acNode struct {
	key          string
	suffix, dict *acNode
	transitions  transitionMap
	dictIndex    *int
}

// acTree stores the root node of the Aho-Corasick parse tree.
type acTree struct {
	root *acNode
}

// Match provides information on a single keyword within a byte-stream.
type Match struct {
	// The dictionary value found in the byte stream
	Value string
	// The start index of the value in the byte stream
	Index int
	// The index of this value in the dictionary used to create this AC matcher
	DictIndex int
}

// transition is used by transitionMap to store transitions within the AC tree.
type transition struct {
	b byte
	n *acNode
}

// transitionMap provides a sorted vector of transitions for nodes in the AC tree,
// with binary search to find the next node for a single byte.
// This seems to take about 50% more time than a [256]*acNode, but with a
// substantial size savings.
// Note:  This idea taken from http://cs.haifa.ac.il/~landau/gadi/shiri.pdf
type transitionMap []*transition

// Implements sort.Interface for our transitionMap
func (v transitionMap) Len() int {
	return len(v)
}
func (v transitionMap) Less(i, j int) bool {
	return v[i].b < v[j].b
}
func (v transitionMap) Swap(i, j int) {
	v[i], v[j] = v[j], v[i]
}

// add adds a transition to the transitionMap
func (v *transitionMap) add(b byte, n *acNode) {
	for _, e := range *v {
		if e.b == b {
			e.n = n
			return
		}
	}
	*v = append(*v, &transition{b, n})
}

// compile sorts our transition map so binary search will work
func (v transitionMap) compile() {
	sort.Sort(v)
}

// get returns the node we should transition to if we see the given byte,
// or nil if we have no viable transition.
func (v transitionMap) get(b byte) *acNode {
	top, bottom := len(v), 0
	for top > bottom {
		i := (top-bottom)/2 + bottom
		b2 := v[i].b
		if b2 > b {
			top = i
		} else if b2 < b {
			bottom = i + 1
		} else {
			return v[i].n
		}
	}
	return nil
}

// AhoCorasick provides Aho-Corsick matching for byte streams.
type AhoCorasick interface {
	// Match reads in bytes from its input and streams matches to the
	// channel it returns.  It closes the channel when it reaches the
	// EOF of the byte reader.
	//
	// Note: matches are returned in the order they're found, NOT the
	// order of their index.  So given the strings 'abcd' and 'b' in the
	// stream 'zabcdef', Match{'b', 2} will be returned BEFORE
	// Match{'abcd', 1}, since 'b' ends first.
	Match(io.ByteReader) chan Match
}

// MatchString is a helpful wrapper to look for matches in a string.
func MatchString(input string, aho AhoCorasick) chan Match {
	return MatchBytes([]byte(input), aho)
}

// MatchBytes is a helpful wrapper to look for matches in a []byte.
func MatchBytes(input []byte, aho AhoCorasick) chan Match {
	return aho.Match(bytes.NewBuffer(input))
}

// MatchReader is a helpful wrapper to look for matches in an io.Reader.
func MatchReader(input io.Reader, aho AhoCorasick) chan Match {
	return aho.Match(bufio.NewReader(input))
}

// NewAhoCorasick creates a new AhoCorasick string matcher that matches
// the given set of strings against byte streams.  Constructs the tree in O(n),
// where n is the total number of characters in all strings in the dictionary.
func NewAhoCorasick(dict []string) AhoCorasick {
	root := &acNode{
		key:         "",
		transitions: transitionMap{},
	}
	tree := &acTree{}
	tree.root = root
	substrs := map[string]*acNode{}
	substrs[""] = root
	// This first for loop constructs the Trie for all of our string elements
	// O(n) time, since the outer loop iterates over all strings, and the inner
	// loop iterates over each character, thus the combination loops once over
	// each character in all strings.
	for index, str := range dict {
		if str == "" {
			panic(errors.New("Can't look for empty string with Aho"))
		}
		from := root
		for j := 1; j <= len(str); j++ {
			substr := str[:j]
			to := substrs[substr]
			if to == nil {
				to = &acNode{
					key:         substr,
					transitions: transitionMap{},
				}
				substrs[substr] = to
			}
			if j == len(str) {
				di := index
				to.dictIndex = &di
			}
			b := str[j-1]
			from.transitions.add(b, to)
			from = to
		}
	}
	// This second for loop post-processes the Trie, making it Aho-Corasick
	// compatible.  It adds suffix and dictionary links to each internal node.
	// Again, this loop takes O(n) time, since the inner loops are done once per
	// character in a string, while the outer loop is done once per string.
	// There's a bunch of map lookups, but since they're hash maps, they're O(1)
	// apiece.
	for str, node := range substrs {
		// Compile all transitions, now that we've added everything.  This takes
		// O(c) time for each node, since the number of transitions is capped at
		// 256 (we're using bytes).
		node.transitions.compile()
		// All nodes WILL have a suffix link except root.  Nodes without other
		// suffix nodes will link to root.
		for i := 1; i <= len(str); i++ {
			suffix := str[i:]
			if sNode, ok := substrs[suffix]; ok {
				node.suffix = sNode
				break
			}
		}
		for i := 1; i < len(str); i++ {
			suffix := str[i:]
			if sNode, ok := substrs[suffix]; ok && sNode.dictIndex != nil {
				node.dict = sNode
				break
			}
		}
	}
	return tree
}

// Print prints out a node in an easy-to-read manner, for debugging.
func (n *acNode) Print() {
	fmt.Printf("Node %q dictIndex:%v\n", n.key, n.dictIndex)
	for _, t := range n.transitions {
		fmt.Printf("  transition: %q -> %q\n", t.b, t.n.key)
	}
	if n.suffix != nil {
		fmt.Printf("  suffix: %q\n", n.suffix.key)
	}
	if n.dict != nil {
		fmt.Printf("  dict: %q\n", n.dict.key)
	}
}

// Match implements the AhoCorasick.Match interface, matching input from
// an io.ByteReader against its internal tree and streaming matces out through
// its returned channel.  The internal tree is not modified during this,
// so code should be fully concurrency-safe.
func (t *acTree) Match(r io.ByteReader) chan Match {
	// We add a little extra space so we don't need to block immediately if we
	// find a string.  For inputs with lots of matches, this seems to have a good
	// affect on speed (~30-35% speedup).  In BenchmarkMatchingManyMatches
	// in aho_test.go, this brings processing time down from 485ns -> 320ns on
	// a 2.53GHz Intel Core i5.  Without matches, we regularly get 20-30ns per
	// character, so many matches still slows us down substantially.
	c := make(chan Match, 20)
	go t.match(r, c)
	return c
}

// match implements the internal logic for running a single match, reading
// in each byte of the ByteReader and traversing the internal AC tree to look
// for matches.
func (t *acTree) match(reader io.ByteReader, output chan Match) {
	current := t.root
	index := 0
	for b, err := reader.ReadByte(); err != io.EOF; b, err = reader.ReadByte() {
		index++
		for current != nil {
			if to := current.transitions.get(b); to != nil {
				current = to
				if current.dictIndex != nil {
					// We've hit a node whose substring is in our dictionary, so output its key
					output <- Match{current.key, index - len(current.key), *current.dictIndex}
				}
				// If this node links to others in the dictionary, output their keys
				for dict := current.dict; dict != nil; dict = dict.dict {
					output <- Match{dict.key, index - len(dict.key), *dict.dictIndex}
				}
				// We've found a transition in the graph, so we're done
				break
			}
			// No transition found yet, so check our suffix
			current = current.suffix
		}
		if current == nil {
			current = t.root
		}
	}
	close(output)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
)

type CreateLoadBalancerOptions struct {
	AvailabilityZones string
	Listeners         []ListenerOptions
	LoadBalancerName  string
	Scheme            string
	SecurityGroups    []string
	Subnets           []string
}

type CreateLoadBalancerListenersOptions struct {
	Listeners        []ListenerOptions
	LoadBalancerName string
}

type CreateLoadBalancerPolicyOptions struct {
	PolicyAttributes []string
	PolicyName       string
	PolicyTypeName   string
}

type RegisterInstancesWithLoadBalancerOptions struct {
	Instances        []string
	LoadBalancerName string
}

type ListenerOptions struct {
	LoadBalancerPort int
	InstancePort     int
	InstanceProtocol string
	SSLCertificateId string
}

func SetupApiHandlers() {
	http.HandleFunc("/", ELBHandler)
	http.ListenAndServe(":8080", nil)
}

func ELBHandler(w http.ResponseWriter, r *http.Request) {
	action := r.FormValue("Action")
	version := r.FormValue("Version")

	fmt.Println(action, version)

	// This is going to be nasty, but we need to redispatch the
	// request to the correct handler.
	switch action {
	case "CreateLoadBalancer":
		CreateLoadBalancerHandler(w, r)
	case "CreateLoadBalancerListeners":
		CreateLoadBalancerListenersHandler(w, r)
	case "CreateLoadBalancerPolicy":
		CreateLoadBalancerPolicyHandler(w, r)
	case "RegisterInstancesWithLoadBalancer":
		RegisterInstancesWithLoadBalancerHandler(w, r)
	}
}

func CreateLoadBalancerHandler(w http.ResponseWriter, r *http.Request) {
	loadBalancerName := r.FormValue("LoadBalancerName")
	fmt.Println(loadBalancerName)
}

func CreateLoadBalancerListenersHandler(w http.ResponseWriter, r *http.Request) {
	loadBalancerName := r.FormValue("LoadBalancerName")
	fmt.Println(loadBalancerName)
}

func CreateLoadBalancerPolicyHandler(w http.ResponseWriter, r *http.Request) {
	loadBalancerName := r.FormValue("LoadBalancerName")
	fmt.Println(loadBalancerName)
}

func RegisterInstancesWithLoadBalancerHandler(w http.ResponseWriter, r *http.Request) {
	loadBalancerName := r.FormValue("LoadBalancerName")
	fmt.Println(loadBalancerName)
}
<commit_msg>basic logic to read form values into the createlb optionset<commit_after>package main

import (
	"fmt"
	"net/http"
)

type CreateLoadBalancerOptions struct {
	AvailabilityZones string
	Listeners         []ListenerOptions
	LoadBalancerName  string
	Scheme            string
	SecurityGroups    []string
	Subnets           []string
}

type CreateLoadBalancerListenersOptions struct {
	Listeners        []ListenerOptions
	LoadBalancerName string
}

type CreateLoadBalancerPolicyOptions struct {
	PolicyAttributes []string
	PolicyName       string
	PolicyTypeName   string
}

type RegisterInstancesWithLoadBalancerOptions struct {
	Instances        []string
	LoadBalancerName string
}

type ListenerOptions struct {
	LoadBalancerPort string
	Protocol         string
	InstancePort     string
	InstanceProtocol string
	SSLCertificateId string
}

func SetupApiHandlers() {
	http.HandleFunc("/", ELBHandler)
	http.ListenAndServe(":8080", nil)
}

func ELBHandler(w http.ResponseWriter, r *http.Request) {
	action := r.FormValue("Action")
	version := r.FormValue("Version")

	fmt.Println(action, version)

	// This is going to be nasty, but we need to redispatch the
	// request to the correct handler.
	switch action {
	case "CreateLoadBalancer":
		CreateLoadBalancerHandler(w, r)
	case "CreateLoadBalancerListeners":
		CreateLoadBalancerListenersHandler(w, r)
	case "CreateLoadBalancerPolicy":
		CreateLoadBalancerPolicyHandler(w, r)
	case "RegisterInstancesWithLoadBalancer":
		RegisterInstancesWithLoadBalancerHandler(w, r)
	}
}

func CreateLoadBalancerHandler(w http.ResponseWriter, r *http.Request) {

	optionSet := new(CreateLoadBalancerOptions)

	optionSet.LoadBalancerName = r.FormValue("LoadBalancerName")
	optionSet.Scheme = r.FormValue("Scheme")

	listenerSet := ListenerOptions{}
	listenerSet.LoadBalancerPort = r.FormValue("Listeners.member.1.LoadBalancerPort")
	listenerSet.InstancePort = r.FormValue("Listeners.member.1.InstancePort")
	listenerSet.Protocol = r.FormValue("Listeners.member.1.Protocol")
	listenerSet.InstanceProtocol = r.FormValue("Listeners.member.1.InstanceProtocol")

	optionSet.Listeners = append(optionSet.Listeners, listenerSet)

	fmt.Println(optionSet)
}

func CreateLoadBalancerListenersHandler(w http.ResponseWriter, r *http.Request) {
	loadBalancerName := r.FormValue("LoadBalancerName")
	fmt.Println(loadBalancerName)
}

func CreateLoadBalancerPolicyHandler(w http.ResponseWriter, r *http.Request) {
	loadBalancerName := r.FormValue("LoadBalancerName")
	fmt.Println(loadBalancerName)
}

func RegisterInstancesWithLoadBalancerHandler(w http.ResponseWriter, r *http.Request) {
	loadBalancerName := r.FormValue("LoadBalancerName")
	fmt.Println(loadBalancerName)
}
<|endoftext|>


<commit_before>package pvpleaderboardupdater

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

const baseURI string = "https://%s.api.blizzard.com/%s%s"
const oauthURI string = "https://us.battle.net/oauth/token"
const requiredParams string = "?locale=en_US&access_token=%s&namespace=%s"

var clienID string = getEnvVar("BATTLE_NET_CLIENT_ID")
var secret string = getEnvVar("BATTLE_NET_SECRET")
var token string = createToken()

func getStatic(region, path string) *[]byte {
	var namespace = "static-" + region
	var staticPath = "data/wow/" + path
	return get(region, namespace, staticPath)
}

func getDynamic(region, path string) *[]byte {
	var namespace = "dynamic-" + region
	var dynamicPath = "data/wow/" + path
	return get(region, namespace, dynamicPath)
}

func getProfile(region, path string) *[]byte {
	var namespace = "profile-" + region
	var profilePath = "profile/wow/character/" + path
	return get(region, namespace, profilePath)
}

func getMedia(region, path string) *[]byte {
	var namespace = "static-" + region
	var mediaPath = "data/wow/media/" + path
	return get(region, namespace, mediaPath)
}

func getIcon(region, path string) string {
	type AssetJSON struct {
		Key   string
		Value string
	}
	type IconJSON struct {
		Assets []AssetJSON
	}
	var data *[]byte = getMedia(region, path)
	var iconJSON IconJSON
	err := json.Unmarshal(*data, &iconJSON)
	if err != nil {
		logger.Printf("%s json parsing failed: %s", errPrefix, err)
		return ""
	}
	for _, asset := range iconJSON.Assets {
		if asset.Key == "icon" {
			href := asset.Value
			start := strings.LastIndex(href, "/") + 1
			end := strings.LastIndex(href, ".")
			return href[start:end]
		}
	}
	return ""
}

func get(region, namespace, path string) *[]byte {
	var params string = fmt.Sprintf(requiredParams, token, strings.ToLower(namespace))
	var url string = fmt.Sprintf(baseURI, strings.ToLower(region), path, params)
	resp, err := http.Get(url)
	if err != nil {
		logger.Printf("%s GET '%s' failed: %s", errPrefix, path, err)
		return nil
	}
	if resp.StatusCode != 200 {
		logger.Printf("%s non-200 status code for '%s': %v", warnPrefix, path, resp.StatusCode)
		return nil
	}

	body, err := ioutil.ReadAll(resp.Body)
	defer resp.Body.Close()
	if err != nil {
		logger.Printf("%s reading body of '%s' failed: %s", errPrefix, path, err)
		return nil
	}

	return &body
}

func createToken() string {
	d := url.Values{"grant_type": {"client_credentials"}}
	req, err := http.NewRequest("POST", oauthURI, strings.NewReader(d.Encode()))
	if err != nil {
		logger.Printf("%s creating token failed: %s", errPrefix, err)
		return ""
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	req.SetBasicAuth(clienID, secret)
	cli := &http.Client{}
	resp, err := cli.Do(req)
	if err != nil {
		logger.Printf("%s creating token failed: %s", errPrefix, err)
		return ""
	}
	if resp.StatusCode != 200 {
		logger.Printf("%s received %d creating token: %s", errPrefix, resp.StatusCode, resp.Body)
		return ""
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		logger.Printf("%s reading token body failed: %s", errPrefix, err)
		return ""
	}
	var accessTokenResponse = new(AccessTokenResponse)
	err = json.Unmarshal(body, &accessTokenResponse)
	if err != nil {
		logger.Printf("%s unmarshalling token response failed: %s", errPrefix, err)
		return ""
	}

	return accessTokenResponse.Token
}

// AccessTokenResponse : response from an OAuth token request
type AccessTokenResponse struct {
	Token   string `json:"access_token"`
	Type    string `json:"token_type"`
	Expires int    `json:"expires_in"`
}

// Key : API key containing an HREF
type Key struct {
	Href string
}

type KeyedValue struct {
	Key  Key
	Name string
	ID   int
}

// Media : API media link and ID
type Media struct {
	Key Key
	ID  int
}

// TalentJSON : talent ID JSON
type TalentJSON struct {
	ID int
}

// TalentEntryJSON : talent JSON
type TalentEntryJSON struct {
	Talent      TalentJSON
	ColumnIndex int `json:"column_index"`
}

// TalentTierJSON : talent tier JSON
type TalentTierJSON struct {
	Talents   []TalentEntryJSON
	TierIndex int `json:"tier_index"`
}
<commit_msg>sleep and retry after hitting a 429<commit_after>package pvpleaderboardupdater

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"
)

const baseURI string = "https://%s.api.blizzard.com/%s%s"
const oauthURI string = "https://us.battle.net/oauth/token"
const requiredParams string = "?locale=en_US&access_token=%s&namespace=%s"
const rateLimitRetryWaitSeconds int = 2

var clienID string = getEnvVar("BATTLE_NET_CLIENT_ID")
var secret string = getEnvVar("BATTLE_NET_SECRET")
var token string = createToken()

func getStatic(region, path string) *[]byte {
	var namespace = "static-" + region
	var staticPath = "data/wow/" + path
	return get(region, namespace, staticPath)
}

func getDynamic(region, path string) *[]byte {
	var namespace = "dynamic-" + region
	var dynamicPath = "data/wow/" + path
	return get(region, namespace, dynamicPath)
}

func getProfile(region, path string) *[]byte {
	var namespace = "profile-" + region
	var profilePath = "profile/wow/character/" + path
	return get(region, namespace, profilePath)
}

func getMedia(region, path string) *[]byte {
	var namespace = "static-" + region
	var mediaPath = "data/wow/media/" + path
	return get(region, namespace, mediaPath)
}

func getIcon(region, path string) string {
	type AssetJSON struct {
		Key   string
		Value string
	}
	type IconJSON struct {
		Assets []AssetJSON
	}
	var data *[]byte = getMedia(region, path)
	var iconJSON IconJSON
	err := json.Unmarshal(*data, &iconJSON)
	if err != nil {
		logger.Printf("%s json parsing failed: %s", errPrefix, err)
		return ""
	}
	for _, asset := range iconJSON.Assets {
		if asset.Key == "icon" {
			href := asset.Value
			start := strings.LastIndex(href, "/") + 1
			end := strings.LastIndex(href, ".")
			return href[start:end]
		}
	}
	return ""
}

func get(region, namespace, path string) *[]byte {
	var params string = fmt.Sprintf(requiredParams, token, strings.ToLower(namespace))
	var url string = fmt.Sprintf(baseURI, strings.ToLower(region), path, params)
	resp, err := http.Get(url)
	if err != nil {
		logger.Printf("%s GET '%s' failed: %s", errPrefix, path, err)
		return nil
	}
	if resp.StatusCode == 429 {
		logger.Printf("Received 429 - retrying after %d seconds", rateLimitRetryWaitSeconds)
		time.Sleep(time.Duration(rateLimitRetryWaitSeconds) * time.Second)
		return get(region, namespace, path)
	}
	if resp.StatusCode != 200 {
		logger.Printf("%s non-200 status code for '%s': %v", warnPrefix, path, resp.StatusCode)
		return nil
	}

	body, err := ioutil.ReadAll(resp.Body)
	defer resp.Body.Close()
	if err != nil {
		logger.Printf("%s reading body of '%s' failed: %s", errPrefix, path, err)
		return nil
	}

	return &body
}

func createToken() string {
	d := url.Values{"grant_type": {"client_credentials"}}
	req, err := http.NewRequest("POST", oauthURI, strings.NewReader(d.Encode()))
	if err != nil {
		logger.Printf("%s creating token failed: %s", errPrefix, err)
		return ""
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	req.SetBasicAuth(clienID, secret)
	cli := &http.Client{}
	resp, err := cli.Do(req)
	if err != nil {
		logger.Printf("%s creating token failed: %s", errPrefix, err)
		return ""
	}
	if resp.StatusCode != 200 {
		logger.Printf("%s received %d creating token: %s", errPrefix, resp.StatusCode, resp.Body)
		return ""
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		logger.Printf("%s reading token body failed: %s", errPrefix, err)
		return ""
	}
	var accessTokenResponse = new(AccessTokenResponse)
	err = json.Unmarshal(body, &accessTokenResponse)
	if err != nil {
		logger.Printf("%s unmarshalling token response failed: %s", errPrefix, err)
		return ""
	}

	return accessTokenResponse.Token
}

// AccessTokenResponse : response from an OAuth token request
type AccessTokenResponse struct {
	Token   string `json:"access_token"`
	Type    string `json:"token_type"`
	Expires int    `json:"expires_in"`
}

// Key : API key containing an HREF
type Key struct {
	Href string
}

// KeyedValue : API element containing a name, ID, and Key
type KeyedValue struct {
	Key  Key
	Name string
	ID   int
}

// Media : API media link and ID
type Media struct {
	Key Key
	ID  int
}

// TalentJSON : talent ID JSON
type TalentJSON struct {
	ID int
}

// TalentEntryJSON : talent JSON
type TalentEntryJSON struct {
	Talent      TalentJSON
	ColumnIndex int `json:"column_index"`
}

// TalentTierJSON : talent tier JSON
type TalentTierJSON struct {
	Talents   []TalentEntryJSON
	TierIndex int `json:"tier_index"`
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"time"
	"net/http"
	"encoding/json"
)

const AUTH_URI = "/auth/"
const ACCOUNT_URI = "/account/"
const TASKS_URI = "/tasks/"

func main() {
	fmt.Println("hello")
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "hello") })
	http.HandleFunc(AUTH_URI, apiHandler(AUTH_URI, auth_index, auth_post, auth_get, auth_put, auth_delete))
	//http.HandleFunc(ACCOUNT_URI, apiHandler(ACCOUNT_URI, acct_index, acct_post, acct_get, acct_put, acct_delete))
	//http.HandleFunc(TASKS_URI, apiHandler(TASKS_URI, tasks_index, tasks_post, tasks_get, tasks_put, tasks_delete))
	http.ListenAndServe(":8000", nil)
	fmt.Println("goodbye")
}

func apiHandler(root string, index, post func(*http.Request) interface{}, get, put, delete func(*http.Request, string) interface{}) func(http.ResponseWriter, *http.Request) {
	handler := func(w http.ResponseWriter, r *http.Request) {
		var data interface{}
		var error int
		var endpoint_root bool
		error = 0
		endpoint_root = (len(r.URL.Path) <= len(root))

		if endpoint_root {
			if r.Method == "GET" {
				data = index(r)
			} else if r.Method == "POST" {
				data = post(r)
			} else {
				error = 400
				data = "\"bad request\""
			}
		} else {
			id := r.URL.Path[len(root):]
			if r.Method == "GET" {
				data = get(r, id)
			} else if r.Method == "PUT" {
				data = put(r, id)
			} else if r.Method == "DELETE" {
				data = delete(r, id)
			} else {
				error = 400
				data = "\"bad request (not root though)\""
			}
		}
		w.Header().Set("Content-Type", "application/json")
		if error == 0 {
			jdata, _ := json.Marshal(data)
			w.Write(jdata)
		} else {
			http.Error(w, data.(string), error)
		}
	}
	return handler
}

func bad_request(r *http.Request) interface{} {
	return "auth baaaaaaaad"
}

func auth_index(r *http.Request) interface{} {
	return "auth hello"
}
func auth_post(r *http.Request) interface{} {
	return "auth post"
}
func auth_get(r *http.Request, id string) interface{} {
	return "auth get"
}
func auth_put(r *http.Request, id string) interface{} {
	return "auth put"
}
func auth_delete(r *http.Request, id string) interface{} {
	return "auth delete"
}

type Account struct {
	Email string
}

func (a *Account) save() error {
	return nil
}

func (a *Account) update() error {
	return nil
}

func (a *Account) remove() error {
	return nil
}

func LoadAccount(email string) (*Account, error) {
	return &Account{Email: email}, nil
}

type Task struct {
	ID string
	Project string
	Description string
	Duration time.Duration
	Date time.Time
	Creation time.Time
}

func (t *Task) save() error {
	return nil
}

func (t *Task) update() error {
	return nil
}

func (t *Task) remove() error {
	return nil
}

func LoadTask(task_id string) (*Task, error) {
	dur, _ := time.ParseDuration("10s")
	return &Task{ID: task_id,
				 Project: "test",
				 Description: "hello world",
				 Duration: dur,
				 Date: time.Date(2013, time.November, 15, 10, 0, 0, 0, time.UTC),
				 Creation: time.Date(2013, time.November, 15, 11, 0, 0, 0, time.UTC),
				 }, nil
}

func LoadAccountTasks(account_id string) ([]*Task, error) {
	dur, _ := time.ParseDuration("10s")
	return []*Task{&Task{ID: account_id,
			 Project: "test",
			 Description: "hello world",
			 Duration: dur,
			 Date: time.Date(2013, time.November, 15, 10, 0, 0, 0, time.UTC),
			 Creation: time.Date(2013, time.November, 15, 11, 0, 0, 0, time.UTC),
			 }}, nil
}
<commit_msg>dead code<commit_after>package main

import (
	"fmt"
	"time"
	"net/http"
	"encoding/json"
)

const AUTH_URI = "/auth/"
const ACCOUNT_URI = "/account/"
const TASKS_URI = "/tasks/"

func main() {
	fmt.Println("hello")
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "hello") })
	http.HandleFunc(AUTH_URI, apiHandler(AUTH_URI, auth_index, auth_post, auth_get, auth_put, auth_delete))
	//http.HandleFunc(ACCOUNT_URI, apiHandler(ACCOUNT_URI, acct_index, acct_post, acct_get, acct_put, acct_delete))
	//http.HandleFunc(TASKS_URI, apiHandler(TASKS_URI, tasks_index, tasks_post, tasks_get, tasks_put, tasks_delete))
	http.ListenAndServe(":8000", nil)
	fmt.Println("goodbye")
}

func apiHandler(root string, index, post func(*http.Request) interface{}, get, put, delete func(*http.Request, string) interface{}) func(http.ResponseWriter, *http.Request) {
	handler := func(w http.ResponseWriter, r *http.Request) {
		var data interface{}
		var error int
		var endpoint_root bool
		error = 0
		endpoint_root = (len(r.URL.Path) <= len(root))

		if endpoint_root {
			if r.Method == "GET" {
				data = index(r)
			} else if r.Method == "POST" {
				data = post(r)
			} else {
				error = 400
				data = "\"bad request\""
			}
		} else {
			id := r.URL.Path[len(root):]
			if r.Method == "GET" {
				data = get(r, id)
			} else if r.Method == "PUT" {
				data = put(r, id)
			} else if r.Method == "DELETE" {
				data = delete(r, id)
			} else {
				error = 400
				data = "\"bad request (not root though)\""
			}
		}
		w.Header().Set("Content-Type", "application/json")
		if error == 0 {
			jdata, _ := json.Marshal(data)
			w.Write(jdata)
		} else {
			http.Error(w, data.(string), error)
		}
	}
	return handler
}

func auth_index(r *http.Request) interface{} {
	return "auth hello"
}
func auth_post(r *http.Request) interface{} {
	return "auth post"
}
func auth_get(r *http.Request, id string) interface{} {
	return "auth get"
}
func auth_put(r *http.Request, id string) interface{} {
	return "auth put"
}
func auth_delete(r *http.Request, id string) interface{} {
	return "auth delete"
}

type Account struct {
	Email string
}

func (a *Account) save() error {
	return nil
}

func (a *Account) update() error {
	return nil
}

func (a *Account) remove() error {
	return nil
}

func LoadAccount(email string) (*Account, error) {
	return &Account{Email: email}, nil
}

type Task struct {
	ID string
	Project string
	Description string
	Duration time.Duration
	Date time.Time
	Creation time.Time
}

func (t *Task) save() error {
	return nil
}

func (t *Task) update() error {
	return nil
}

func (t *Task) remove() error {
	return nil
}

func LoadTask(task_id string) (*Task, error) {
	dur, _ := time.ParseDuration("10s")
	return &Task{ID: task_id,
				 Project: "test",
				 Description: "hello world",
				 Duration: dur,
				 Date: time.Date(2013, time.November, 15, 10, 0, 0, 0, time.UTC),
				 Creation: time.Date(2013, time.November, 15, 11, 0, 0, 0, time.UTC),
				 }, nil
}

func LoadAccountTasks(account_id string) ([]*Task, error) {
	dur, _ := time.ParseDuration("10s")
	return []*Task{&Task{ID: account_id,
			 Project: "test",
			 Description: "hello world",
			 Duration: dur,
			 Date: time.Date(2013, time.November, 15, 10, 0, 0, 0, time.UTC),
			 Creation: time.Date(2013, time.November, 15, 11, 0, 0, 0, time.UTC),
			 }}, nil
}
<|endoftext|>


<commit_before>package gohttp

import (
	"bytes"
	"encoding/json"
	"fmt"
)

// ApiOutput is sturct data need responsed.
type ApiOutput struct {
	Code int         `json:"code"`
	Msg  string      `json:"msg"`
	Err  error       `json:"err"`
	Data interface{} `json:"data"`
}

// MarshalJSON rewrite format to json, implement json.Marshaler interface.
func (api ApiOutput) MarshalJSON() ([]byte, error) {
	var buf bytes.Buffer
	buf.WriteByte('{')
	fmt.Fprintf(&buf, `"code":%d,`, api.Code)
	fmt.Fprintf(&buf, `"msg":"%s",`, api.Msg)
	fmt.Fprintf(&buf, `"err":"%v",`, api.Err)

	fmt.Fprintf(&buf, `"data":`)
	b, err := json.Marshal(api.Data)
	if err != nil {
		return nil, err
	}
	_, err = buf.Write(b)

	buf.WriteByte('}')

	bytes := buf.Bytes()
	for index, char := range bytes {
		if char == '\t' || char == '\n' || char == '\r' {
			bytes[index] = ' '
		}
	}
	return bytes, err
}

func (api *ApiOutput) Set(code int, msg string) {
	api.Code = code
	api.Msg = msg
}

func (api *ApiOutput) Error() string {
	return fmt.Sprintf("%d: %s", api.Code, api.Msg)
}

func (api *ApiOutput) String() string {
	return fmt.Sprintf("%d: %s|%s", api.Code, api.Msg, api.Err.Error())
}

func (api *ApiOutput) Detail() string {
	return fmt.Sprintf("%d: %s\n%s\n%v", api.Code, api.Msg, api.Err.Error(), api.Data)
}

// ApiHandler designed for http api. It can used easy.
type ApiHandler struct {
	ApiOutput
	HttpHandler
}

func (ctx *ApiHandler) Finish() {
	ctx.Output(ctx.ApiOutput)
}

// RegistErrCode regist api error code and msg.
func RegistErrCode(code int, msg string) error {
	if value, ok := ErrMap[code]; !ok {
		return fmt.Errorf("code[%d] is used, msg=%s", code, value)
	}
	ErrMap[code] = msg
	return nil
}

// ErrMap
var ErrMap = map[int]string{
	0:    "success",
	1000: "success",
	1001: "fail",
	1002: "",
}
<commit_msg>update api json output<commit_after>package gohttp

import (
	"bytes"
	"encoding/json"
	"fmt"
	"strings"
)

// ApiOutput is sturct data need responsed.
type ApiOutput struct {
	Code int         `json:"code"`
	Msg  string      `json:"msg"`
	Err  error       `json:"err"`
	Data interface{} `json:"data"`
}

// MarshalJSON rewrite format to json, implement json.Marshaler interface.
func (api ApiOutput) MarshalJSON() ([]byte, error) {
	var buf bytes.Buffer
	buf.WriteByte('{')
	fmt.Fprintf(&buf, `"code":%d,`, api.Code)
	fmt.Fprintf(&buf, `"msg":"%s",`, api.Msg)
	fmt.Fprintf(&buf, `"err":"%v",`, strings.Replace(fmt.Sprintf("%v", api.Err), "\"", " ", -1))
	fmt.Fprintf(&buf, `"data":`)
	b, err := json.Marshal(api.Data)
	if err != nil {
		return nil, err
	}
	_, err = buf.Write(b)

	buf.WriteByte('}')

	bytes := buf.Bytes()
	for index, char := range bytes {
		if char == '\t' || char == '\n' || char == '\r' {
			bytes[index] = ' '
		}
	}
	return bytes, err
}

func (api *ApiOutput) Set(code int, msg string) {
	api.Code = code
	api.Msg = msg
}

func (api *ApiOutput) Error() string {
	return fmt.Sprintf("%d: %s", api.Code, api.Msg)
}

func (api *ApiOutput) String() string {
	return fmt.Sprintf("%d: %s|%s", api.Code, api.Msg, api.Err.Error())
}

func (api *ApiOutput) Detail() string {
	return fmt.Sprintf("%d: %s\n%s\n%v", api.Code, api.Msg, api.Err.Error(), api.Data)
}

// ApiHandler designed for http api. It can used easy.
type ApiHandler struct {
	ApiOutput
	HttpHandler
}

func (ctx *ApiHandler) Finish() {
	ctx.Output(ctx.ApiOutput)
}

// RegistErrCode regist api error code and msg.
func RegistErrCode(code int, msg string) error {
	if value, ok := ErrMap[code]; !ok {
		return fmt.Errorf("code[%d] is used, msg=%s", code, value)
	}
	ErrMap[code] = msg
	return nil
}

// ErrMap
var ErrMap = map[int]string{
	0:    "success",
	1000: "success",
	1001: "fail",
	1002: "",
}
<|endoftext|>


<commit_before>package main

import (
  "os"
  "github.com/gorilla/mux"
  "net/http"
)

func main() {

  port := os.Getenv("HTTP_PLATFORM_PORT")
  if port == "" {
    port = "3000"
  }

  r := mux.NewRouter()

  r.HandleFunc("/route1", route1Handler)
  r.HandleFunc("/route2/{param}", route2Handler)
  r.HandleFunc("/route3/{param}/{id:[0-9]+}", route3Handler)
  r.HandleFunc("/", homeHandler)

  http.Handle("/", r)
  http.ListenAndServe(":" + port, nil)
}

func route1Handler(w http.ResponseWriter, r *http.Request) {
    w.Write([]byte("Route1Handler"))
}

func route2Handler(w http.ResponseWriter, r *http.Request) {
  param :=  mux.Vars(r)["param"]
  w.Write([]byte("Route2Handler " + param))
}

func route3Handler(w http.ResponseWriter, r *http.Request) {
  param :=  mux.Vars(r)["param"]
  id :=  mux.Vars(r)["id"]

  w.Write([]byte("Route3Handler " + param + " " + id))
}

func homeHandler(w http.ResponseWriter, r *http.Request) {
  w.Write([]byte("HomeHandler updated 3"))
}<commit_msg>updated app<commit_after>package main

import (
  "os"
  "github.com/gorilla/mux"
  "net/http"
)

func main() {

  port := os.Getenv("HTTP_PLATFORM_PORT")
  if port == "" {
    port = "3000"
  }

  r := mux.NewRouter()

  r.HandleFunc("/route1", route1Handler)
  r.HandleFunc("/route2/{param}", route2Handler)
  r.HandleFunc("/route3/{param}/{id:[0-9]+}", route3Handler)
  r.HandleFunc("/", homeHandler)

  http.Handle("/", r)
  http.ListenAndServe(":" + port, nil)
}

func route1Handler(w http.ResponseWriter, r *http.Request) {
    w.Write([]byte("Route1Handler"))
}

func route2Handler(w http.ResponseWriter, r *http.Request) {
  param :=  mux.Vars(r)["param"]
  w.Write([]byte("Route2Handler " + param))
}

func route3Handler(w http.ResponseWriter, r *http.Request) {
  param :=  mux.Vars(r)["param"]
  id :=  mux.Vars(r)["id"]

  w.Write([]byte("Route3Handler " + param + " " + id))
}

func homeHandler(w http.ResponseWriter, r *http.Request) {
  w.Write([]byte("HomeHandler updated 4"))
}<|endoftext|>


<commit_before>package main

import (
//    "log"
    "fmt"
    "net/http"
    "strconv"
    "flag"
    "time"
    "io/ioutil"
    "encoding/json"
)

type Dsp struct {
    DspId   int
    ReqId   string
    SleepMs int
    Status   int
    Price    int
}

// {"id":"gZtaC8svEsDMyXYUukU3GM62gRPZm4mAT7MNINjrnTB3l47gBFbRxmeI4djNejm1","status":1,"price":1000}
type BidResponse struct {
    Id     string
    Status int
    Price  int
}

type Result struct {
    DspId  int
    ReqId  string
    Status  int
    Price   int
}

type Win struct {
    DspId  int
    ReqId  string
    Price  int
}

type Response struct {
    Id      string      `json:"id"`
    DspId   int         `json:"dsp_id"`
    Price   int         `json:"price"`
}

func makeClient() http.Client {
    transport := http.Transport{
        ResponseHeaderTimeout: time.Millisecond * 120,
    }

    client := http.Client{
        Transport: &transport,
    }

    return client
}

func doRequests(dsps []Dsp) <-chan Result {
    receiver := make(chan Result, len(dsps))

    for _, dsp := range dsps {
        go doRequest(dsp, receiver)
    }

    return receiver
}

func doRequest(dsp Dsp, receiver chan Result) {
    client := makeClient()
    url := fmt.Sprintf("http://localhost:8080/ad?id=%s&t=%d&s=%d&p=%d", dsp.ReqId, dsp.SleepMs, dsp.Status, dsp.Price)
    //log.Println(url)
    resp, err := client.Get(url)

    result := Result{}
    if err != nil {
        //log.Println("error")
        result.DspId = dsp.DspId
    } else {
        defer resp.Body.Close()
        body, _ := ioutil.ReadAll(resp.Body)
        //log.Println(string(body))
        var bid_res BidResponse
        err := json.Unmarshal(body, &bid_res)
        if err != nil {
            //log.Println("json parse error")
        }

        result.DspId  = dsp.DspId
        result.ReqId  = bid_res.Id
        result.Status = bid_res.Status
        result.Price  = bid_res.Price
    }

    receiver <- result
}

func doAuction(results []Result) Win {
    win := Win{}
    var max_result Result
    for _, result := range results {
        if result.Status != 1 { continue }

        if max_result.Price == 0 {
            max_result = result
        } else if result.Price > max_result.Price {
            max_result = result
        }
    }

    win.DspId = max_result.DspId
    win.ReqId = max_result.ReqId
    win.Price = max_result.Price

    return win
}

func handler(w http.ResponseWriter, r *http.Request) {
    params     := r.URL.Query()
    id         := params["id"][0]
    dsp_num, _ := strconv.Atoi( params["dsp"][0] )
    dsps       := make([]Dsp, dsp_num)

    // parse dsp parameters
    for i := 0; i < dsp_num; i++ {
        sleep_ms, _ := strconv.Atoi( params[fmt.Sprintf("d%d_t", i)][0] )
        status, _   := strconv.Atoi( params[fmt.Sprintf("d%d_s", i)][0] )
        price       := 0
        price_key   := fmt.Sprintf("d%d_p", i)
        if params[price_key] != nil {
            price, _ = strconv.Atoi( params[price_key][0] )
        }

        dsp     := &Dsp{ i, id, sleep_ms, status, price }
        dsps[i] = *dsp
    }

    // do request to dsps
    results  := make([]Result, len(dsps))
    receiver := doRequests(dsps)

    // receive result
    result_num := 0
    for {
        result := <-receiver
        //log.Println(result)
        results[result.DspId] = result

        result_num++
        if len(results) == result_num { break }
    }
    //log.Println(results)

    win := doAuction(results)
    //log.Println(win)

    w.Header().Set("Content-Type", "application/json")
    response := Response{
        Id: win.ReqId,
        DspId: win.DspId,
        Price: win.Price,
    }
    bytes, _ := json.Marshal(response)
    json_str := string(bytes)
//    log.Println(json_str)
    fmt.Fprint(w, json_str)
}

func main() {
    port := flag.Int("port", 5000, "PORT")
    flag.Parse()

    http.HandleFunc("/ad", handler)
    http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)
}
<commit_msg>とりあえず<commit_after>package main

import (
    "log"
    "fmt"
    "net/http"
    "strconv"
    "flag"
    "time"
    "io/ioutil"
    "encoding/json"
)

type RTBHTTPHandler struct {
    m *http.Handler
}

func (h *RTBHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
    params     := r.URL.Query()
    id         := params["id"][0]
    dsp_num, _ := strconv.Atoi( params["dsp"][0] )
    dsps       := make([]Dsp, dsp_num)

    // parse dsp parameters
    for i := 0; i < dsp_num; i++ {
        sleep_ms, _ := strconv.Atoi( params[fmt.Sprintf("d%d_t", i)][0] )
        status, _   := strconv.Atoi( params[fmt.Sprintf("d%d_s", i)][0] )
        price       := 0
        price_key   := fmt.Sprintf("d%d_p", i)
        if params[price_key] != nil {
            price, _ = strconv.Atoi( params[price_key][0] )
        }

        dsp     := &Dsp{ i, id, sleep_ms, status, price }
        dsps[i] = *dsp
    }

    // do request to dsps
    results  := make([]Result, len(dsps))
    receiver := doRequests(dsps)

    // receive result
    result_num := 0
    for {
        result := <-receiver
        //log.Println(result)
        results[result.DspId] = result

        result_num++
        if len(results) == result_num { break }
    }
    //log.Println(results)

    win := doAuction(results)
    //log.Println(win)

    w.Header().Set("Content-Type", "application/json")
    response := Response{
        Id: win.ReqId,
        DspId: win.DspId,
        Price: win.Price,
    }
    bytes, _ := json.Marshal(response)
    json_str := string(bytes)
//    log.Println(json_str)
    fmt.Fprint(w, json_str)

}

type Dsp struct {
    DspId   int
    ReqId   string
    SleepMs int
    Status   int
    Price    int
}

// {"id":"gZtaC8svEsDMyXYUukU3GM62gRPZm4mAT7MNINjrnTB3l47gBFbRxmeI4djNejm1","status":1,"price":1000}
type BidResponse struct {
    Id     string
    Status int
    Price  int
}

type Result struct {
    DspId  int
    ReqId  string
    Status  int
    Price   int
}

type Win struct {
    DspId  int
    ReqId  string
    Price  int
}

type Response struct {
    Id      string      `json:"id"`
    DspId   int         `json:"dsp_id"`
    Price   int         `json:"price"`
}

func makeClient() http.Client {
    transport := http.Transport{
        ResponseHeaderTimeout: time.Millisecond * 120,
    }

    client := http.Client{
        Transport: &transport,
    }

    return client
}

func doRequests(dsps []Dsp) <-chan Result {
    receiver := make(chan Result, len(dsps))

    for _, dsp := range dsps {
        go doRequest(dsp, receiver)
    }

    return receiver
}

func doRequest(dsp Dsp, receiver chan Result) {
    client := makeClient()
    url := fmt.Sprintf("http://localhost:8080/ad?id=%s&t=%d&s=%d&p=%d", dsp.ReqId, dsp.SleepMs, dsp.Status, dsp.Price)
    //log.Println(url)
    resp, err := client.Get(url)

    result := Result{}
    if err != nil {
        //log.Println("error")
        result.DspId = dsp.DspId
    } else {
        defer resp.Body.Close()
        body, _ := ioutil.ReadAll(resp.Body)
        //log.Println(string(body))
        var bid_res BidResponse
        err := json.Unmarshal(body, &bid_res)
        if err != nil {
            //log.Println("json parse error")
        }

        result.DspId  = dsp.DspId
        result.ReqId  = bid_res.Id
        result.Status = bid_res.Status
        result.Price  = bid_res.Price
    }

    receiver <- result
}

func doAuction(results []Result) Win {
    win := Win{}
    var max_result Result
    for _, result := range results {
        if result.Status != 1 { continue }

        if max_result.Price == 0 {
            max_result = result
        } else if result.Price > max_result.Price {
            max_result = result
        }
    }

    win.DspId = max_result.DspId
    win.ReqId = max_result.ReqId
    win.Price = max_result.Price

    return win
}

func handler(w http.ResponseWriter, r *http.Request) {
    params     := r.URL.Query()
    id         := params["id"][0]
    dsp_num, _ := strconv.Atoi( params["dsp"][0] )
    dsps       := make([]Dsp, dsp_num)

    // parse dsp parameters
    for i := 0; i < dsp_num; i++ {
        sleep_ms, _ := strconv.Atoi( params[fmt.Sprintf("d%d_t", i)][0] )
        status, _   := strconv.Atoi( params[fmt.Sprintf("d%d_s", i)][0] )
        price       := 0
        price_key   := fmt.Sprintf("d%d_p", i)
        if params[price_key] != nil {
            price, _ = strconv.Atoi( params[price_key][0] )
        }

        dsp     := &Dsp{ i, id, sleep_ms, status, price }
        dsps[i] = *dsp
    }

    // do request to dsps
    results  := make([]Result, len(dsps))
    receiver := doRequests(dsps)

    // receive result
    result_num := 0
    for {
        result := <-receiver
        //log.Println(result)
        results[result.DspId] = result

        result_num++
        if len(results) == result_num { break }
    }
    //log.Println(results)

    win := doAuction(results)
    //log.Println(win)

    w.Header().Set("Content-Type", "application/json")
    response := Response{
        Id: win.ReqId,
        DspId: win.DspId,
        Price: win.Price,
    }
    bytes, _ := json.Marshal(response)
    json_str := string(bytes)
//    log.Println(json_str)
    fmt.Fprint(w, json_str)
}

func main() {
    port := flag.Int("port", 5000, "PORT")
    flag.Parse()

    rtbHandler := RTBHTTPHandler{}
    s := &http.Server{
        Addr:    fmt.Sprintf(":%d", *port),
        Handler: &rtbHandler,
        ReadTimeout: 500 * time.Millisecond,
    }

    //http.HandleFunc("/ad", handler)
    //log.Fatal( http.ListenAndServe(fmt.Sprintf(":%d", *port), nil) )
    log.Fatal( s.ListenAndServe() )
}
<|endoftext|>